diff --git a/scripts/setup-dns-proxy.sh b/scripts/setup-dns-proxy.sh index 6f08f581afc..ff8ba4a7e25 100755 --- a/scripts/setup-dns-proxy.sh +++ b/scripts/setup-dns-proxy.sh @@ -2,329 +2,23 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Set up a DNS forwarder inside the sandbox pod so the isolated sandbox -# network namespace can resolve hostnames. -# -# Problem: The sandbox runs in an isolated namespace (10.200.0.0/24) -# where all non-proxy traffic is rejected by iptables. DNS (UDP:53) -# is blocked, causing getaddrinfo EAI_AGAIN for every outbound request. -# -# Fix (three steps): -# 1. Run a Python DNS forwarder on the pod-side veth gateway IP -# (10.200.0.1:53), forwarding to the real CoreDNS pod IP. -# 2. Add an iptables rule in the sandbox namespace to allow UDP -# to the gateway on port 53 (the only non-proxy exception). -# Sandbox images may not have iptables on PATH, so we probe -# well-known paths (/sbin, /usr/sbin) to find the binary. -# 3. Update the sandbox's /etc/resolv.conf to point to 10.200.0.1. -# -# Requires: sandbox must be in Ready state. Run after sandbox creation. +# Compatibility wrapper for the TypeScript sandbox DNS proxy setup. # # Usage: ./scripts/setup-dns-proxy.sh [gateway-name] set -euo pipefail -GATEWAY_NAME="${1:-}" -SANDBOX_NAME="${2:-}" - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -# shellcheck source=./lib/runtime.sh -. "$SCRIPT_DIR/lib/runtime.sh" - -if [ -z "$SANDBOX_NAME" ]; then +if [ "$#" -lt 2 ]; then echo "Usage: $0 [gateway-name] " exit 1 fi -# ── Find the gateway container ────────────────────────────────────── - -if [ -z "${DOCKER_HOST:-}" ]; then - if docker_host="$(detect_docker_host)"; then - export DOCKER_HOST="$docker_host" - fi -fi - -CLUSTERS="$(docker ps --filter "name=openshell-cluster" --format '{{.Names}}' 2>/dev/null || true)" -CLUSTER="$(select_openshell_cluster_container "$GATEWAY_NAME" "$CLUSTERS" || true)" - -if [ -z "$CLUSTER" ]; then - if [ -n "$GATEWAY_NAME" ]; then - echo "WARNING: Could not find gateway container for '$GATEWAY_NAME'. DNS proxy not installed." - else - echo "WARNING: Could not find any openshell cluster container. DNS proxy not installed." - fi - exit 1 -fi - -# ── Helper: kubectl via gateway ───────────────────────────────────── - -kctl() { - # Target the `agent` container explicitly for `exec` so kubectl stops - # emitting `Defaulted container "agent" out of: agent, workspace-init (init)` - # on every call. - if [ "${1:-}" = "exec" ]; then - shift - docker exec "$CLUSTER" kubectl exec -c agent "$@" - else - docker exec "$CLUSTER" kubectl "$@" - fi -} - -# ── Discover CoreDNS pod IP ───────────────────────────────────────── -# -# Forward to the real CoreDNS pod (not 8.8.8.8) so k8s-internal names -# like openshell-0.openshell.svc.cluster.local still resolve. CoreDNS -# handles both k8s names (kubernetes plugin) and external names -# (forward plugin, patched by fix-coredns.sh). - -DNS_UPSTREAM="$(kctl get endpoints kube-dns \ - -n kube-system -o jsonpath='{.subsets[0].addresses[0].ip}' 2>/dev/null || true)" - -if [ -z "$DNS_UPSTREAM" ]; then - echo "WARNING: Could not discover CoreDNS pod IP. Falling back to 8.8.8.8." - echo "WARNING: k8s-internal names (inference.local routing) will NOT work." - DNS_UPSTREAM="8.8.8.8" -fi - -# ── Find the sandbox pod ──────────────────────────────────────────── - -POD="$(kctl get pods -n openshell -o name 2>/dev/null \ - | grep -F -- "$SANDBOX_NAME" | head -1 | sed 's|pod/||' || true)" - -if [ -z "$POD" ]; then - echo "WARNING: Could not find pod for sandbox '$SANDBOX_NAME'. DNS proxy not installed." - exit 1 -fi - -# ── Discover the pod-side veth gateway IP ─────────────────────────── -# -# The sandbox connects to the pod via a veth pair. The pod side is -# typically 10.200.0.1. The forwarder must listen on this IP so -# packets from the sandbox (10.200.0.2) can reach it. - -VETH_GW="$(kctl exec -n openshell "$POD" -- sh -c \ - "ip addr show | grep 'inet 10\\.200\\.0\\.' | awk '{print \$2}' | cut -d/ -f1" \ - 2>/dev/null || true)" -VETH_GW="${VETH_GW:-10.200.0.1}" - -echo "Setting up DNS proxy in pod '$POD' (${VETH_GW}:53 -> ${DNS_UPSTREAM})..." - -# ── Step 1: Write DNS forwarder to the pod ────────────────────────── - -kctl exec -n openshell "$POD" -- sh -c "cat > /tmp/dns-proxy.py << 'DNSPROXY' -import socket, threading, os, sys - -UPSTREAM = (sys.argv[1] if len(sys.argv) > 1 else '8.8.8.8', 53) -BIND_IP = sys.argv[2] if len(sys.argv) > 2 else '0.0.0.0' - -sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) -sock.bind((BIND_IP, 53)) - -with open('/tmp/dns-proxy.pid', 'w') as pf: - pf.write(str(os.getpid())) - -msg = 'dns-proxy: {}:53 -> {}:{} pid={}'.format(BIND_IP, UPSTREAM[0], UPSTREAM[1], os.getpid()) -print(msg, flush=True) -with open('/tmp/dns-proxy.log', 'w') as log: - log.write(msg + '\n') - -def forward(data, addr): - try: - f = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - f.settimeout(5) - f.sendto(data, UPSTREAM) - r, _ = f.recvfrom(4096) - sock.sendto(r, addr) - f.close() - except Exception: - pass - -while True: - d, a = sock.recvfrom(4096) - threading.Thread(target=forward, args=(d, a), daemon=True).start() -DNSPROXY" - -# ── Step 2: Kill any existing DNS proxy ───────────────────────────── - -OLD_PID="$(kctl exec -n openshell "$POD" -- cat /tmp/dns-proxy.pid 2>/dev/null || true)" -if [ -n "$OLD_PID" ]; then - kctl exec -n openshell "$POD" -- kill "$OLD_PID" 2>/dev/null || true - sleep 1 -fi - -# ── Step 3: Launch forwarder on pod-side veth gateway ─────────────── -# -# Use kubectl exec with nohup to start the forwarder as a background -# process inside the pod. This avoids the nsenter PID namespace -# mismatch that caused PR #732's launch to silently fail. -# -# Bind on the pod-side veth IP so the sandbox namespace can reach it -# once the iptables UDP exception is in place. - -kctl exec -n openshell "$POD" -- \ - sh -c "nohup python3 -u /tmp/dns-proxy.py '${DNS_UPSTREAM}' '${VETH_GW}' \ - > /tmp/dns-proxy.log 2>&1 &" - -# Wait for forwarder to actually be serving (up to 10s). -# The PID file is written before the socket is bound, so we probe -# with a real DNS query instead of just checking the file. See #2017. -_dns_ready=0 -for _i in $(seq 1 10); do - # Probe via python3, not socat — socat is not installed in the sandbox image. - if kctl exec -n openshell "$POD" -- python3 -c " -import socket, sys -s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -s.settimeout(1) -try: - s.sendto(b'\x00\x1e\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x06google\x03com\x00\x00\x01\x00\x01', - ('${VETH_GW}', 53)) - data, _ = s.recvfrom(4096) - sys.stdout.write('ok' if data else '') -except Exception: - pass -" 2>/dev/null | grep -q ok; then - _dns_ready=1 - break - fi - sleep 1 -done -if [ "$_dns_ready" -eq 0 ]; then - echo "WARNING: DNS forwarder not responding after 10s — verification may fail" -fi - -# ── Step 4: Allow UDP DNS in sandbox iptables ─────────────────────── -# -# OpenShell's sandbox network policy rejects all non-proxy traffic -# (only TCP to 10.200.0.1:3128 is allowed). Insert a rule at the top -# of the OUTPUT chain to allow UDP to the gateway on port 53. -# -# Sandbox images may not have iptables on PATH (e.g. minimal images -# ship it in /sbin or /usr/sbin without updating PATH). We run -# `ip netns exec` from the *pod*, so the binary is resolved from the -# pod's filesystem — probe well-known paths to find it. See #557. - -SANDBOX_NS="$(kctl exec -n openshell "$POD" -- sh -c \ - "ls /run/netns/ 2>/dev/null | grep sandbox | head -1" 2>/dev/null || true)" - -if [ -z "$SANDBOX_NS" ]; then - echo "WARNING: Could not find sandbox network namespace. DNS may not work." -else - # Find iptables binary — check PATH first, then well-known locations. - # The sandbox image may not include iptables at all, but the pod's - # root filesystem (which ip-netns-exec inherits) usually has it in - # /sbin or /usr/sbin even when those dirs are not on PATH. - IPTABLES_BIN="" - for candidate in iptables /sbin/iptables /usr/sbin/iptables; do - if kctl exec -n openshell "$POD" -- sh -c "test -x \"\$(command -v $candidate 2>/dev/null || echo $candidate)\"" 2>/dev/null; then - IPTABLES_BIN="$candidate" - break - fi - done - - # Back up the original resolv.conf before we touch it. On reruns the - # file may already contain our rewritten content, so only save once. - kctl exec -n openshell "$POD" -- \ - ip netns exec "$SANDBOX_NS" sh -c " - [ -f /tmp/resolv.conf.orig ] || cp /etc/resolv.conf /tmp/resolv.conf.orig - " 2>/dev/null || true - - if [ -n "$IPTABLES_BIN" ]; then - kctl exec -n openshell "$POD" -- \ - ip netns exec "$SANDBOX_NS" \ - "$IPTABLES_BIN" -C OUTPUT -p udp -d "$VETH_GW" --dport 53 -j ACCEPT 2>/dev/null \ - || kctl exec -n openshell "$POD" -- \ - ip netns exec "$SANDBOX_NS" \ - "$IPTABLES_BIN" -I OUTPUT 1 -p udp -d "$VETH_GW" --dport 53 -j ACCEPT - - # ── Step 5: Update sandbox resolv.conf ────────────────────────── - # Only rewrite resolv.conf when the iptables rule was added. - # Without the UDP exception, pointing resolv.conf at the forwarder - # would make DNS queries silently time out instead of failing fast - # with the system default resolver — a worse failure mode. - kctl exec -n openshell "$POD" -- \ - ip netns exec "$SANDBOX_NS" sh -c " - printf 'nameserver ${VETH_GW}\noptions ndots:5\n' > /etc/resolv.conf - " - else - echo "WARNING: iptables not found in pod (checked PATH, /sbin, /usr/sbin)." - echo "WARNING: Cannot add UDP DNS exception. Sandbox DNS resolution will not work." - # Restore original resolv.conf in case a previous run overwrote it. - kctl exec -n openshell "$POD" -- \ - ip netns exec "$SANDBOX_NS" sh -c " - [ -f /tmp/resolv.conf.orig ] && cp /tmp/resolv.conf.orig /etc/resolv.conf - " 2>/dev/null || true - fi -fi - -# ── Step 6: Runtime verification ───────────────────────────────────── -# -# Verify all three layers of the DNS bridge actually work, not just -# that the forwarder process started. This catches silent failures that -# static checks miss. - -VERIFY_PASS=0 -VERIFY_FAIL=0 - -# 6a. Forwarder process running -PID="$(kctl exec -n openshell "$POD" -- cat /tmp/dns-proxy.pid 2>/dev/null || true)" -LOG="$(kctl exec -n openshell "$POD" -- cat /tmp/dns-proxy.log 2>/dev/null || true)" - -if [ -n "$PID" ] && echo "$LOG" | grep -q "dns-proxy:"; then - echo " [PASS] DNS forwarder running (pid=$PID): $LOG" - VERIFY_PASS=$((VERIFY_PASS + 1)) -else - echo " [FAIL] DNS forwarder not running. PID=${PID:-none} Log: ${LOG:-empty}" - VERIFY_FAIL=$((VERIFY_FAIL + 1)) -fi - -# 6b-6d run inside sandbox namespace (require SANDBOX_NS) -if [ -n "$SANDBOX_NS" ]; then - sb_exec() { - kctl exec -n openshell "$POD" -- ip netns exec "$SANDBOX_NS" "$@" - } - - # 6b. resolv.conf points to the veth gateway - RESOLV="$(sb_exec cat /etc/resolv.conf 2>/dev/null || true)" - if echo "$RESOLV" | grep -q "nameserver ${VETH_GW}"; then - echo " [PASS] resolv.conf -> nameserver ${VETH_GW}" - VERIFY_PASS=$((VERIFY_PASS + 1)) - else - echo " [FAIL] resolv.conf does not point to ${VETH_GW}: ${RESOLV}" - VERIFY_FAIL=$((VERIFY_FAIL + 1)) - fi - - # 6c. iptables UDP DNS rule present (use discovered binary path) - IPTABLES_CHECK="${IPTABLES_BIN:-iptables}" - if sb_exec "$IPTABLES_CHECK" -C OUTPUT -p udp -d "$VETH_GW" --dport 53 -j ACCEPT 2>/dev/null; then - echo " [PASS] iptables: UDP ${VETH_GW}:53 ACCEPT rule present" - VERIFY_PASS=$((VERIFY_PASS + 1)) - else - echo " [FAIL] iptables: UDP DNS ACCEPT rule missing" - VERIFY_FAIL=$((VERIFY_FAIL + 1)) - fi +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +CLI_JS="${NEMOCLAW_CLI_JS:-$REPO_ROOT/dist/nemoclaw.js}" - # 6d. Actual DNS resolution from sandbox (getent hosts) - # Retry up to 3 times — on slower hardware (Jetson ARM64) the forwarder - # may need a few extra seconds after binding. See #2017. - DNS_RESULT="" - for _dns_try in 1 2 3; do - DNS_RESULT="$(sb_exec getent hosts github.com 2>/dev/null || true)" - [ -n "$DNS_RESULT" ] && break - [ "$_dns_try" -lt 3 ] && sleep 2 - done - if [ -n "$DNS_RESULT" ]; then - echo " [PASS] getent hosts github.com -> ${DNS_RESULT}" - VERIFY_PASS=$((VERIFY_PASS + 1)) - else - echo " [FAIL] getent hosts github.com returned empty after 3 attempts (DNS not resolving)" - VERIFY_FAIL=$((VERIFY_FAIL + 1)) - fi -else - echo " [SKIP] Sandbox namespace not found; cannot verify resolv.conf, iptables, or DNS" +if [ -f "$CLI_JS" ]; then + exec node "$CLI_JS" internal dns setup-proxy "$@" fi -echo " DNS verification: ${VERIFY_PASS} passed, ${VERIFY_FAIL} failed" -if [ "$VERIFY_FAIL" -gt 0 ]; then - echo "WARNING: DNS setup incomplete. Sandbox DNS resolution may not work. See issue #626, #557." -fi +exec nemoclaw internal dns setup-proxy "$@" diff --git a/src/commands/internal/dns/setup-proxy.ts b/src/commands/internal/dns/setup-proxy.ts new file mode 100644 index 00000000000..ab70f982d47 --- /dev/null +++ b/src/commands/internal/dns/setup-proxy.ts @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Args, Command, Flags } from "@oclif/core"; + +import { runSetupDnsProxy } from "../../../lib/actions/dns"; + +export default class InternalDnsSetupProxyCommand extends Command { + static hidden = true; + static strict = true; + static summary = "Internal: configure sandbox DNS proxy"; + static description = "Configure the DNS forwarder bridge inside a sandbox pod."; + static usage = ["internal dns setup-proxy "]; + static examples = ["<%= config.bin %> internal dns setup-proxy nemoclaw my-sandbox"]; + static args = { + gatewayName: Args.string({ description: "OpenShell gateway name", required: true }), + sandboxName: Args.string({ description: "Sandbox name", required: true }), + }; + static flags = { + help: Flags.help({ char: "h" }), + }; + + public async run(): Promise { + const { args } = await this.parse(InternalDnsSetupProxyCommand); + const result = runSetupDnsProxy({ gatewayName: args.gatewayName, sandboxName: args.sandboxName }); + if (result.exitCode !== 0) { + if (result.message) console.error(result.message); + process.exit(result.exitCode); + } + } +} diff --git a/src/lib/actions/dns.test.ts b/src/lib/actions/dns.test.ts index 4c3ca1ccec8..9d64091971e 100644 --- a/src/lib/actions/dns.test.ts +++ b/src/lib/actions/dns.test.ts @@ -3,7 +3,8 @@ import { describe, expect, it, vi } from "vitest"; -import { runFixCoreDns, type CommandResult } from "./dns"; +import { runFixCoreDns, runSetupDnsProxy } from "../../../dist/lib/actions/dns.js"; +import type { CommandResult } from "./dns"; function ok(stdout = ""): CommandResult { return { status: 0, stdout, stderr: "" }; @@ -129,3 +130,65 @@ describe("runFixCoreDns", () => { expect(result.message).toBe("patch failed"); }); }); + +describe("runSetupDnsProxy", () => { + it("configures the DNS proxy through kubectl-in-docker argv calls", () => { + const calls: string[][] = []; + const log = vi.fn(); + const runDocker = vi.fn((args: string[]) => { + calls.push(args); + const cmd = args.join(" "); + if (args[0] === "ps") return ok("openshell-cluster-nemoclaw\n"); + if (cmd.includes("get endpoints kube-dns")) return ok("10.43.0.10"); + if (cmd.includes("get pods -n openshell -o name")) return ok("pod/box[1]-abc\n"); + if (cmd.includes("ip addr show")) return ok("10.200.0.1\n"); + if (cmd.includes("cat /tmp/dns-proxy.pid")) return ok("12345\n"); + if (cmd.includes("cat /tmp/dns-proxy.log")) return ok("dns-proxy: 10.200.0.1:53 -> 10.43.0.10:53 pid=12345\n"); + if (cmd.includes("python3 -c")) return ok("ok"); + if (cmd.includes("ls /run/netns/")) return ok("sandbox-ns\n"); + if (cmd.includes("test -x")) return ok(); + if (cmd.includes("cat /etc/resolv.conf")) return ok("nameserver 10.200.0.1\n"); + if (cmd.includes("getent hosts github.com")) return ok("140.82.112.4 github.com\n"); + return ok(); + }); + + const result = runSetupDnsProxy( + { gatewayName: "nemoclaw", sandboxName: "box[1]" }, + { + env: { DOCKER_HOST: "unix:///tmp/fake-docker.sock" }, + log, + runDocker, + sleep: vi.fn(), + }, + ); + + expect(result.exitCode).toBe(0); + expect(result.pod).toBe("box[1]-abc"); + expect(result.verificationPass).toBe(4); + expect(calls.some((args) => args.includes("box[1]-abc"))).toBe(true); + expect(calls.some((args) => args.join(" ").includes("nohup python3 -u /tmp/dns-proxy.py '10.43.0.10' '10.200.0.1'"))).toBe(true); + expect(log).toHaveBeenCalledWith(" DNS verification: 4 passed, 0 failed"); + }); + + it("rejects unsafe DNS upstreams before launching the forwarder", () => { + const calls: string[][] = []; + const result = runSetupDnsProxy( + { gatewayName: "nemoclaw", sandboxName: "box" }, + { + env: { DOCKER_HOST: "unix:///tmp/fake-docker.sock" }, + runDocker: (args) => { + calls.push(args); + const cmd = args.join(" "); + if (args[0] === "ps") return ok("openshell-cluster-nemoclaw\n"); + if (cmd.includes("get endpoints kube-dns")) return ok("bad;rm\n"); + return ok(); + }, + sleep: vi.fn(), + }, + ); + + expect(result.exitCode).toBe(1); + expect(result.message).toContain("contains invalid characters"); + expect(calls.some((args) => args.join(" ").includes("dns-proxy.py"))).toBe(false); + }); +}); diff --git a/src/lib/actions/dns.ts b/src/lib/actions/dns.ts index 263e7fc71b6..4cd8f4f72ac 100644 --- a/src/lib/actions/dns.ts +++ b/src/lib/actions/dns.ts @@ -15,6 +15,16 @@ import { selectOpenshellClusterContainer, type ContainerRuntime, } from "../domain/dns/coredns"; +import { + buildDnsProxyPython, + buildDnsReadyProbePython, + buildResolvConf, + DEFAULT_DNS_UPSTREAM, + isSafeDnsAddress, + parseVethGateway, + selectSandboxNamespace, + selectSandboxPod, +} from "../domain/dns/setup-proxy"; export type CommandResult = Pick, "stderr" | "stdout" | "status">; @@ -34,6 +44,27 @@ export interface FixCoreDnsOptions { gatewayName?: string; } +export interface SetupDnsProxyOptions { + gatewayName: string; + sandboxName: string; +} + +export interface SetupDnsProxyDeps extends FixCoreDnsDeps { + sleep?: (ms: number) => void; +} + +export interface SetupDnsProxyResult { + cluster?: string; + dnsUpstream?: string; + exitCode: number; + message?: string; + pod?: string; + sandboxNamespace?: string; + verificationFail?: number; + verificationPass?: number; + vethGateway?: string; +} + export interface FixCoreDnsResult { cluster?: string; exitCode: number; @@ -111,6 +142,26 @@ function defaultRunDocker(args: string[], options: { env?: NodeJS.ProcessEnv } = }; } +function sleepSync(ms: number): void { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function kctl( + runDocker: (args: string[], options?: { env?: NodeJS.ProcessEnv }) => CommandResult, + cluster: string, + args: string[], + env: NodeJS.ProcessEnv, +): CommandResult { + if (args[0] === "exec") { + return runDocker(["exec", cluster, "kubectl", "exec", "-c", "agent", ...args.slice(1)], { env }); + } + return runDocker(["exec", cluster, "kubectl", ...args], { env }); +} + +function shellSingleQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'`; +} + function getColimaVmResolvConf(deps: FixCoreDnsDeps, env: NodeJS.ProcessEnv): string { const commandExists = deps.commandExists ?? defaultCommandExists; if (!commandExists("colima")) return ""; @@ -208,3 +259,291 @@ export function runFixCoreDns( log("Done. DNS should resolve in ~10 seconds."); return { cluster, exitCode: 0, runtime: detected.runtime, upstreamDns }; } + +export function runSetupDnsProxy( + options: SetupDnsProxyOptions, + deps: SetupDnsProxyDeps = {}, +): SetupDnsProxyResult { + const env = { ...process.env, ...(deps.env ?? {}) }; + const log = deps.log ?? console.log; + const runDocker = deps.runDocker ?? defaultRunDocker; + const sleep = deps.sleep ?? sleepSync; + const detected = detectDockerHost(env, deps); + const dockerEnv = detected.dockerHost ? { ...env, DOCKER_HOST: detected.dockerHost } : env; + + const clustersOutput = commandOutput( + runDocker(["ps", "--filter", "name=openshell-cluster", "--format", "{{.Names}}"], { + env: dockerEnv, + }), + ); + const cluster = selectOpenshellClusterContainer(options.gatewayName, clustersOutput); + if (!cluster) { + const message = options.gatewayName + ? `WARNING: Could not find gateway container for '${options.gatewayName}'. DNS proxy not installed.` + : "WARNING: Could not find any openshell cluster container. DNS proxy not installed."; + log(message); + return { exitCode: 1, message }; + } + + let dnsUpstream = commandOutput( + kctl( + runDocker, + cluster, + ["get", "endpoints", "kube-dns", "-n", "kube-system", "-o", "jsonpath={.subsets[0].addresses[0].ip}"], + dockerEnv, + ), + ).trim(); + if (!dnsUpstream) { + log("WARNING: Could not discover CoreDNS pod IP. Falling back to 8.8.8.8."); + log("WARNING: k8s-internal names (inference.local routing) will NOT work."); + dnsUpstream = DEFAULT_DNS_UPSTREAM; + } + if (!isSafeDnsAddress(dnsUpstream)) { + return { cluster, dnsUpstream, exitCode: 1, message: `ERROR: DNS upstream '${dnsUpstream}' contains invalid characters.` }; + } + + const podsOutput = commandOutput(kctl(runDocker, cluster, ["get", "pods", "-n", "openshell", "-o", "name"], dockerEnv)); + const pod = selectSandboxPod(options.sandboxName, podsOutput); + if (!pod) { + const message = `WARNING: Could not find pod for sandbox '${options.sandboxName}'. DNS proxy not installed.`; + log(message); + return { cluster, dnsUpstream, exitCode: 1, message }; + } + + const vethGateway = parseVethGateway( + commandOutput( + kctl( + runDocker, + cluster, + [ + "exec", + "-n", + "openshell", + pod, + "--", + "sh", + "-c", + "ip addr show | grep 'inet 10\\.200\\.0\\.' | awk '{print $2}' | cut -d/ -f1", + ], + dockerEnv, + ), + ), + ); + if (!isSafeDnsAddress(vethGateway)) { + return { cluster, dnsUpstream, exitCode: 1, pod, message: `ERROR: VETH gateway '${vethGateway}' contains invalid characters.` }; + } + + log(`Setting up DNS proxy in pod '${pod}' (${vethGateway}:53 -> ${dnsUpstream})...`); + + const proxyWriter = `cat > /tmp/dns-proxy.py << 'DNSPROXY'\n${buildDnsProxyPython()}DNSPROXY`; + kctl(runDocker, cluster, ["exec", "-n", "openshell", pod, "--", "sh", "-c", proxyWriter], dockerEnv); + + const oldPid = commandOutput(kctl(runDocker, cluster, ["exec", "-n", "openshell", pod, "--", "cat", "/tmp/dns-proxy.pid"], dockerEnv)).trim(); + if (oldPid) { + kctl(runDocker, cluster, ["exec", "-n", "openshell", pod, "--", "kill", oldPid], dockerEnv); + sleep(1000); + } + + kctl( + runDocker, + cluster, + [ + "exec", + "-n", + "openshell", + pod, + "--", + "sh", + "-c", + `nohup python3 -u /tmp/dns-proxy.py ${shellSingleQuote(dnsUpstream)} ${shellSingleQuote(vethGateway)} > /tmp/dns-proxy.log 2>&1 &`, + ], + dockerEnv, + ); + + let dnsReady = false; + for (let attempt = 0; attempt < 10; attempt += 1) { + const probe = kctl( + runDocker, + cluster, + ["exec", "-n", "openshell", pod, "--", "python3", "-c", buildDnsReadyProbePython(vethGateway)], + dockerEnv, + ); + if (probe.stdout.includes("ok")) { + dnsReady = true; + break; + } + sleep(1000); + } + if (!dnsReady) log("WARNING: DNS forwarder not responding after 10s — verification may fail"); + + const sandboxNamespace = selectSandboxNamespace( + commandOutput(kctl(runDocker, cluster, ["exec", "-n", "openshell", pod, "--", "sh", "-c", "ls /run/netns/ 2>/dev/null"], dockerEnv)), + ); + + let iptablesBin = ""; + if (!sandboxNamespace) { + log("WARNING: Could not find sandbox network namespace. DNS may not work."); + } else { + for (const candidate of ["iptables", "/sbin/iptables", "/usr/sbin/iptables"]) { + const test = kctl( + runDocker, + cluster, + ["exec", "-n", "openshell", pod, "--", "sh", "-c", `test -x "$(command -v ${candidate} 2>/dev/null || echo ${candidate})"`], + dockerEnv, + ); + if (test.status === 0) { + iptablesBin = candidate; + break; + } + } + + kctl( + runDocker, + cluster, + [ + "exec", + "-n", + "openshell", + pod, + "--", + "ip", + "netns", + "exec", + sandboxNamespace, + "sh", + "-c", + "[ -f /tmp/resolv.conf.orig ] || cp /etc/resolv.conf /tmp/resolv.conf.orig", + ], + dockerEnv, + ); + + if (iptablesBin) { + const iptablesPrefix = [ + "exec", + "-n", + "openshell", + pod, + "--", + "ip", + "netns", + "exec", + sandboxNamespace, + iptablesBin, + ]; + const iptablesRule = ["-p", "udp", "-d", vethGateway, "--dport", "53", "-j", "ACCEPT"]; + const check = kctl(runDocker, cluster, [...iptablesPrefix, "-C", "OUTPUT", ...iptablesRule], dockerEnv); + if (check.status !== 0) { + kctl(runDocker, cluster, [...iptablesPrefix, "-I", "OUTPUT", "1", ...iptablesRule], dockerEnv); + } + + kctl( + runDocker, + cluster, + [ + "exec", + "-n", + "openshell", + pod, + "--", + "ip", + "netns", + "exec", + sandboxNamespace, + "sh", + "-c", + `printf ${shellSingleQuote(buildResolvConf(vethGateway))} > /etc/resolv.conf`, + ], + dockerEnv, + ); + } else { + log("WARNING: iptables not found in pod (checked PATH, /sbin, /usr/sbin)."); + log("WARNING: Cannot add UDP DNS exception. Sandbox DNS resolution will not work."); + kctl( + runDocker, + cluster, + [ + "exec", + "-n", + "openshell", + pod, + "--", + "ip", + "netns", + "exec", + sandboxNamespace, + "sh", + "-c", + "[ -f /tmp/resolv.conf.orig ] && cp /tmp/resolv.conf.orig /etc/resolv.conf", + ], + dockerEnv, + ); + } + } + + let verificationPass = 0; + let verificationFail = 0; + const pid = commandOutput(kctl(runDocker, cluster, ["exec", "-n", "openshell", pod, "--", "cat", "/tmp/dns-proxy.pid"], dockerEnv)).trim(); + const dnsLog = commandOutput(kctl(runDocker, cluster, ["exec", "-n", "openshell", pod, "--", "cat", "/tmp/dns-proxy.log"], dockerEnv)).trim(); + if (pid && dnsLog.includes("dns-proxy:")) { + log(` [PASS] DNS forwarder running (pid=${pid}): ${dnsLog}`); + verificationPass += 1; + } else { + log(` [FAIL] DNS forwarder not running. PID=${pid || "none"} Log: ${dnsLog || "empty"}`); + verificationFail += 1; + } + + const sbExec = (args: string[]) => + sandboxNamespace + ? kctl(runDocker, cluster, ["exec", "-n", "openshell", pod, "--", "ip", "netns", "exec", sandboxNamespace, ...args], dockerEnv) + : null; + + if (sandboxNamespace) { + const resolv = commandOutput(sbExec(["cat", "/etc/resolv.conf"]) ?? { status: 1, stdout: "", stderr: "" }); + if (resolv.includes(`nameserver ${vethGateway}`)) { + log(` [PASS] resolv.conf -> nameserver ${vethGateway}`); + verificationPass += 1; + } else { + log(` [FAIL] resolv.conf does not point to ${vethGateway}: ${resolv}`); + verificationFail += 1; + } + + const iptablesCheck = sbExec([iptablesBin || "iptables", "-C", "OUTPUT", "-p", "udp", "-d", vethGateway, "--dport", "53", "-j", "ACCEPT"]); + if (iptablesCheck?.status === 0) { + log(` [PASS] iptables: UDP ${vethGateway}:53 ACCEPT rule present`); + verificationPass += 1; + } else { + log(" [FAIL] iptables: UDP DNS ACCEPT rule missing"); + verificationFail += 1; + } + + let dnsResult = ""; + for (let attempt = 1; attempt <= 3; attempt += 1) { + dnsResult = commandOutput(sbExec(["getent", "hosts", "github.com"]) ?? { status: 1, stdout: "", stderr: "" }).trim(); + if (dnsResult) break; + if (attempt < 3) sleep(2000); + } + if (dnsResult) { + log(` [PASS] getent hosts github.com -> ${dnsResult}`); + verificationPass += 1; + } else { + log(" [FAIL] getent hosts github.com returned empty after 3 attempts (DNS not resolving)"); + verificationFail += 1; + } + } else { + log(" [SKIP] Sandbox namespace not found; cannot verify resolv.conf, iptables, or DNS"); + } + + log(` DNS verification: ${verificationPass} passed, ${verificationFail} failed`); + if (verificationFail > 0) log("WARNING: DNS setup incomplete. Sandbox DNS resolution may not work. See issue #626, #557."); + + return { + cluster, + dnsUpstream, + exitCode: 0, + pod, + sandboxNamespace: sandboxNamespace ?? undefined, + verificationFail, + verificationPass, + vethGateway, + }; +} diff --git a/src/lib/domain/dns/setup-proxy.test.ts b/src/lib/domain/dns/setup-proxy.test.ts new file mode 100644 index 00000000000..fbe55b9d363 --- /dev/null +++ b/src/lib/domain/dns/setup-proxy.test.ts @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildDnsProxyPython, + buildDnsReadyProbePython, + buildResolvConf, + isSafeDnsAddress, + parseVethGateway, + selectSandboxNamespace, + selectSandboxPod, +} from "../../../../dist/lib/domain/dns/setup-proxy.js"; + +describe("DNS setup proxy domain helpers", () => { + it("selects a sandbox pod using fixed-string style matching", () => { + expect(selectSandboxPod("box[1]", "pod/alpha\npod/box[1]-abc\n")).toBe("box[1]-abc"); + expect(selectSandboxPod("missing", "pod/alpha\n")).toBeNull(); + }); + + it("falls back to the default veth gateway when discovery is empty", () => { + expect(parseVethGateway("10.200.0.1\n")).toBe("10.200.0.1"); + expect(parseVethGateway("\n")).toBe("10.200.0.1"); + }); + + it("selects the first sandbox namespace", () => { + expect(selectSandboxNamespace("other\nsandbox-ns\n")).toBe("sandbox-ns"); + expect(selectSandboxNamespace("other\n")).toBeNull(); + }); + + it("builds checked-in DNS proxy payloads and resolv.conf content", () => { + expect(buildDnsProxyPython()).toContain("sock.bind((BIND_IP, 53))"); + expect(buildDnsReadyProbePython("10.200.0.1")).toContain("10.200.0.1"); + expect(buildResolvConf("10.200.0.1")).toBe("nameserver 10.200.0.1\noptions ndots:5\n"); + }); + + it("rejects unsafe DNS address strings", () => { + expect(isSafeDnsAddress("10.43.0.10")).toBe(true); + expect(isSafeDnsAddress("bad;rm")).toBe(false); + }); +}); diff --git a/src/lib/domain/dns/setup-proxy.ts b/src/lib/domain/dns/setup-proxy.ts new file mode 100644 index 00000000000..930c71f7ba5 --- /dev/null +++ b/src/lib/domain/dns/setup-proxy.ts @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const DEFAULT_DNS_UPSTREAM = "8.8.8.8"; +export const DEFAULT_VETH_GATEWAY = "10.200.0.1"; + +export function buildDnsProxyPython(): string { + return String.raw`import socket, threading, os, sys + +UPSTREAM = (sys.argv[1] if len(sys.argv) > 1 else '8.8.8.8', 53) +BIND_IP = sys.argv[2] if len(sys.argv) > 2 else '0.0.0.0' + +sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +sock.bind((BIND_IP, 53)) + +with open('/tmp/dns-proxy.pid', 'w') as pf: + pf.write(str(os.getpid())) + +msg = 'dns-proxy: {}:53 -> {}:{} pid={}'.format(BIND_IP, UPSTREAM[0], UPSTREAM[1], os.getpid()) +print(msg, flush=True) +with open('/tmp/dns-proxy.log', 'w') as log: + log.write(msg + '\n') + +def forward(data, addr): + try: + f = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + f.settimeout(5) + f.sendto(data, UPSTREAM) + r, _ = f.recvfrom(4096) + sock.sendto(r, addr) + f.close() + except Exception: + pass + +while True: + d, a = sock.recvfrom(4096) + threading.Thread(target=forward, args=(d, a), daemon=True).start() +`; +} + +export function buildDnsReadyProbePython(vethGateway: string): string { + return String.raw` +import socket, sys +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +s.settimeout(1) +try: + s.sendto(b'\x00\x1e\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x06google\x03com\x00\x00\x01\x00\x01', + (${JSON.stringify(vethGateway)}, 53)) + data, _ = s.recvfrom(4096) + sys.stdout.write('ok' if data else '') +except Exception: + pass +`; +} + +// Kubernetes pod names append generated suffixes to the requested sandbox name. +// Keep substring matching so names such as "box[1]" match "pod/box[1]-abc", +// then strip the leading "pod/" prefix before returning the pod name. +export function selectSandboxPod(sandboxName: string, podsOutput: string): string | null { + for (const line of podsOutput.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed.includes(sandboxName)) continue; + return trimmed.replace(/^pod\//, ""); + } + return null; +} + +export function parseVethGateway(output: string): string { + const trimmed = output.trim(); + return trimmed || DEFAULT_VETH_GATEWAY; +} + +export function selectSandboxNamespace(output: string): string | null { + return ( + output + .split(/\r?\n/) + .map((line) => line.trim()) + .find((line) => line.includes("sandbox")) ?? null + ); +} + +export function buildResolvConf(vethGateway: string): string { + return `nameserver ${vethGateway}\noptions ndots:5\n`; +} + +export function isSafeDnsAddress(value: string): boolean { + return /^[a-zA-Z0-9.:_-]+$/.test(value); +} diff --git a/test/internal-cli.test.ts b/test/internal-cli.test.ts index 7d9e065a63e..b66620774ec 100644 --- a/test/internal-cli.test.ts +++ b/test/internal-cli.test.ts @@ -17,4 +17,14 @@ describe("internal oclif namespace", () => { expect(result.stdout).toContain("Internal: patch CoreDNS"); expect(result.stdout).toContain("nemoclaw internal dns fix-coredns [gateway-name]"); }); + + it("exposes setup-proxy as an oclif-routed internal subcommand", () => { + const result = spawnSync(process.execPath, [CLI, "internal", "dns", "setup-proxy", "--help"], { + encoding: "utf-8", + }); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("Internal: configure sandbox DNS proxy"); + expect(result.stdout).toContain("nemoclaw internal dns setup-proxy "); + }); });