fix: sandbox DNS resolution via CoreDNS proxy in sandbox pod

.pre-commit-config.yaml

@@ -200,7 +200,7 @@ repos:
 
       - id: pyright-check
         name: Pyright (nemoclaw-blueprint)
-        entry: bash -c 'cd nemoclaw-blueprint && uv run --with pyright pyright'
+        entry: bash -c 'cd nemoclaw-blueprint && uv run --with pyright --with pytest pyright'
         language: system
         pass_filenames: false
         always_run: true

bin/lib/onboard.js

@@ -459,14 +459,15 @@ async function startGateway(gpu) {
     sleep(2);
   }
 
-  // CoreDNS fix — always run. k3s-inside-Docker has broken DNS on all platforms.
+  // CoreDNS fix — k3s-inside-Docker has broken DNS forwarding on all platforms.
   const runtime = getContainerRuntime();
   if (shouldPatchCoredns(runtime)) {
-    console.log("  Patching CoreDNS for Colima...");
+    console.log("  Patching CoreDNS DNS forwarding...");
     run(`bash "${path.join(SCRIPTS, "fix-coredns.sh")}" nemoclaw 2>&1 || true`, { ignoreError: true });
   }
   // Give DNS a moment to propagate
   sleep(5);
+
 }
 
 // ── Step 3: Sandbox ──────────────────────────────────────────────
@@ -613,6 +614,11 @@ async function createSandbox(gpu) {
     gpuEnabled: !!gpu,
   });
 
+  // DNS proxy — run a forwarder in the sandbox pod so the isolated
+  // sandbox namespace can resolve DNS. Must run after sandbox is Ready.
+  console.log("  Setting up sandbox DNS proxy...");
+  run(`bash "${path.join(SCRIPTS, "setup-dns-proxy.sh")}" nemoclaw "${sandboxName}" 2>&1 || true`, { ignoreError: true });
+
   console.log(`  ✓ Sandbox '${sandboxName}' created`);
   return sandboxName;
 }

bin/lib/platform.js

@@ -36,7 +36,10 @@ function isUnsupportedMacosRuntime(runtime, opts = {}) {
 }
 
 function shouldPatchCoredns(runtime) {
-  return runtime === "colima";
+  // k3s-inside-Docker has broken DNS forwarding on all platforms
+  // (systemd-resolved, Docker Desktop DNS, Colima DNS).
+  // Always patch CoreDNS to use a non-loopback upstream.
+  return runtime !== "unknown";
 }
 
 function getColimaDockerSocketCandidates(opts = {}) {

scripts/fix-coredns.sh

@@ -2,17 +2,18 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
-# Fix CoreDNS on local OpenShell gateways running under Colima.
+# Fix CoreDNS on local OpenShell gateways.
 #
 # Problem: k3s CoreDNS forwards to /etc/resolv.conf which inside the
-# CoreDNS pod resolves to 127.0.0.11 (Docker's embedded DNS). That
-# address is NOT reachable from k3s pods, causing DNS to fail and
-# CoreDNS to CrashLoop.
+# CoreDNS pod resolves to a loopback address (127.0.0.11 on Docker,
+# 127.0.0.53 on systemd-resolved hosts). That address is NOT reachable
+# from k3s pods, causing DNS to fail and CoreDNS to CrashLoop.
 #
-# Fix: forward CoreDNS to the container's default gateway IP, which
-# is reachable from pods and routes DNS through Docker to the host.
+# Fix: forward CoreDNS to a non-loopback upstream — either the
+# container's default gateway IP (routes through Docker to the host)
+# or a public DNS server (8.8.8.8) as a last resort.
 #
-# Run this after `openshell gateway start` on Colima setups.
+# Run this after `openshell gateway start`.
 #
 # Usage: ./scripts/fix-coredns.sh [gateway-name]
 
@@ -23,15 +24,11 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 # shellcheck source=./lib/runtime.sh
 . "$SCRIPT_DIR/lib/runtime.sh"
 
-COLIMA_SOCKET="$(find_colima_docker_socket || true)"
-
 if [ -z "${DOCKER_HOST:-}" ]; then
-  if [ -n "$COLIMA_SOCKET" ]; then
-    export DOCKER_HOST="unix://$COLIMA_SOCKET"
-  else
-    echo "Skipping CoreDNS patch: Colima socket not found."
-    exit 0
+  if docker_host="$(detect_docker_host)"; then
+    export DOCKER_HOST="$docker_host"
   fi
+  # If still unset, Docker CLI will use the default socket
 fi
 
 # Find the cluster container
@@ -48,10 +45,17 @@ fi
 
 CONTAINER_RESOLV_CONF="$(docker exec "$CLUSTER" cat /etc/resolv.conf 2>/dev/null || true)"
 HOST_RESOLV_CONF="$(cat /etc/resolv.conf 2>/dev/null || true)"
-UPSTREAM_DNS="$(resolve_coredns_upstream "$CONTAINER_RESOLV_CONF" "$HOST_RESOLV_CONF" "colima" || true)"
+
+# Detect runtime for Colima-specific DNS discovery paths
+RUNTIME="unknown"
+if [ -n "${DOCKER_HOST:-}" ]; then
+  RUNTIME="$(docker_host_runtime "$DOCKER_HOST" || echo "unknown")"
+fi
+
+UPSTREAM_DNS="$(resolve_coredns_upstream "$CONTAINER_RESOLV_CONF" "$HOST_RESOLV_CONF" "$RUNTIME" || true)"
 
 if [ -z "$UPSTREAM_DNS" ]; then
-  echo "ERROR: Could not determine a non-loopback DNS upstream for Colima."
+  echo "ERROR: Could not determine a non-loopback DNS upstream."
   exit 1
 fi
 

scripts/lib/runtime.sh

@@ -163,7 +163,9 @@ resolve_coredns_upstream() {
     return 0
   fi
 
-  return 1
+  # Last resort: public DNS. Needed on hosts where all nameservers are
+  # loopback (e.g. systemd-resolved uses 127.0.0.53).
+  printf '8.8.8.8\n'
 }
 
 select_openshell_cluster_container() {

scripts/setup-dns-proxy.sh

@@ -0,0 +1,193 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Fix sandbox DNS by running a lightweight DNS forwarder in the sandbox pod.
+#
+# Problem: The sandbox runs in an isolated network namespace (10.200.0.0/24).
+# Its /etc/resolv.conf points to the k3s CoreDNS service IP (10.43.0.10), but
+# DNS packets from the sandbox route through the pod namespace — where the
+# CoreDNS service IP is not locally handled. The result: dns.lookup() fails
+# with EAI_AGAIN for every outbound request.
+#
+# Fix: Run a Python DNS forwarder in the sandbox pod's namespace that:
+#   1. Adds 10.43.0.10 as a local address on lo (so packets from the sandbox
+#      are delivered locally instead of forwarded)
+#   2. Listens on 0.0.0.0:53 (UDP) and forwards to public DNS (8.8.8.8)
+#
+# The sandbox's existing resolv.conf (nameserver 10.43.0.10) works without
+# modification — the forwarder intercepts the traffic transparently.
+#
+# The DNS proxy is launched via `docker exec -d` + `nsenter` from the gateway
+# container, which keeps it alive as a persistent background process.
+#
+# Requires: sandbox must be in Ready state. Run after sandbox creation.
+#
+# Usage: ./scripts/setup-dns-proxy.sh [gateway-name] <sandbox-name>
+
+set -euo pipefail
+
+GATEWAY_NAME="${1:-}"
+SANDBOX_NAME="${2:-}"
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=./lib/runtime.sh
+. "$SCRIPT_DIR/lib/runtime.sh"
+
+if [ -z "$SANDBOX_NAME" ]; then
+  echo "Usage: $0 [gateway-name] <sandbox-name>"
+  exit 1
+fi
+
+# CoreDNS service IP that the sandbox's /etc/resolv.conf points to
+COREDNS_SERVICE_IP="10.43.0.10"
+# DNS_UPSTREAM is set below after we discover the CoreDNS pod IP
+
+# ── Find the gateway container ──────────────────────────────────────
+
+if [ -z "${DOCKER_HOST:-}" ]; then
+  if docker_host="$(detect_docker_host)"; then
+    export DOCKER_HOST="$docker_host"
+  fi
+fi
+
+CLUSTERS="$(docker ps --filter "name=openshell-cluster" --format '{{.Names}}' 2>/dev/null || true)"
+CLUSTER="$(select_openshell_cluster_container "$GATEWAY_NAME" "$CLUSTERS" || true)"
+
+if [ -z "$CLUSTER" ]; then
+  if [ -n "$GATEWAY_NAME" ]; then
+    echo "ERROR: Could not find gateway container for '$GATEWAY_NAME'."
+  else
+    echo "ERROR: Could not find any openshell cluster container."
+  fi
+  exit 1
+fi
+
+# ── Helper: kubectl via gateway ─────────────────────────────────────
+
+kctl() {
+  docker exec "$CLUSTER" kubectl "$@"
+}
+
+# ── Discover CoreDNS pod IP ─────────────────────────────────────────
+#
+# Forward to CoreDNS (not 8.8.8.8) so k8s-internal names like
+# openshell-0.openshell.svc.cluster.local still resolve. CoreDNS
+# handles both k8s names (kubernetes plugin) and external names
+# (forward plugin, patched by fix-coredns.sh).
+
+DNS_UPSTREAM="$(kctl get endpoints kube-dns \
+  -n kube-system -o jsonpath='{.subsets[0].addresses[0].ip}' 2>/dev/null || true)"
+
+if [ -z "$DNS_UPSTREAM" ]; then
+  echo "WARNING: Could not discover CoreDNS pod IP. Falling back to 8.8.8.8."
+  echo "WARNING: k8s-internal names (inference.local routing) will NOT work."
+  DNS_UPSTREAM="8.8.8.8"
+fi
+
+# ── Find the sandbox pod and its PID ────────────────────────────────
+
+POD="$(kctl get pods -n openshell -o name 2>/dev/null \
+  | grep -- "$SANDBOX_NAME" | head -1 | sed 's|pod/||' || true)"
+
+if [ -z "$POD" ]; then
+  echo "ERROR: Could not find pod for sandbox '$SANDBOX_NAME'."
+  exit 1
+fi
+
+# Get the pod's init PID as seen from the gateway container (for nsenter)
+POD_PID="$(docker exec "$CLUSTER" sh -c "
+  # Find PID that has the pod's hostname in its UTS namespace
+  for pid in /proc/[0-9]*/ns; do
+    p=\${pid%/ns}; p=\${p##*/}
+    if [ -f /proc/\$p/root/etc/hostname ] 2>/dev/null; then
+      hn=\$(cat /proc/\$p/root/etc/hostname 2>/dev/null)
+      if [ \"\$hn\" = \"$POD\" ]; then
+        echo \$p
+        break
+      fi
+    fi
+  done
+" 2>/dev/null || true)"
+
+if [ -z "$POD_PID" ]; then
+  echo "WARNING: Could not find pod PID via hostname. Trying kubectl..."
+  # Fallback: use kubectl exec to find a PID we can nsenter into
+  POD_PID="$(kctl exec -n openshell "$POD" -- sh -c 'echo $$' 2>/dev/null || true)"
+fi
+
+if [ -z "$POD_PID" ]; then
+  echo "ERROR: Could not determine pod PID for nsenter."
+  exit 1
+fi
+
+echo "Setting up DNS proxy in pod '$POD' (pid=$POD_PID, ${COREDNS_SERVICE_IP} → ${DNS_UPSTREAM})..."
+
+# ── Step 1: Add CoreDNS service IP as local address ─────────────────
+
+kctl exec -n openshell "$POD" -- \
+  ip addr add "${COREDNS_SERVICE_IP}/32" dev lo 2>/dev/null || true
+
+# ── Step 2: Write DNS proxy script to the pod ───────────────────────
+
+kctl exec -n openshell "$POD" -- sh -c "cat > /tmp/dns-proxy.py << DNSPROXY
+import socket, threading, os
+
+UPSTREAM = ('${DNS_UPSTREAM}', 53)
+
+sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+sock.bind(('10.43.0.10', 53))
+
+with open('/tmp/dns-proxy.pid', 'w') as pf:
+    pf.write(str(os.getpid()))
+
+with open('/tmp/dns-proxy.log', 'w') as log:
+    log.write('dns-proxy: 10.43.0.10:53 -> {}:{} pid={}\n'.format(
+        UPSTREAM[0], UPSTREAM[1], os.getpid()))
+
+def forward(data, addr):
+    try:
+        f = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        f.settimeout(5)
+        f.sendto(data, UPSTREAM)
+        r, _ = f.recvfrom(4096)
+        sock.sendto(r, addr)
+        f.close()
+    except Exception:
+        pass
+
+while True:
+    d, a = sock.recvfrom(4096)
+    threading.Thread(target=forward, args=(d, a), daemon=True).start()
+DNSPROXY"
+
+# ── Step 3: Kill any existing DNS proxy ─────────────────────────────
+
+OLD_PID="$(kctl exec -n openshell "$POD" -- cat /tmp/dns-proxy.pid 2>/dev/null || true)"
+if [ -n "$OLD_PID" ]; then
+  kctl exec -n openshell "$POD" -- kill "$OLD_PID" 2>/dev/null || true
+  sleep 1
+fi
+
+# ── Step 4: Launch DNS proxy via docker exec -d (persistent) ────────
+#
+# Using `docker exec -d` (detached) + `nsenter` to enter the pod's
+# network and mount namespaces. This creates a persistent process that
+# survives after the script exits — unlike kubectl exec which kills
+# child processes on session end.
+
+docker exec -d "$CLUSTER" \
+  nsenter -t "$POD_PID" -n -m -- \
+  python3 -u /tmp/dns-proxy.py
+
+sleep 2
+
+# ── Step 5: Verify ──────────────────────────────────────────────────
+
+LOG="$(kctl exec -n openshell "$POD" -- cat /tmp/dns-proxy.log 2>/dev/null || true)"
+if echo "$LOG" | grep -q "dns-proxy:"; then
+  echo "DNS proxy started: $LOG"
+else
+  echo "WARNING: DNS proxy may not have started. Log: $LOG"
+fi

scripts/setup.sh

@@ -123,9 +123,9 @@ for i in 1 2 3 4 5; do
 done
 info "Gateway is healthy"
 
-# 2. CoreDNS fix (Colima only)
-if [ "$CONTAINER_RUNTIME" = "colima" ]; then
-  info "Patching CoreDNS for Colima..."
+# 2. CoreDNS fix — k3s-inside-Docker has broken DNS forwarding on all platforms.
+if [ "$CONTAINER_RUNTIME" != "unknown" ]; then
+  info "Patching CoreDNS DNS forwarding..."
   bash "$SCRIPT_DIR/fix-coredns.sh" nemoclaw 2>&1 || warn "CoreDNS patch failed (may not be needed)"
 fi
 
@@ -230,6 +230,10 @@ if ! echo "$SANDBOX_LINE" | grep -q "Ready"; then
   fail "Sandbox created but not Ready (phase: ${SANDBOX_PHASE:-unknown}). Check 'openshell sandbox get ${SANDBOX_NAME}'."
 fi
 
+# 5b. DNS proxy for sandbox — run after sandbox is Ready.
+info "Setting up sandbox DNS proxy..."
+bash "$SCRIPT_DIR/setup-dns-proxy.sh" nemoclaw "$SANDBOX_NAME" 2>&1 || warn "DNS proxy setup failed (may not be needed)"
+
 # 6. Done
 echo ""
 info "Setup complete!"