fix(runtime): repair stale inference DNS routes

src/commands/uninstall.ts

@@ -2,12 +2,11 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import Command from "../lib/commands/uninstall";
-import { CLI_NAME } from "../lib/cli/branding";
 import { withCommandDisplay } from "../lib/cli/command-display";
 
 export default withCommandDisplay(Command, [
   {
-    usage: `${CLI_NAME} uninstall`,
+    usage: "nemoclaw uninstall",
     description: "Run uninstall.sh (local only; no remote fallback)",
     group: "Cleanup",
     scope: "global",

src/lib/actions/dns/index.test.ts

@@ -158,7 +158,8 @@ describe("runSetupDnsProxy", () => {
       calls.push(args);
       const cmd = args.join(" ");
       if (args[0] === "ps") return ok("openshell-cluster-nemoclaw\n");
-      if (cmd.includes("get endpoints kube-dns")) return ok("10.43.0.10");
+      if (cmd.includes("get service kube-dns")) return ok("10.43.0.10");
+      if (cmd.includes("get endpoints kube-dns")) return ok("10.42.0.15");
       if (cmd.includes("get pods -n openshell -o name")) return ok("pod/box[1]-abc\n");
       if (cmd.includes("ip addr show")) return ok("10.200.0.1\n");
       if (cmd.includes("cat /tmp/dns-proxy.pid")) return ok("12345\n");
@@ -182,13 +183,52 @@ describe("runSetupDnsProxy", () => {
     );
 
     expect(result.exitCode).toBe(0);
+    expect(result.dnsUpstream).toBe("10.43.0.10");
     expect(result.pod).toBe("box[1]-abc");
     expect(result.verificationPass).toBe(4);
+    expect(calls.some((args) => args.join(" ").includes("get service kube-dns"))).toBe(true);
+    expect(calls.some((args) => args.join(" ").includes("get endpoints kube-dns"))).toBe(false);
     expect(calls.some((args) => args.includes("box[1]-abc"))).toBe(true);
     expect(calls.some((args) => args.join(" ").includes("nohup python3 -u /tmp/dns-proxy.py '10.43.0.10' '10.200.0.1'"))).toBe(true);
     expect(log).toHaveBeenCalledWith("  DNS verification: 4 passed, 0 failed");
   });
 
+  it("falls back to the CoreDNS pod endpoint when the kube-dns service IP is unavailable", () => {
+    const calls: string[][] = [];
+    const runDocker = vi.fn((args: string[]) => {
+      calls.push(args);
+      const cmd = args.join(" ");
+      if (args[0] === "ps") return ok("openshell-cluster-nemoclaw\n");
+      if (cmd.includes("get service kube-dns")) return ok("");
+      if (cmd.includes("get endpoints kube-dns")) return ok("10.42.0.15");
+      if (cmd.includes("get pods -n openshell -o name")) return ok("pod/box-abc\n");
+      if (cmd.includes("ip addr show")) return ok("10.200.0.1\n");
+      if (cmd.includes("cat /tmp/dns-proxy.pid")) return ok("12345\n");
+      if (cmd.includes("cat /tmp/dns-proxy.log")) return ok("dns-proxy: 10.200.0.1:53 -> 10.42.0.15:53 pid=12345\n");
+      if (cmd.includes("python3 -c")) return ok("ok");
+      if (cmd.includes("ls /run/netns/")) return ok("sandbox-ns\n");
+      if (cmd.includes("test -x")) return ok();
+      if (cmd.includes("cat /etc/resolv.conf")) return ok("nameserver 10.200.0.1\n");
+      if (cmd.includes("getent hosts github.com")) return ok("140.82.112.4 github.com\n");
+      return ok();
+    });
+
+    const result = runSetupDnsProxy(
+      { gatewayName: "nemoclaw", sandboxName: "box" },
+      {
+        env: { DOCKER_HOST: "unix:///tmp/fake-docker.sock" },
+        log: vi.fn(),
+        runDocker,
+        sleep: vi.fn(),
+      },
+    );
+
+    expect(result.exitCode).toBe(0);
+    expect(result.dnsUpstream).toBe("10.42.0.15");
+    expect(calls.some((args) => args.join(" ").includes("get service kube-dns"))).toBe(true);
+    expect(calls.some((args) => args.join(" ").includes("get endpoints kube-dns"))).toBe(true);
+  });
+
   it("rejects unsafe DNS upstreams before launching the forwarder", () => {
     const calls: string[][] = [];
     const result = runSetupDnsProxy(
@@ -199,7 +239,7 @@ describe("runSetupDnsProxy", () => {
           calls.push(args);
           const cmd = args.join(" ");
           if (args[0] === "ps") return ok("openshell-cluster-nemoclaw\n");
-          if (cmd.includes("get endpoints kube-dns")) return ok("bad;rm\n");
+          if (cmd.includes("get service kube-dns")) return ok("bad;rm\n");
           return ok();
         },
         sleep: vi.fn(),

src/lib/actions/dns/index.ts

@@ -289,12 +289,31 @@ export function runSetupDnsProxy(
     kctl(
       runDocker,
       cluster,
-      ["get", "endpoints", "kube-dns", "-n", "kube-system", "-o", "jsonpath={.subsets[0].addresses[0].ip}"],
+      ["get", "service", "kube-dns", "-n", "kube-system", "-o", "jsonpath={.spec.clusterIP}"],
       dockerEnv,
     ),
   ).trim();
+  if (!dnsUpstream || dnsUpstream.toLowerCase() === "none") {
+    log("WARNING: Could not discover kube-dns service IP. Falling back to CoreDNS pod IP.");
+    dnsUpstream = commandOutput(
+      kctl(
+        runDocker,
+        cluster,
+        [
+          "get",
+          "endpoints",
+          "kube-dns",
+          "-n",
+          "kube-system",
+          "-o",
+          "jsonpath={.subsets[0].addresses[0].ip}",
+        ],
+        dockerEnv,
+      ),
+    ).trim();
+  }
   if (!dnsUpstream) {
-    log("WARNING: Could not discover CoreDNS pod IP. Falling back to 8.8.8.8.");
+    log("WARNING: Could not discover CoreDNS service or pod IP. Falling back to 8.8.8.8.");
     log("WARNING: k8s-internal names (inference.local routing) will NOT work.");
     dnsUpstream = DEFAULT_DNS_UPSTREAM;
   }

src/lib/actions/sandbox/connect.ts

@@ -16,7 +16,9 @@ import {
 } from "../../adapters/openshell/runtime";
 import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts";
 import * as registry from "../../state/registry";
+import type { SandboxEntry } from "../../state/registry";
 import { ROOT } from "../../runner";
+import { runSetupDnsProxy } from "../dns";
 import { ensureLiveSandboxOrExit } from "./gateway-state";
 import {
   createSystemDeps as createSessionDeps,
@@ -29,6 +31,9 @@ import { resolveOpenshell } from "../../adapters/openshell/resolve";
 
 const agentRuntime = require("../../../../bin/lib/agent-runtime");
 
+const NEMOCLAW_GATEWAY_NAME = "nemoclaw";
+const INFERENCE_ROUTE_PROBE_TIMEOUT_MS = 10_000;
+
 export type SandboxConnectOptions = {
   probeOnly?: boolean;
 };
@@ -102,6 +107,7 @@ function runSandboxConnectProbe(sandboxName: string): void {
     process.exit(1);
   }
   if (processCheck.wasRunning) {
+    ensureSandboxInferenceRoute(sandboxName, { quiet: false });
     if (processCheck.forwardRecovered) {
       console.log(
         `  Probe complete: ${agentName} gateway is running in '${sandboxName}'; restored dashboard port forward.`,
@@ -112,16 +118,111 @@ function runSandboxConnectProbe(sandboxName: string): void {
     return;
   }
   if (processCheck.recovered) {
+    ensureSandboxInferenceRoute(sandboxName, { quiet: false });
     console.log(`  Probe complete: recovered ${agentName} gateway in '${sandboxName}'.`);
     return;
   }
+  ensureSandboxInferenceRoute(sandboxName, { quiet: false });
   console.error(
     `  Probe failed: ${agentName} gateway is not running in '${sandboxName}' and automatic recovery failed.`,
   );
   console.error("  Check /tmp/gateway.log inside the sandbox for details.");
   process.exit(1);
 }
 
+function isSandboxInferenceRouteHealthy(sandboxName: string): boolean {
+  const probe = captureOpenshell(
+    [
+      "sandbox",
+      "exec",
+      "--name",
+      sandboxName,
+      "--",
+      "sh",
+      "-c",
+      [
+        "OUT=/tmp/nemoclaw-inference-route-probe.out",
+        "HTTP_CODE=$(curl -sk -o \"$OUT\" -w '%{http_code}' --connect-timeout 3 --max-time 8 https://inference.local/v1/models 2>/dev/null || printf '000')",
+        "case \"$HTTP_CODE\" in 000|5*) printf 'BROKEN %s ' \"$HTTP_CODE\"; head -c 160 \"$OUT\" 2>/dev/null ;; *) printf 'OK %s' \"$HTTP_CODE\" ;; esac",
+      ].join("; "),
+    ],
+    { ignoreError: true, timeout: INFERENCE_ROUTE_PROBE_TIMEOUT_MS },
+  );
+  return probe.status === 0 && /^OK\s+[0-9]{3}\b/.test(probe.output.trim());
+}
+
+function repairSandboxInferenceRouteIfNeeded(
+  sandboxName: string,
+  { quiet = false }: { quiet?: boolean } = {},
+): boolean {
+  if (process.env.NEMOCLAW_DISABLE_INFERENCE_ROUTE_REPAIR === "1") return false;
+  if (isSandboxInferenceRouteHealthy(sandboxName)) return false;
+
+  if (!quiet) {
+    console.log("");
+    console.log(`  inference.local is unavailable inside '${sandboxName}'. Repairing sandbox DNS proxy...`);
+  }
+  const repair = runSetupDnsProxy(
+    { gatewayName: NEMOCLAW_GATEWAY_NAME, sandboxName },
+    { log: quiet ? () => undefined : console.log },
+  );
+  if (repair.exitCode !== 0) {
+    if (!quiet) {
+      console.error("  Warning: failed to repair sandbox DNS proxy.");
+      if (repair.message) console.error(`  ${repair.message}`);
+    }
+    return false;
+  }
+
+  const healthy = isSandboxInferenceRouteHealthy(sandboxName);
+  if (!quiet) {
+    if (healthy) {
+      console.log("  inference.local route repaired.");
+    } else {
+      console.error("  Warning: inference.local is still unavailable after DNS proxy repair.");
+    }
+  }
+  return healthy;
+}
+
+function ensureSandboxInferenceRoute(
+  sandboxName: string,
+  { quiet = false }: { quiet?: boolean } = {},
+): SandboxEntry | null {
+  let sb: SandboxEntry | null = null;
+  try {
+    sb = registry.getSandbox(sandboxName);
+    if (sb && sb.provider && sb.model) {
+      const live = parseGatewayInference(
+        captureOpenshell(["inference", "get"], {
+          ignoreError: true,
+          timeout: OPENSHELL_PROBE_TIMEOUT_MS,
+        }).output,
+      );
+      if (!live || live.provider !== sb.provider || live.model !== sb.model) {
+        if (!quiet) {
+          console.log(
+            `  Switching inference route to ${sb.provider}/${sb.model} for sandbox '${sandboxName}'`,
+          );
+        }
+        const swapResult = runOpenshell(
+          ["inference", "set", "--provider", sb.provider, "--model", sb.model, "--no-verify"],
+          { ignoreError: true },
+        );
+        if (swapResult.status !== 0 && !quiet) {
+          console.error(
+            `  ${YW}Warning: failed to switch inference route — connect will proceed anyway.${R}`,
+          );
+        }
+      }
+      repairSandboxInferenceRouteIfNeeded(sandboxName, { quiet });
+    }
+  } catch {
+    /* non-fatal — don't block connect on inference route repair */
+  }
+  return sb;
+}
+
 function exitWithSpawnResult(result: SpawnLikeResult): void {
   if (result.status !== null) {
     process.exit(result.status);
@@ -182,34 +283,7 @@ export async function connectSandbox(
   // When the user has multiple sandboxes with different providers, the
   // cluster-wide inference.local route may still point at the *other*
   // provider. Re-set it to match this sandbox's persisted config.
-  let sb;
-  try {
-    sb = registry.getSandbox(sandboxName);
-    if (sb && sb.provider && sb.model) {
-      const live = parseGatewayInference(
-        captureOpenshell(["inference", "get"], {
-          ignoreError: true,
-          timeout: OPENSHELL_PROBE_TIMEOUT_MS,
-        }).output,
-      );
-      if (!live || live.provider !== sb.provider || live.model !== sb.model) {
-        console.log(
-          `  Switching inference route to ${sb.provider}/${sb.model} for sandbox '${sandboxName}'`,
-        );
-        const swapResult = runOpenshell(
-          ["inference", "set", "--provider", sb.provider, "--model", sb.model, "--no-verify"],
-          { ignoreError: true },
-        );
-        if (swapResult.status !== 0) {
-          console.error(
-            `  ${YW}Warning: failed to switch inference route — connect will proceed anyway.${R}`,
-          );
-        }
-      }
-    }
-  } catch {
-    /* non-fatal — don't block connect on inference route swap failure */
-  }
+  const sb = ensureSandboxInferenceRoute(sandboxName);
 
   const rawTimeout = process.env.NEMOCLAW_CONNECT_TIMEOUT;
   let timeout = 120;