refactor(core): centralize result retries

ci/source-architecture-budget.json

@@ -18,7 +18,7 @@
       "src/lib/core/ports.ts": 89,
       "src/lib/core/shell-quote.ts": 28,
       "src/lib/core/url-utils.ts": 29,
-      "src/lib/core/wait.ts": 35,
+      "src/lib/core/wait.ts": 36,
       "src/lib/credentials/store.ts": 46,
       "src/lib/inference/config.ts": 30,
       "src/lib/inference/web-search.ts": 21,
@@ -37,15 +37,17 @@
     "defaultMax": 20,
     "maxByFile": {
       "src/lib/actions/inference-set.ts": 32,
-      "src/lib/actions/sandbox/connect.ts": 40,
+      "src/lib/actions/sandbox/connect.ts": 41,
       "src/lib/actions/sandbox/destroy.ts": 29,
       "src/lib/actions/sandbox/doctor.ts": 30,
-      "src/lib/actions/sandbox/status-snapshot.ts": 20,
+      "src/lib/actions/sandbox/status-snapshot.ts": 21,
       "src/lib/actions/sandbox/policy-channel.ts": 30,
       "src/lib/actions/sandbox/process-recovery.ts": 21,
       "src/lib/actions/sandbox/rebuild-pipeline.ts": 28,
       "src/lib/actions/sandbox/snapshot.ts": 40,
       "src/lib/actions/uninstall/run-plan.ts": 26,
+
+      "src/lib/inference/local.ts": 21,
       "src/lib/inference/onboard-probes.ts": 21,
       "src/lib/inference/vllm.ts": 21,
       "src/lib/onboard.ts": 202,

src/lib/actions/dns/index.test.ts

@@ -162,6 +162,10 @@ describe("runSetupDnsProxy", () => {
   it("configures the DNS proxy through kubectl-in-docker argv calls", () => {
     const calls: string[][] = [];
     const log = vi.fn();
+    let dnsReadyCalls = 0;
+    let pidReads = 0;
+    let getentCalls = 0;
+    const sleep = vi.fn();
     const runDocker = vi.fn((args: string[]) => {
       calls.push(args);
       const cmd = args.join(" ");
@@ -170,14 +174,23 @@ describe("runSetupDnsProxy", () => {
       if (cmd.includes("get endpoints kube-dns")) return ok("10.42.0.15");
       if (cmd.includes("get pods -n openshell -o name")) return ok("pod/box[1]-abc\n");
       if (cmd.includes("ip addr show")) return ok("10.200.0.1\n");
-      if (cmd.includes("cat /tmp/dns-proxy.pid")) return ok("12345\n");
+      if (cmd.includes("cat /tmp/dns-proxy.pid")) {
+        pidReads += 1;
+        return pidReads === 1 ? ok("") : ok("12345\n");
+      }
       if (cmd.includes("cat /tmp/dns-proxy.log"))
         return ok("dns-proxy: 10.200.0.1:53 -> 10.43.0.10:53 pid=12345\n");
-      if (cmd.includes("python3 -c")) return ok("ok");
+      if (cmd.includes("python3 -c")) {
+        dnsReadyCalls += 1;
+        return dnsReadyCalls === 3 ? ok("ok") : ok("");
+      }
       if (cmd.includes("ls /run/netns/")) return ok("sandbox-ns\n");
       if (cmd.includes("test -x")) return ok();
       if (cmd.includes("cat /etc/resolv.conf")) return ok("nameserver 10.200.0.1\n");
-      if (cmd.includes("getent hosts github.com")) return ok("140.82.112.4 github.com\n");
+      if (cmd.includes("getent hosts github.com")) {
+        getentCalls += 1;
+        return getentCalls === 3 ? ok("140.82.112.4 github.com\n") : ok("");
+      }
       return ok();
     });
 
@@ -187,7 +200,7 @@ describe("runSetupDnsProxy", () => {
         env: { DOCKER_HOST: "unix:///tmp/fake-docker.sock" },
         log,
         runDocker,
-        sleep: vi.fn(),
+        sleep,
       },
     );
 
@@ -204,6 +217,10 @@ describe("runSetupDnsProxy", () => {
       ),
     ).toBe(true);
     expect(log).toHaveBeenCalledWith("  DNS verification: 4 passed, 0 failed");
+    expect(getentCalls).toBe(3);
+    expect(sleep.mock.calls.filter(([milliseconds]) => milliseconds === 2_000)).toHaveLength(2);
+    expect(dnsReadyCalls).toBe(3);
+    expect(sleep.mock.calls).toEqual([[1_000], [1_000], [2_000], [2_000]]);
   });
 
   it("falls back to the CoreDNS pod endpoint when the kube-dns service IP is unavailable", () => {

src/lib/actions/dns/index.ts

@@ -7,6 +7,7 @@ import os from "node:os";
 import path from "node:path";
 
 import { dockerSpawnSync } from "../../adapters/docker/exec";
+import { retryUntil } from "../../core/retry";
 import {
   buildCoreDnsPatchJson,
   dockerHostRuntime,
@@ -462,30 +463,33 @@ export function runSetupDnsProxy(
     dockerEnv,
   );
 
-  let dnsReady = false;
-  for (let attempt = 0; attempt < 10; attempt += 1) {
-    const probe = kctl(
-      runDocker,
-      cluster,
-      [
-        "exec",
-        "-n",
-        "openshell",
-        pod,
-        "--",
-        "python3",
-        "-c",
-        buildDnsReadyProbePython(vethGateway),
-      ],
-      dockerEnv,
+  const dnsReady = retryUntil(
+    () =>
+      kctl(
+        runDocker,
+        cluster,
+        [
+          "exec",
+          "-n",
+          "openshell",
+          pod,
+          "--",
+          "python3",
+          "-c",
+          buildDnsReadyProbePython(vethGateway),
+        ],
+        dockerEnv,
+      ).stdout.includes("ok"),
+    {
+      accept: Boolean,
+      retryDelaysMs: Array.from({ length: 9 }, () => 1_000),
+      sleep,
+    },
+  );
+  if (!dnsReady)
+    log(
+      "WARNING: DNS forwarder did not respond after 10 attempts. The following DNS checks can report failures.",
     );
-    if (probe.stdout.includes("ok")) {
-      dnsReady = true;
-      break;
-    }
-    sleep(1000);
-  }
-  if (!dnsReady) log("WARNING: DNS forwarder not responding after 10s — verification may fail");
 
   const sandboxNamespace = selectSandboxNamespace(
     commandOutput(
@@ -686,14 +690,17 @@ export function runSetupDnsProxy(
       verificationFail += 1;
     }
 
-    let dnsResult = "";
-    for (let attempt = 1; attempt <= 3; attempt += 1) {
-      dnsResult = commandOutput(
-        sbExec(["getent", "hosts", "github.com"]) ?? { status: 1, stdout: "", stderr: "" },
-      ).trim();
-      if (dnsResult) break;
-      if (attempt < 3) sleep(2000);
-    }
+    const dnsResult = retryUntil(
+      () =>
+        commandOutput(
+          sbExec(["getent", "hosts", "github.com"]) ?? { status: 1, stdout: "", stderr: "" },
+        ).trim(),
+      {
+        accept: Boolean,
+        retryDelaysMs: [2_000, 2_000],
+        sleep,
+      },
+    );
     if (dnsResult) {
       log(`  [PASS] getent hosts github.com -> ${dnsResult}`);
       verificationPass += 1;

src/lib/actions/sandbox/connect-route-repair.test.ts

@@ -4,6 +4,8 @@
 import { describe, expect, it, vi } from "vitest";
 import type { SandboxEntry } from "../../state/registry";
 
+import { captureOpenshell } from "../../adapters/openshell/runtime";
+
 vi.mock("../../adapters/openshell/runtime", () => ({
   captureOpenshell: vi.fn(() => ({ status: 0, output: "" })),
   getOpenshellBinary: vi.fn(() => "openshell"),
@@ -37,6 +39,8 @@ vi.mock("./gateway-state", () => ({
 
 import {
   type ManagedInferenceRouteResetDeps,
+
+  probeSandboxInferenceRoute,
   repairSandboxInferenceRouteWithDeps,
   resetManagedInferenceRouteWithDeps,
   type SandboxInferenceRouteProbe,
@@ -391,3 +395,35 @@ describe("managed inference route reset unit flow", () => {
     expect(calls.unrecoverable).toEqual([{ sandboxName: "demo", detail: "BROKEN 503 still down" }]);
   });
 });
+
+
+describe("connect inference route retries", () => {
+  it("returns the third healthy probe result after two unhealthy probe results (#9218)", () => {
+    vi.mocked(captureOpenshell)
+      .mockReturnValueOnce({ status: 0, output: "BROKEN 503", stderr: "" })
+      .mockReturnValueOnce({ status: 0, output: "BROKEN 503", stderr: "" })
+      .mockReturnValueOnce({ status: 0, output: "OK 200", stderr: "" });
+
+    const result = probeSandboxInferenceRoute(
+      "alpha",
+      { name: "hermes" },
+      { attempts: 3, delayMs: 2_000 },
+    );
+
+    expect(result).toMatchObject({ healthy: true, broken: false, httpStatus: 200 });
+    expect(captureOpenshell).toHaveBeenCalledTimes(3);
+  });
+
+  it("returns the final unhealthy probe result after exhausting attempts (#9218)", () => {
+    vi.mocked(captureOpenshell).mockReturnValue({ status: 0, output: "BROKEN 503", stderr: "" });
+
+    const result = probeSandboxInferenceRoute(
+      "alpha",
+      { name: "hermes" },
+      { attempts: 2, delayMs: 500 },
+    );
+
+    expect(result).toMatchObject({ healthy: false, broken: true, httpStatus: 503 });
+    expect(captureOpenshell).toHaveBeenCalledTimes(2);
+  });
+});

src/lib/actions/sandbox/connect.ts

@@ -3,12 +3,9 @@
 
 import { spawnSync } from "node:child_process";
 import { resolveOpenshell } from "../../adapters/openshell/resolve";
+import { captureOpenshell, getOpenshellBinary, runOpenshell } from "../../adapters/openshell/runtime";
 import {
-  captureOpenshell,
-  getOpenshellBinary,
-  runOpenshell,
-} from "../../adapters/openshell/runtime";
-import {
+
   OPENSHELL_INFERENCE_ROUTE_PROBE_TIMEOUT_MS,
   OPENSHELL_OPERATION_TIMEOUT_MS,
   OPENSHELL_PROBE_TIMEOUT_MS,
@@ -17,6 +14,8 @@ import type { AgentDefinition } from "../../agent/defs";
 import * as agentRuntime from "../../agent/runtime";
 import { CLI_NAME } from "../../cli/branding";
 import { D, G, R, YW } from "../../cli/terminal-style";
+import { retryUntil } from "../../core/retry";
+
 import { spawnExitCode } from "../../core/process-exit";
 import { shellQuote } from "../../core/shell-quote";
 import { getNamedGatewayLifecycleState } from "../../gateway-runtime-action";
@@ -128,23 +127,24 @@ type SandboxListProbe = {
   output: string;
 };
 
+
 export type SandboxInferenceRouteProbe = {
   healthy: boolean;
   broken: boolean;
   httpStatus?: number;
   detail: string;
 };
 
-type SandboxInferenceRouteEnsureResult = {
-  sandbox: SandboxEntry | null;
-  routeHealthy: boolean | null;
-};
-
 type InferenceRouteProbeOptions = {
   attempts?: number;
   delayMs?: number;
 };
 
+type SandboxInferenceRouteEnsureResult = {
+  sandbox: SandboxEntry | null;
+  routeHealthy: boolean | null;
+};
+
 export type SandboxInferenceRouteRepairResult = {
   healthy: boolean;
   repairAttempted: boolean;
@@ -352,15 +352,6 @@ async function runSandboxConnectProbe(sandboxName: string): Promise<void> {
   process.exit(1);
 }
 
-function sleepSync(ms: number): void {
-  if (ms <= 0) return;
-  if (process.env.VITEST === "true" || process.env.NEMOCLAW_TEST_NO_SLEEP === "1") return;
-  spawnSync(process.execPath, ["-e", `setTimeout(() => {}, ${ms})`], {
-    stdio: "ignore",
-    timeout: ms + 1_000,
-  });
-}
-
 const GATEWAY_UNAVAILABLE_RE =
   /No gateway configured|No active gateway|Connection refused|client error \(Connect\)|tcp connect error|Status:\s*Disconnected/i;
 
@@ -412,43 +403,49 @@ function failIfGatewayBlocksConnectReadiness(sandboxName: string): void {
   }
 }
 
-function probeSandboxInferenceRoute(
+
+function sleepSync(milliseconds: number): void {
+  if (milliseconds <= 0) return;
+  if (process.env.VITEST === "true" || process.env.NEMOCLAW_TEST_NO_SLEEP === "1") return;
+  spawnSync(process.execPath, ["-e", `setTimeout(() => {}, ${milliseconds})`], {
+    stdio: "ignore",
+    timeout: milliseconds + 1_000,
+  });
+}
+
+export function probeSandboxInferenceRoute(
   sandboxName: string,
   agent: InferenceRouteProbeAgent,
   { attempts = 1, delayMs = 0 }: InferenceRouteProbeOptions = {},
 ): SandboxInferenceRouteProbe {
-  let lastProbe: SandboxInferenceRouteProbe | null = null;
-  const boundedAttempts = Math.max(1, attempts);
-
-  for (let attempt = 1; attempt <= boundedAttempts; attempt += 1) {
-    // Keep the shell string inside the sandbox: curl write-out, body capture,
-    // and status classification must run as one bounded probe. sandboxName
-    // remains an argv value, so no user input is interpolated into the script.
-    const probe = captureOpenshell(buildSandboxInferenceRouteProbeArgs(sandboxName, agent), {
-      ignoreError: true,
-      includeStreams: true,
-      timeout: OPENSHELL_INFERENCE_ROUTE_PROBE_TIMEOUT_MS,
-    });
-    const parsed = parseSandboxInferenceRouteProbeResult(probe);
-    lastProbe = {
-      healthy: parsed.healthy,
-      broken: parsed.broken,
-      httpStatus: parsed.httpStatus,
-      detail: parsed.detail,
-    };
-    if (lastProbe.healthy || attempt === boundedAttempts) return lastProbe;
-    sleepSync(delayMs);
-  }
-
-  return (
-    lastProbe ?? {
-      healthy: false,
-      broken: false,
-      detail: "inference route probe did not run",
-    }
+  const attemptCount = Math.max(1, Math.floor(attempts));
+  return retryUntil(
+    () => {
+      // Keep the shell string inside the sandbox: curl write-out, body capture,
+      // and status classification must run as one bounded probe. sandboxName
+      // remains an argv value, so no user input is interpolated into the script.
+      const probe = captureOpenshell(buildSandboxInferenceRouteProbeArgs(sandboxName, agent), {
+        ignoreError: true,
+        includeStreams: true,
+        timeout: OPENSHELL_INFERENCE_ROUTE_PROBE_TIMEOUT_MS,
+      });
+      const parsed = parseSandboxInferenceRouteProbeResult(probe);
+      return {
+        healthy: parsed.healthy,
+        broken: parsed.broken,
+        httpStatus: parsed.httpStatus,
+        detail: parsed.detail,
+      };
+    },
+    {
+      accept: (result) => result.healthy,
+      retryDelaysMs: Array.from({ length: attemptCount - 1 }, () => delayMs),
+      sleep: sleepSync,
+    },
   );
 }
 
+
 function shouldUseLegacyDnsProxyRepair(sb: SandboxEntry | null): boolean {
   // The legacy repair patches CoreDNS inside an `openshell-cluster-<name>`
   // container, which only the k3s/kubernetes gateway runs. The docker driver

src/lib/actions/sandbox/inference-route-health.ts

@@ -21,6 +21,7 @@ import {
 export type { SandboxInferenceInvocationResult } from "./inference-invocation-probe";
 export type ProbeSandboxInferenceInvocation = typeof probeSandboxInferenceInvocation;
 
+
 export type SandboxInferenceRouteHealth = {
   ok: boolean;
   endpoint: string;