fix(e2e): stabilize double onboard route probe

src/lib/actions/sandbox/connect.ts

@@ -64,6 +64,14 @@ type SandboxInferenceRouteEnsureResult = {
   routeHealthy: boolean | null;
 };
 
+type InferenceRouteProbeOptions = {
+  attempts?: number;
+  delayMs?: number;
+};
+
+const INFERENCE_ROUTE_POST_REPAIR_PROBE_ATTEMPTS = 3;
+const INFERENCE_ROUTE_POST_REPAIR_PROBE_DELAY_MS = 2_000;
+
 const SANDBOX_CONNECT_FLAGS = new Set([
   "--dangerously-skip-permissions",
   "--probe-only",
@@ -151,32 +159,56 @@ function runSandboxConnectProbe(sandboxName: string): void {
   process.exit(1);
 }
 
-function probeSandboxInferenceRoute(sandboxName: string): SandboxInferenceRouteProbe {
-  // Keep the shell string inside the sandbox: curl write-out, body capture,
-  // and status classification must run as one bounded probe. sandboxName
-  // remains an argv value, so no user input is interpolated into the script.
-  const probe = captureOpenshell(
-    [
-      "sandbox",
-      "exec",
-      "--name",
-      sandboxName,
-      "--",
-      "sh",
-      "-c",
+function sleepSync(ms: number): void {
+  if (ms <= 0) return;
+  spawnSync(process.execPath, ["-e", `setTimeout(() => {}, ${ms})`], {
+    stdio: "ignore",
+    timeout: ms + 1_000,
+  });
+}
+
+function probeSandboxInferenceRoute(
+  sandboxName: string,
+  { attempts = 1, delayMs = 0 }: InferenceRouteProbeOptions = {},
+): SandboxInferenceRouteProbe {
+  let lastProbe: SandboxInferenceRouteProbe | null = null;
+  const boundedAttempts = Math.max(1, attempts);
+
+  for (let attempt = 1; attempt <= boundedAttempts; attempt += 1) {
+    // Keep the shell string inside the sandbox: curl write-out, body capture,
+    // and status classification must run as one bounded probe. sandboxName
+    // remains an argv value, so no user input is interpolated into the script.
+    const probe = captureOpenshell(
       [
-        "OUT=/tmp/nemoclaw-inference-route-probe.out",
-        "HTTP_CODE=$(curl -sk -o \"$OUT\" -w '%{http_code}' --connect-timeout 3 --max-time 8 https://inference.local/v1/models 2>/dev/null) || HTTP_CODE=000",
-        "case \"$HTTP_CODE\" in 000|5*) printf 'BROKEN %s ' \"$HTTP_CODE\"; head -c 160 \"$OUT\" 2>/dev/null || true ;; *) printf 'OK %s' \"$HTTP_CODE\" ;; esac",
-      ].join("; "),
-    ],
-    { ignoreError: true, timeout: OPENSHELL_INFERENCE_ROUTE_PROBE_TIMEOUT_MS },
-  );
-  const detail = probe.output.trim();
-  return {
-    healthy: probe.status === 0 && /^OK\s+[0-9]{3}\b/.test(detail),
-    broken: /^BROKEN\s+[0-9]{3}\b/.test(detail),
-    detail: detail || `openshell sandbox exec exited with status ${String(probe.status)}`,
+        "sandbox",
+        "exec",
+        "--name",
+        sandboxName,
+        "--",
+        "sh",
+        "-c",
+        [
+          "OUT=/tmp/nemoclaw-inference-route-probe.out",
+          "HTTP_CODE=$(curl -sk -o \"$OUT\" -w '%{http_code}' --connect-timeout 3 --max-time 8 https://inference.local/v1/models 2>/dev/null) || HTTP_CODE=000",
+          "case \"$HTTP_CODE\" in 000|5*) printf 'BROKEN %s ' \"$HTTP_CODE\"; head -c 160 \"$OUT\" 2>/dev/null || true ;; *) printf 'OK %s' \"$HTTP_CODE\" ;; esac",
+        ].join("; "),
+      ],
+      { ignoreError: true, timeout: OPENSHELL_INFERENCE_ROUTE_PROBE_TIMEOUT_MS },
+    );
+    const detail = probe.output.trim();
+    lastProbe = {
+      healthy: probe.status === 0 && /^OK\s+[0-9]{3}\b/.test(detail),
+      broken: /^BROKEN\s+[0-9]{3}\b/.test(detail),
+      detail: detail || `openshell sandbox exec exited with status ${String(probe.status)}`,
+    };
+    if (lastProbe.healthy || attempt === boundedAttempts) return lastProbe;
+    sleepSync(delayMs);
+  }
+
+  return lastProbe ?? {
+    healthy: false,
+    broken: false,
+    detail: "inference route probe did not run",
   };
 }
 
@@ -237,7 +269,10 @@ function repairSandboxInferenceRouteIfNeeded(
         );
       }
       const patch = applyOpenShellVmDnsMonkeypatch(sandboxName, sb);
-      const patchedProbe = patch.ok ? probeSandboxInferenceRoute(sandboxName) : null;
+      const patchedProbe = patch.ok ? probeSandboxInferenceRoute(sandboxName, {
+        attempts: INFERENCE_ROUTE_POST_REPAIR_PROBE_ATTEMPTS,
+        delayMs: INFERENCE_ROUTE_POST_REPAIR_PROBE_DELAY_MS,
+      }) : null;
       if (patchedProbe?.healthy) {
         if (!quiet) {
           console.log("  inference.local route repaired.");
@@ -316,7 +351,10 @@ function repairSandboxInferenceRouteIfNeeded(
     };
   }
 
-  const repairedProbe = probeSandboxInferenceRoute(sandboxName);
+  const repairedProbe = probeSandboxInferenceRoute(sandboxName, {
+    attempts: INFERENCE_ROUTE_POST_REPAIR_PROBE_ATTEMPTS,
+    delayMs: INFERENCE_ROUTE_POST_REPAIR_PROBE_DELAY_MS,
+  });
   if (!quiet) {
     if (repairedProbe.healthy) {
       console.log("  inference.local route repaired.");
@@ -411,7 +449,10 @@ function resetManagedInferenceRoute(
     timeout: OPENSHELL_OPERATION_TIMEOUT_MS,
   });
   if (resetResult.status !== 0) {
-    const finalProbe = probeSandboxInferenceRoute(sandboxName);
+    const finalProbe = probeSandboxInferenceRoute(sandboxName, {
+      attempts: INFERENCE_ROUTE_POST_REPAIR_PROBE_ATTEMPTS,
+      delayMs: INFERENCE_ROUTE_POST_REPAIR_PROBE_DELAY_MS,
+    });
     if (finalProbe.healthy) {
       if (!quiet) {
         console.log("  inference.local route repaired.");
@@ -433,7 +474,10 @@ function resetManagedInferenceRoute(
     return false;
   }
 
-  const finalProbe = probeSandboxInferenceRoute(sandboxName);
+  const finalProbe = probeSandboxInferenceRoute(sandboxName, {
+    attempts: INFERENCE_ROUTE_POST_REPAIR_PROBE_ATTEMPTS,
+    delayMs: INFERENCE_ROUTE_POST_REPAIR_PROBE_DELAY_MS,
+  });
   if (finalProbe.healthy) {
     if (!quiet) {
       console.log("  inference.local route repaired.");