fix(onboard): retry and fallback for container reachability check

src/lib/local-inference.test.ts

@@ -79,6 +79,10 @@ describe("local inference helpers", () => {
       "--add-host",
       "host.openshell.internal:host-gateway",
       CONTAINER_REACHABILITY_IMAGE,
+      "--connect-timeout",
+      "5",
+      "--max-time",
+      "10",
       "-sf",
       "http://host.openshell.internal:8000/v1/models",
     ]);
@@ -92,6 +96,10 @@ describe("local inference helpers", () => {
       "--add-host",
       "host.openshell.internal:host-gateway",
       CONTAINER_REACHABILITY_IMAGE,
+      "--connect-timeout",
+      "5",
+      "--max-time",
+      "10",
       "-sf",
       `http://host.openshell.internal:${OLLAMA_CONTAINER_PORT}/api/tags`,
     ]);
@@ -149,14 +157,96 @@ describe("local inference helpers", () => {
     let callCount = 0;
     const mockCapture = () => {
       callCount += 1;
-      return callCount === 1 ? '{"models":[]}' : "";
+      // Call 1: host check succeeds
+      if (callCount === 1) return '{"models":[]}';
+      // Calls 2-4: container check fails (3 retries)
+      // Calls 5-6: diagnostic commands fail
+      return "";
     };
-    const result = validateLocalProvider("ollama-local", mockCapture);
+    const noopSleep = () => {};
+    const result = validateLocalProvider("ollama-local", mockCapture, noopSleep);
     expect(result.ok).toBe(false);
     expect(result.message).toMatch(
       new RegExp(`host\\.openshell\\.internal:${OLLAMA_CONTAINER_PORT}`),
     );
-    expect(result.message).toMatch(/auth proxy/);
+    expect(result.message).toMatch(/Docker container reachability check failed/);
+    expect(result.message).toMatch(/sandbox uses a different network path/);
+    expect(result.message).not.toMatch(/Ensure the Ollama auth proxy is running/);
+    expect(result.diagnostic).toMatch(/Docker command failed/);
+  });
+
+  it("succeeds after container check retry", () => {
+    let callCount = 0;
+    const mockCapture = () => {
+      callCount += 1;
+      // Call 1: host check succeeds
+      if (callCount === 1) return '{"models":[]}';
+      // Call 2: container attempt 1 fails
+      if (callCount === 2) return "";
+      // Call 3: container attempt 2 succeeds
+      return '{"models":[]}';
+    };
+    const sleepCalls: number[] = [];
+    const mockSleep = (s: number) => { sleepCalls.push(s); };
+    const result = validateLocalProvider("ollama-local", mockCapture, mockSleep);
+    expect(result).toEqual({ ok: true });
+    expect(sleepCalls).toEqual([2]);
+  });
+
+  it("includes HTTP diagnostic when retries exhausted and diagnostic commands succeed", () => {
+    let callCount = 0;
+    const mockCapture = () => {
+      callCount += 1;
+      // Call 1: host check succeeds
+      if (callCount === 1) return '{"models":[]}';
+      // Calls 2-4: container check fails (3 retries)
+      if (callCount <= 4) return "";
+      // Call 5: diagnostic HTTP status
+      if (callCount === 5) return "502";
+      // Call 6: diagnostic /etc/hosts
+      return "172.17.0.1\thost.openshell.internal";
+    };
+    const sleepCalls: number[] = [];
+    const mockSleep = (s: number) => { sleepCalls.push(s); };
+    const result = validateLocalProvider("ollama-local", mockCapture, mockSleep);
+    expect(result.ok).toBe(false);
+    expect(result.diagnostic).toMatch(/HTTP 502/);
+    expect(result.diagnostic).toMatch(/host-gateway resolved to/);
+    expect(sleepCalls).toEqual([2, 2]);
+  });
+
+  it("includes docker-failed diagnostic when diagnostic commands also fail", () => {
+    let callCount = 0;
+    const mockCapture = () => {
+      callCount += 1;
+      if (callCount === 1) return '{"models":[]}';
+      return "";
+    };
+    const noopSleep = () => {};
+    const result = validateLocalProvider("ollama-local", mockCapture, noopSleep);
+    expect(result.ok).toBe(false);
+    expect(result.diagnostic).toMatch(/Docker command failed/);
+  });
+
+  it("calls sleepFn between container check retries", () => {
+    let callCount = 0;
+    const mockCapture = () => {
+      callCount += 1;
+      if (callCount === 1) return '{"models":[]}';
+      return "";
+    };
+    const sleepCalls: number[] = [];
+    const mockSleep = (s: number) => { sleepCalls.push(s); };
+    validateLocalProvider("ollama-local", mockCapture, mockSleep);
+    expect(sleepCalls).toEqual([2, 2]);
+  });
+
+  it("does not retry when host check fails", () => {
+    const sleepCalls: number[] = [];
+    const mockSleep = (s: number) => { sleepCalls.push(s); };
+    const result = validateLocalProvider("ollama-local", () => "", mockSleep);
+    expect(result.ok).toBe(false);
+    expect(sleepCalls).toEqual([]);
   });
 
   it("returns a clear error when vllm-local is unavailable", () => {
@@ -211,11 +301,18 @@ describe("local inference helpers", () => {
     let callCount = 0;
     const mockCapture = () => {
       callCount += 1;
-      return callCount === 1 ? '{"data":[]}' : "";
+      // Call 1: host check succeeds
+      if (callCount === 1) return '{"data":[]}';
+      // Calls 2+: container check + diagnostics all fail
+      return "";
     };
-    const result = validateLocalProvider("vllm-local", mockCapture);
+    const noopSleep = () => {};
+    const result = validateLocalProvider("vllm-local", mockCapture, noopSleep);
     expect(result.ok).toBe(false);
     expect(result.message).toMatch(/host\.openshell\.internal:8000/);
+    expect(result.message).toMatch(/Docker container reachability check failed/);
+    expect(result.message).toMatch(/sandbox uses a different network path/);
+    expect(result.message).not.toMatch(/Ensure the server is reachable from containers/);
   });
 
   it("treats unknown local providers as already valid", () => {

src/lib/local-inference.ts

@@ -13,6 +13,7 @@ import { runCurlProbe } from "./http-probe";
 const { shellQuote, runCapture } = require("./runner");
 
 import { VLLM_PORT, OLLAMA_PORT, OLLAMA_PROXY_PORT } from "./ports";
+import { sleepSeconds } from "./wait";
 
 // eslint-disable-next-line @typescript-eslint/no-require-imports
 const { isWsl } = require("./platform");
@@ -35,6 +36,7 @@ export interface GpuInfo {
 export interface ValidationResult {
   ok: boolean;
   message?: string;
+  diagnostic?: string;
 }
 
 export interface LocalProviderHealthStatus {
@@ -177,6 +179,10 @@ export function getLocalProviderContainerReachabilityCheck(provider: string): st
         "--add-host",
         "host.openshell.internal:host-gateway",
         CONTAINER_REACHABILITY_IMAGE,
+        "--connect-timeout",
+        "5",
+        "--max-time",
+        "10",
         "-sf",
         `http://host.openshell.internal:${VLLM_PORT}/v1/models`,
       ];
@@ -190,6 +196,10 @@ export function getLocalProviderContainerReachabilityCheck(provider: string): st
         "--add-host",
         "host.openshell.internal:host-gateway",
         CONTAINER_REACHABILITY_IMAGE,
+        "--connect-timeout",
+        "5",
+        "--max-time",
+        "10",
         "-sf",
         `http://host.openshell.internal:${OLLAMA_CONTAINER_PORT}/api/tags`,
       ];
@@ -198,9 +208,13 @@ export function getLocalProviderContainerReachabilityCheck(provider: string): st
   }
 }
 
+const CONTAINER_CHECK_MAX_ATTEMPTS = 3;
+const CONTAINER_CHECK_RETRY_DELAY_SECS = 2;
+
 export function validateLocalProvider(
   provider: string,
   runCaptureImpl?: RunCaptureFn,
+  sleepFn?: (seconds: number) => void,
 ): ValidationResult {
   if (provider === "ollama-local") {
     const portValidation = validateOllamaPortConfiguration();
@@ -210,6 +224,7 @@ export function validateLocalProvider(
   }
 
   const capture = runCaptureImpl ?? runCapture;
+  const sleep = sleepFn ?? sleepSeconds;
   const command = getLocalProviderHealthCheck(provider);
   if (!command) {
     return { ok: true };
@@ -238,30 +253,102 @@ export function validateLocalProvider(
     return { ok: true };
   }
 
-  const containerOutput = capture(containerCommand, { ignoreError: true });
-  if (containerOutput) {
-    return { ok: true };
+  // Retry container reachability check with backoff
+  for (let attempt = 1; attempt <= CONTAINER_CHECK_MAX_ATTEMPTS; attempt++) {
+    const containerOutput = capture(containerCommand, { ignoreError: true });
+    if (containerOutput) {
+      return { ok: true };
+    }
+    if (attempt < CONTAINER_CHECK_MAX_ATTEMPTS) {
+      sleep(CONTAINER_CHECK_RETRY_DELAY_SECS);
+    }
   }
 
+  // All retries exhausted — collect diagnostics
+  const diagnostic = collectContainerDiagnostic(provider, capture);
+
   switch (provider) {
     case "vllm-local":
       return {
         ok: false,
-        message: `Local vLLM is responding on 127.0.0.1, but containers cannot reach http://host.openshell.internal:${VLLM_PORT}. Ensure the server is reachable from containers, not only from the host shell.`,
+        message: `Local vLLM is responding on 127.0.0.1, but the Docker container reachability check failed for http://host.openshell.internal:${VLLM_PORT}. This may be a Docker networking issue — the sandbox uses a different network path and may still work.`,
+        diagnostic,
       };
     case "ollama-local":
       return {
         ok: false,
-        message: `Local Ollama is responding on 127.0.0.1, but containers cannot reach the auth proxy at http://host.openshell.internal:${OLLAMA_CONTAINER_PORT}. Ensure the Ollama auth proxy is running.`,
+        message: `Local Ollama is responding on 127.0.0.1, but the Docker container reachability check failed for http://host.openshell.internal:${OLLAMA_CONTAINER_PORT}. This may be a Docker networking issue — the sandbox uses a different network path and may still work.`,
+        diagnostic,
       };
     default:
       return {
         ok: false,
         message: "The selected local inference provider is unavailable from containers.",
+        diagnostic,
       };
   }
 }
 
+function getContainerCheckUrl(provider: string): string {
+  switch (provider) {
+    case "vllm-local":
+      return `http://host.openshell.internal:${VLLM_PORT}/v1/models`;
+    case "ollama-local":
+      return `http://host.openshell.internal:${OLLAMA_CONTAINER_PORT}/api/tags`;
+    default:
+      return "http://host.openshell.internal/";
+  }
+}
+
+function collectContainerDiagnostic(provider: string, capture: RunCaptureFn): string {
+  const url = getContainerCheckUrl(provider);
+  try {
+    // Get HTTP status code
+    const httpStatus = capture(
+      [
+        "docker", "run", "--rm",
+        "--add-host", "host.openshell.internal:host-gateway",
+        CONTAINER_REACHABILITY_IMAGE,
+        "-s", "-o", "/dev/null", "-w", "%{http_code}",
+        "--connect-timeout", "5", "--max-time", "10",
+        url,
+      ],
+      { ignoreError: true },
+    );
+
+    // Get /etc/hosts to see host-gateway resolution
+    const hostsOutput = capture(
+      [
+        "docker", "run", "--rm",
+        "--add-host", "host.openshell.internal:host-gateway",
+        CONTAINER_REACHABILITY_IMAGE,
+        "cat", "/etc/hosts",
+      ],
+      { ignoreError: true },
+    );
+
+    if (!httpStatus && !hostsOutput) {
+      return `Docker command failed (image pull error or runtime failure). Retried ${CONTAINER_CHECK_MAX_ATTEMPTS} times.`;
+    }
+
+    const parts: string[] = [];
+    if (httpStatus) {
+      parts.push(`Container curl returned HTTP ${httpStatus.trim()}`);
+    }
+    if (hostsOutput) {
+      const gwLine = hostsOutput.split(/\r?\n/).find((l: string) => l.includes("host.openshell.internal"));
+      if (gwLine) {
+        const ip = gwLine.trim().split(/\s+/)[0];
+        parts.push(`host-gateway resolved to: ${ip}`);
+      }
+    }
+    parts.push(`Retried ${CONTAINER_CHECK_MAX_ATTEMPTS} times over ~${(CONTAINER_CHECK_MAX_ATTEMPTS - 1) * CONTAINER_CHECK_RETRY_DELAY_SECS}s`);
+    return parts.join(". ") + ".";
+  } catch {
+    return `Docker command failed (image pull error or runtime failure). Retried ${CONTAINER_CHECK_MAX_ATTEMPTS} times.`;
+  }
+}
+
 export function parseOllamaList(output: string | null | undefined): string[] {
   return String(output || "")
     .split(/\r?\n/)

src/lib/onboard.ts

@@ -63,6 +63,7 @@ const {
   getDefaultOllamaModel,
   getBootstrapOllamaModelOptions,
   getLocalProviderBaseUrl,
+  getLocalProviderHealthCheck,
   getLocalProviderValidationBaseUrl,
   getOllamaModelOptions,
   getOllamaWarmupCommand,
@@ -5471,8 +5472,25 @@ async function setupInference(
   } else if (provider === "vllm-local") {
     const validation = validateLocalProvider(provider);
     if (!validation.ok) {
-      console.error(`  ${validation.message}`);
-      process.exit(1);
+      const hostCheck = getLocalProviderHealthCheck(provider);
+      const hostResponding = hostCheck ? !!runCapture(hostCheck, { ignoreError: true }) : false;
+
+      if (hostResponding) {
+        console.warn(`  ⚠ ${validation.message}`);
+        if (validation.diagnostic) {
+          console.warn(`  Diagnostic: ${validation.diagnostic}`);
+        }
+        console.warn(
+          "  The server is healthy on the host — continuing. " +
+            "The sandbox uses a different network path and may work correctly.",
+        );
+      } else {
+        console.error(`  ${validation.message}`);
+        if (validation.diagnostic) {
+          console.error(`  Diagnostic: ${validation.diagnostic}`);
+        }
+        process.exit(1);
+      }
     }
     const baseUrl = getLocalProviderBaseUrl(provider);
     const providerResult = upsertProvider("vllm-local", "openai", "OPENAI_API_KEY", baseUrl, {
@@ -5496,13 +5514,35 @@ async function setupInference(
   } else if (provider === "ollama-local") {
     const validation = validateLocalProvider(provider);
     if (!validation.ok) {
-      console.error(`  ${validation.message}`);
-      if (process.platform === "darwin") {
-        console.error(
-          "  On macOS, local inference also depends on OpenShell host routing support.",
+      // Before giving up, check whether the proxy is actually healthy on the host.
+      // The container reachability check uses Docker's --add-host host-gateway,
+      // which may not work on all Docker configurations (e.g., Brev, rootless Docker).
+      // The real sandbox uses a different networking stack (k3s CoreDNS + NodeHosts).
+      const proxyAlive = !isWsl() && isOllamaProxyProcess(loadPersistedProxyPid());
+      const hostCheck = getLocalProviderHealthCheck(provider);
+      const hostResponding = hostCheck ? !!runCapture(hostCheck, { ignoreError: true }) : false;
+
+      if (proxyAlive && hostResponding) {
+        console.warn(`  ⚠ ${validation.message}`);
+        if (validation.diagnostic) {
+          console.warn(`  Diagnostic: ${validation.diagnostic}`);
+        }
+        console.warn(
+          "  The auth proxy is healthy on the host — continuing. " +
+            "The sandbox uses a different network path and may work correctly.",
         );
+      } else {
+        console.error(`  ${validation.message}`);
+        if (validation.diagnostic) {
+          console.error(`  Diagnostic: ${validation.diagnostic}`);
+        }
+        if (process.platform === "darwin") {
+          console.error(
+            "  On macOS, local inference also depends on OpenShell host routing support.",
+          );
+        }
+        process.exit(1);
       }
-      process.exit(1);
     }
     const baseUrl = getLocalProviderBaseUrl(provider);
     let ollamaCredential = "ollama";