diff --git a/src/lib/local-inference.test.ts b/src/lib/local-inference.test.ts index 77136f99529..29912a9df8d 100644 --- a/src/lib/local-inference.test.ts +++ b/src/lib/local-inference.test.ts @@ -79,6 +79,10 @@ describe("local inference helpers", () => { "--add-host", "host.openshell.internal:host-gateway", CONTAINER_REACHABILITY_IMAGE, + "--connect-timeout", + "5", + "--max-time", + "10", "-sf", "http://host.openshell.internal:8000/v1/models", ]); @@ -92,6 +96,10 @@ describe("local inference helpers", () => { "--add-host", "host.openshell.internal:host-gateway", CONTAINER_REACHABILITY_IMAGE, + "--connect-timeout", + "5", + "--max-time", + "10", "-sf", `http://host.openshell.internal:${OLLAMA_CONTAINER_PORT}/api/tags`, ]); @@ -149,14 +157,96 @@ describe("local inference helpers", () => { let callCount = 0; const mockCapture = () => { callCount += 1; - return callCount === 1 ? '{"models":[]}' : ""; + // Call 1: host check succeeds + if (callCount === 1) return '{"models":[]}'; + // Calls 2-4: container check fails (3 retries) + // Calls 5-6: diagnostic commands fail + return ""; }; - const result = validateLocalProvider("ollama-local", mockCapture); + const noopSleep = () => {}; + const result = validateLocalProvider("ollama-local", mockCapture, noopSleep); expect(result.ok).toBe(false); expect(result.message).toMatch( new RegExp(`host\\.openshell\\.internal:${OLLAMA_CONTAINER_PORT}`), ); - expect(result.message).toMatch(/auth proxy/); + expect(result.message).toMatch(/Docker container reachability check failed/); + expect(result.message).toMatch(/sandbox uses a different network path/); + expect(result.message).not.toMatch(/Ensure the Ollama auth proxy is running/); + expect(result.diagnostic).toMatch(/Docker command failed/); + }); + + it("succeeds after container check retry", () => { + let callCount = 0; + const mockCapture = () => { + callCount += 1; + // Call 1: host check succeeds + if (callCount === 1) return '{"models":[]}'; + // Call 2: container attempt 1 fails + if (callCount === 2) return ""; + // Call 3: container attempt 2 succeeds + return '{"models":[]}'; + }; + const sleepCalls: number[] = []; + const mockSleep = (s: number) => { sleepCalls.push(s); }; + const result = validateLocalProvider("ollama-local", mockCapture, mockSleep); + expect(result).toEqual({ ok: true }); + expect(sleepCalls).toEqual([2]); + }); + + it("includes HTTP diagnostic when retries exhausted and diagnostic commands succeed", () => { + let callCount = 0; + const mockCapture = () => { + callCount += 1; + // Call 1: host check succeeds + if (callCount === 1) return '{"models":[]}'; + // Calls 2-4: container check fails (3 retries) + if (callCount <= 4) return ""; + // Call 5: diagnostic HTTP status + if (callCount === 5) return "502"; + // Call 6: diagnostic /etc/hosts + return "172.17.0.1\thost.openshell.internal"; + }; + const sleepCalls: number[] = []; + const mockSleep = (s: number) => { sleepCalls.push(s); }; + const result = validateLocalProvider("ollama-local", mockCapture, mockSleep); + expect(result.ok).toBe(false); + expect(result.diagnostic).toMatch(/HTTP 502/); + expect(result.diagnostic).toMatch(/host-gateway resolved to/); + expect(sleepCalls).toEqual([2, 2]); + }); + + it("includes docker-failed diagnostic when diagnostic commands also fail", () => { + let callCount = 0; + const mockCapture = () => { + callCount += 1; + if (callCount === 1) return '{"models":[]}'; + return ""; + }; + const noopSleep = () => {}; + const result = validateLocalProvider("ollama-local", mockCapture, noopSleep); + expect(result.ok).toBe(false); + expect(result.diagnostic).toMatch(/Docker command failed/); + }); + + it("calls sleepFn between container check retries", () => { + let callCount = 0; + const mockCapture = () => { + callCount += 1; + if (callCount === 1) return '{"models":[]}'; + return ""; + }; + const sleepCalls: number[] = []; + const mockSleep = (s: number) => { sleepCalls.push(s); }; + validateLocalProvider("ollama-local", mockCapture, mockSleep); + expect(sleepCalls).toEqual([2, 2]); + }); + + it("does not retry when host check fails", () => { + const sleepCalls: number[] = []; + const mockSleep = (s: number) => { sleepCalls.push(s); }; + const result = validateLocalProvider("ollama-local", () => "", mockSleep); + expect(result.ok).toBe(false); + expect(sleepCalls).toEqual([]); }); it("returns a clear error when vllm-local is unavailable", () => { @@ -211,11 +301,18 @@ describe("local inference helpers", () => { let callCount = 0; const mockCapture = () => { callCount += 1; - return callCount === 1 ? '{"data":[]}' : ""; + // Call 1: host check succeeds + if (callCount === 1) return '{"data":[]}'; + // Calls 2+: container check + diagnostics all fail + return ""; }; - const result = validateLocalProvider("vllm-local", mockCapture); + const noopSleep = () => {}; + const result = validateLocalProvider("vllm-local", mockCapture, noopSleep); expect(result.ok).toBe(false); expect(result.message).toMatch(/host\.openshell\.internal:8000/); + expect(result.message).toMatch(/Docker container reachability check failed/); + expect(result.message).toMatch(/sandbox uses a different network path/); + expect(result.message).not.toMatch(/Ensure the server is reachable from containers/); }); it("treats unknown local providers as already valid", () => { diff --git a/src/lib/local-inference.ts b/src/lib/local-inference.ts index e0465a94969..7fa39d1148d 100644 --- a/src/lib/local-inference.ts +++ b/src/lib/local-inference.ts @@ -13,6 +13,7 @@ import { runCurlProbe } from "./http-probe"; const { shellQuote, runCapture } = require("./runner"); import { VLLM_PORT, OLLAMA_PORT, OLLAMA_PROXY_PORT } from "./ports"; +import { sleepSeconds } from "./wait"; // eslint-disable-next-line @typescript-eslint/no-require-imports const { isWsl } = require("./platform"); @@ -35,6 +36,7 @@ export interface GpuInfo { export interface ValidationResult { ok: boolean; message?: string; + diagnostic?: string; } export interface LocalProviderHealthStatus { @@ -177,6 +179,10 @@ export function getLocalProviderContainerReachabilityCheck(provider: string): st "--add-host", "host.openshell.internal:host-gateway", CONTAINER_REACHABILITY_IMAGE, + "--connect-timeout", + "5", + "--max-time", + "10", "-sf", `http://host.openshell.internal:${VLLM_PORT}/v1/models`, ]; @@ -190,6 +196,10 @@ export function getLocalProviderContainerReachabilityCheck(provider: string): st "--add-host", "host.openshell.internal:host-gateway", CONTAINER_REACHABILITY_IMAGE, + "--connect-timeout", + "5", + "--max-time", + "10", "-sf", `http://host.openshell.internal:${OLLAMA_CONTAINER_PORT}/api/tags`, ]; @@ -198,9 +208,13 @@ export function getLocalProviderContainerReachabilityCheck(provider: string): st } } +const CONTAINER_CHECK_MAX_ATTEMPTS = 3; +const CONTAINER_CHECK_RETRY_DELAY_SECS = 2; + export function validateLocalProvider( provider: string, runCaptureImpl?: RunCaptureFn, + sleepFn?: (seconds: number) => void, ): ValidationResult { if (provider === "ollama-local") { const portValidation = validateOllamaPortConfiguration(); @@ -210,6 +224,7 @@ export function validateLocalProvider( } const capture = runCaptureImpl ?? runCapture; + const sleep = sleepFn ?? sleepSeconds; const command = getLocalProviderHealthCheck(provider); if (!command) { return { ok: true }; @@ -238,30 +253,102 @@ export function validateLocalProvider( return { ok: true }; } - const containerOutput = capture(containerCommand, { ignoreError: true }); - if (containerOutput) { - return { ok: true }; + // Retry container reachability check with backoff + for (let attempt = 1; attempt <= CONTAINER_CHECK_MAX_ATTEMPTS; attempt++) { + const containerOutput = capture(containerCommand, { ignoreError: true }); + if (containerOutput) { + return { ok: true }; + } + if (attempt < CONTAINER_CHECK_MAX_ATTEMPTS) { + sleep(CONTAINER_CHECK_RETRY_DELAY_SECS); + } } + // All retries exhausted — collect diagnostics + const diagnostic = collectContainerDiagnostic(provider, capture); + switch (provider) { case "vllm-local": return { ok: false, - message: `Local vLLM is responding on 127.0.0.1, but containers cannot reach http://host.openshell.internal:${VLLM_PORT}. Ensure the server is reachable from containers, not only from the host shell.`, + message: `Local vLLM is responding on 127.0.0.1, but the Docker container reachability check failed for http://host.openshell.internal:${VLLM_PORT}. This may be a Docker networking issue — the sandbox uses a different network path and may still work.`, + diagnostic, }; case "ollama-local": return { ok: false, - message: `Local Ollama is responding on 127.0.0.1, but containers cannot reach the auth proxy at http://host.openshell.internal:${OLLAMA_CONTAINER_PORT}. Ensure the Ollama auth proxy is running.`, + message: `Local Ollama is responding on 127.0.0.1, but the Docker container reachability check failed for http://host.openshell.internal:${OLLAMA_CONTAINER_PORT}. This may be a Docker networking issue — the sandbox uses a different network path and may still work.`, + diagnostic, }; default: return { ok: false, message: "The selected local inference provider is unavailable from containers.", + diagnostic, }; } } +function getContainerCheckUrl(provider: string): string { + switch (provider) { + case "vllm-local": + return `http://host.openshell.internal:${VLLM_PORT}/v1/models`; + case "ollama-local": + return `http://host.openshell.internal:${OLLAMA_CONTAINER_PORT}/api/tags`; + default: + return "http://host.openshell.internal/"; + } +} + +function collectContainerDiagnostic(provider: string, capture: RunCaptureFn): string { + const url = getContainerCheckUrl(provider); + try { + // Get HTTP status code + const httpStatus = capture( + [ + "docker", "run", "--rm", + "--add-host", "host.openshell.internal:host-gateway", + CONTAINER_REACHABILITY_IMAGE, + "-s", "-o", "/dev/null", "-w", "%{http_code}", + "--connect-timeout", "5", "--max-time", "10", + url, + ], + { ignoreError: true }, + ); + + // Get /etc/hosts to see host-gateway resolution + const hostsOutput = capture( + [ + "docker", "run", "--rm", + "--add-host", "host.openshell.internal:host-gateway", + CONTAINER_REACHABILITY_IMAGE, + "cat", "/etc/hosts", + ], + { ignoreError: true }, + ); + + if (!httpStatus && !hostsOutput) { + return `Docker command failed (image pull error or runtime failure). Retried ${CONTAINER_CHECK_MAX_ATTEMPTS} times.`; + } + + const parts: string[] = []; + if (httpStatus) { + parts.push(`Container curl returned HTTP ${httpStatus.trim()}`); + } + if (hostsOutput) { + const gwLine = hostsOutput.split(/\r?\n/).find((l: string) => l.includes("host.openshell.internal")); + if (gwLine) { + const ip = gwLine.trim().split(/\s+/)[0]; + parts.push(`host-gateway resolved to: ${ip}`); + } + } + parts.push(`Retried ${CONTAINER_CHECK_MAX_ATTEMPTS} times over ~${(CONTAINER_CHECK_MAX_ATTEMPTS - 1) * CONTAINER_CHECK_RETRY_DELAY_SECS}s`); + return parts.join(". ") + "."; + } catch { + return `Docker command failed (image pull error or runtime failure). Retried ${CONTAINER_CHECK_MAX_ATTEMPTS} times.`; + } +} + export function parseOllamaList(output: string | null | undefined): string[] { return String(output || "") .split(/\r?\n/) diff --git a/src/lib/onboard-ollama-proxy.ts b/src/lib/onboard-ollama-proxy.ts index c9938fbf148..f63388b8227 100644 --- a/src/lib/onboard-ollama-proxy.ts +++ b/src/lib/onboard-ollama-proxy.ts @@ -234,6 +234,40 @@ function getOllamaProxyToken(): string | null { return ollamaProxyToken; } +/** + * Check whether the Ollama auth proxy is actually healthy — not just that + * the PID exists, but that the proxy endpoint responds to HTTP requests. + * + * This is the correct check for the setupInference fallback: if the + * container reachability test fails (Docker bridge issue) but the proxy + * is confirmed healthy on the host, onboarding can safely continue. + */ +function isProxyHealthy(): boolean { + // 1. PID check — informational, but don't early-return on failure. + // The proxy may have been restarted with a new PID that isn't in our + // PID file, so the HTTP probe is the authoritative signal. + const pid = loadPersistedProxyPid(); + const hasValidPid = isOllamaProxyProcess(pid); + + // 2. HTTP probe — confirm the proxy actually responds. This is the + // authoritative check: a successful probe wins even if the PID file + // is missing or stale (e.g., after a manual restart). + const proxyUrl = `http://127.0.0.1:${OLLAMA_PROXY_PORT}/api/tags`; + const token = loadPersistedProxyToken(); + const probeCmd = token + ? ["curl", "-sf", "--connect-timeout", "3", "--max-time", "5", + "-H", `Authorization: Bearer ${token}`, proxyUrl] + : ["curl", "-sf", "--connect-timeout", "3", "--max-time", "5", proxyUrl]; + + const output = runCapture(probeCmd, { ignoreError: true }); + if (output) return true; + + // HTTP probe failed — fall back to PID as a weaker signal. + // This covers edge cases where the probe transiently fails but the + // process is confirmed alive. + return hasValidPid; +} + async function promptOllamaModel(gpu = null) { const installed = getOllamaModelOptions(); const options = installed.length > 0 ? installed : getBootstrapOllamaModelOptions(gpu); @@ -308,6 +342,8 @@ function prepareOllamaModel(model, installedModels = []) { module.exports = { ensureOllamaAuthProxy, getOllamaProxyToken, + isProxyHealthy, + killStaleProxy, persistProxyToken, startOllamaAuthProxy, promptOllamaModel, diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 8d0b4392f0d..82aed2f6101 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -80,6 +80,7 @@ const { getDefaultOllamaModel, getBootstrapOllamaModelOptions, getLocalProviderBaseUrl, + getLocalProviderHealthCheck, getLocalProviderValidationBaseUrl, getOllamaModelOptions, getOllamaWarmupCommand, @@ -87,6 +88,14 @@ const { validateOllamaModel, validateLocalProvider, } = localInference; +const { + ensureOllamaAuthProxy, + getOllamaProxyToken, + isProxyHealthy, + killStaleProxy, + persistProxyToken, + startOllamaAuthProxy, +} = require("./onboard-ollama-proxy"); const inferenceConfig: typeof import("./inference-config") = require("./inference-config"); const { DEFAULT_CLOUD_MODEL, getProviderSelectionConfig, parseGatewayInference } = inferenceConfig; @@ -1891,12 +1900,9 @@ const { shouldIncludeBuildContextPath, copyBuildContextDir, printSandboxCreateRe // classifySandboxCreateFailure — see validation import above // --------------------------------------------------------------------------- -// Ollama auth proxy — moved to onboard-ollama-proxy.ts +// Ollama model prompt/pull/prepare functions — from onboard-ollama-proxy.ts +// (proxy lifecycle functions already imported at the top of this file) const { - ensureOllamaAuthProxy, - getOllamaProxyToken, - persistProxyToken, - startOllamaAuthProxy, promptOllamaModel, printOllamaExposureWarning, pullOllamaModel, @@ -5137,8 +5143,29 @@ async function setupInference( } else if (provider === "vllm-local") { const validation = validateLocalProvider(provider); if (!validation.ok) { - console.error(` ${validation.message}`); - process.exit(1); + const hostCheck = getLocalProviderHealthCheck(provider); + // Use run() and check exit status rather than coercing runCapture() output + // to boolean — curl -sf can leave output even on failure in edge cases. + const hostResponding = hostCheck + ? run(hostCheck, { ignoreError: true, suppressOutput: true }).status === 0 + : false; + + if (hostResponding) { + console.warn(` ⚠ ${validation.message}`); + if (validation.diagnostic) { + console.warn(` Diagnostic: ${validation.diagnostic}`); + } + console.warn( + " The server is healthy on the host — continuing. " + + "The sandbox uses a different network path and may work correctly.", + ); + } else { + console.error(` ${validation.message}`); + if (validation.diagnostic) { + console.error(` Diagnostic: ${validation.diagnostic}`); + } + process.exit(1); + } } const baseUrl = getLocalProviderBaseUrl(provider); // Use a dedicated internal credential env so the gateway does not pick @@ -5172,19 +5199,44 @@ async function setupInference( // to unrelated OpenAI-backed sandboxes. } else if (provider === "ollama-local") { const validation = validateLocalProvider(provider); + let proxyReady = false; if (!validation.ok) { - console.error(` ${validation.message}`); - if (process.platform === "darwin") { - console.error( - " On macOS, local inference also depends on OpenShell host routing support.", + // The container reachability check uses Docker's --add-host host-gateway, + // which may not work on all Docker configurations (e.g., Brev, rootless). + // The real sandbox uses k3s CoreDNS + NodeHosts — a different path. + // Try to start/restart the auth proxy before probing — this recovers + // from stale or missing proxy processes before we decide to abort. + if (!isWsl()) { + ensureOllamaAuthProxy(); + proxyReady = isProxyHealthy(); + } + if (proxyReady) { + console.warn(` ⚠ ${validation.message}`); + if (validation.diagnostic) { + console.warn(` Diagnostic: ${validation.diagnostic}`); + } + console.warn( + " The auth proxy is healthy on the host — continuing. " + + "The sandbox uses a different network path and may work correctly.", ); + } else { + console.error(` ${validation.message}`); + if (validation.diagnostic) { + console.error(` Diagnostic: ${validation.diagnostic}`); + } + if (process.platform === "darwin") { + console.error( + " On macOS, local inference also depends on OpenShell host routing support.", + ); + } + process.exit(1); } - process.exit(1); } const baseUrl = getLocalProviderBaseUrl(provider); let ollamaCredential = "ollama"; if (!isWsl()) { - ensureOllamaAuthProxy(); + // Skip if already started during the fallback recovery above. + if (!proxyReady) ensureOllamaAuthProxy(); const proxyToken = getOllamaProxyToken(); if (!proxyToken) { console.error( diff --git a/src/nemoclaw.ts b/src/nemoclaw.ts index d3755dabc3b..cbb22225687 100644 --- a/src/nemoclaw.ts +++ b/src/nemoclaw.ts @@ -33,10 +33,10 @@ const { fetchGatewayAuthTokenFromSandbox, startGatewayForRecovery, pruneKnownHostsEntries, - ensureOllamaAuthProxy, hydrateCredentialEnv, isNonInteractive, } = require("./lib/onboard"); +const { ensureOllamaAuthProxy } = require("./lib/onboard-ollama-proxy"); const { parseGatewayTokenArgs, runGatewayTokenCommand } = require("./lib/gateway-token-command"); const { getCredential,