diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 8fcd081e676..59dc2eb944a 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1048,6 +1048,38 @@ $ nemoclaw onboard Docker Desktop, WSL, and hosts without the OpenShell Docker network use different routing models. In those cases NemoClaw treats an unavailable sandbox-side probe as non-blocking and relies on the regular proxy health check. +### `host.docker.internal` does not reliably reach the host from the sandbox + +Configuring an inference provider with a base URL like +`http://host.docker.internal:11434/v1` does not reliably reach a host Ollama +service from inside the OpenShell sandbox. +OpenShell runs sandboxes inside a k3s network, where `host.docker.internal` is +not a portable host-service route. Depending on the platform, it may fail DNS +resolution or resolve to an internal gateway/bridge address where the host's +port `11434` is not forwarded. The sandbox then sees a DNS failure or +`connection refused`: + +```console +$ getent hosts host.docker.internal +172.17.0.1 host.docker.internal host.openshell.internal +$ no_proxy=host.docker.internal curl -v http://host.docker.internal:11434/api/tags +* connect to 172.17.0.1 port 11434 failed: Connection refused +``` + +For local Ollama, use the auth-proxy URL that NemoClaw's "Local Ollama" onboard +option configures automatically: + +```text +http://host.openshell.internal:11435/v1 +``` + +`host.openshell.internal` resolves to the same gateway IP, and the +[token-gated Ollama auth proxy](#ollama-auth-proxy-did-not-start) binds port +`11435` there and forwards requests to `127.0.0.1:11434` on the host. +If you need a different host service exposed to the sandbox, route it through +the OpenShell gateway rather than relying on `host.docker.internal`. +See issue [#3136](https://github.com/NVIDIA/NemoClaw/issues/3136). + ### Local inference health check resolves to IPv6 Local inference health checks now use `127.0.0.1` instead of `localhost`. diff --git a/src/lib/inference/onboard-host-docker-internal.test.ts b/src/lib/inference/onboard-host-docker-internal.test.ts new file mode 100644 index 00000000000..7c157491279 --- /dev/null +++ b/src/lib/inference/onboard-host-docker-internal.test.ts @@ -0,0 +1,105 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const { + isHijackedDockerInternalUrl, +} = require("../../../dist/lib/inference/onboard-host-docker-internal"); +const { isSandboxInternalUrl, probeOpenAiLikeEndpoint } = require("../../../dist/lib/inference/onboard-probes"); + +describe("host.docker.internal onboarding inference policy", () => { + it("does not treat host.docker.internal as a usable sandbox URL", () => { + expect(isSandboxInternalUrl("http://host.docker.internal:11434/v1")).toBe(false); + expect(isHijackedDockerInternalUrl("http://host.docker.internal:11434/v1")).toBe(true); + expect(isHijackedDockerInternalUrl("http://host.openshell.internal:11435/v1")).toBe(false); + expect(isHijackedDockerInternalUrl("https://api.openai.com/v1")).toBe(false); + }); + + it("rejects host.docker.internal URLs with an actionable proxy hint (#3136)", () => { + const result = probeOpenAiLikeEndpoint( + "http://host.docker.internal:11434/v1", + "openai/nemotron-mini", + "", + ); + expect(result.ok).toBe(false); + expect(result.message).toMatch(/host\.docker\.internal/); + expect(result.message).toMatch(/host\.openshell\.internal:11435/); + expect(result.failures).toEqual([ + expect.objectContaining({ name: "host.docker.internal reachability" }), + ]); + }); + + it("rejects host.docker.internal even when strict chat-completions tool calling is required", () => { + const result = probeOpenAiLikeEndpoint( + "http://host.docker.internal:11434/v1", + "openai/nemotron-mini", + "", + { skipResponsesProbe: true, requireChatCompletionsToolCalling: true }, + ); + + expect(result).toMatchObject({ ok: false }); + expect(result.message).toMatch(/host\.docker\.internal/); + expect(result.message).toMatch(/host\.openshell\.internal:11435/); + }); + + it("allows explicit Windows-host Ollama validation to probe host.docker.internal", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-host-docker-probe-")); + const fakeBin = path.join(tmpDir, "bin"); + const seenUrl = path.join(tmpDir, "url"); + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync( + path.join(fakeBin, "curl"), + `#!/usr/bin/env bash +outfile="" +url="" +while [ "$#" -gt 0 ]; do + case "$1" in + -o) outfile="$2"; shift 2 ;; + -w) shift 2 ;; + *) url="$1"; shift ;; + esac +done +printf '%s' "$url" > "${seenUrl}" +if [ -n "$outfile" ]; then + cat <<'JSON' > "$outfile" +{"choices":[{"message":{"tool_calls":[{"id":"call_1","type":"function","function":{"name":"sessions_send","arguments":"{\\"message\\":\\"hello\\"}"}}]}}]} +JSON +fi +printf '200' +exit 0 +`, + { mode: 0o755 }, + ); + + const originalPath = process.env.PATH; + process.env.PATH = `${fakeBin}:${originalPath || ""}`; + try { + const result = probeOpenAiLikeEndpoint( + "http://host.docker.internal:11434/v1", + "openai/nemotron-mini", + "", + { + skipResponsesProbe: true, + requireChatCompletionsToolCalling: true, + allowHostDockerInternal: true, + }, + ); + + expect(result).toMatchObject({ + ok: true, + api: "openai-completions", + label: "Chat Completions API", + }); + expect(fs.readFileSync(seenUrl, "utf8")).toBe( + "http://host.docker.internal:11434/v1/chat/completions", + ); + } finally { + process.env.PATH = originalPath; + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/inference/onboard-host-docker-internal.ts b/src/lib/inference/onboard-host-docker-internal.ts new file mode 100644 index 00000000000..d406d50c689 --- /dev/null +++ b/src/lib/inference/onboard-host-docker-internal.ts @@ -0,0 +1,45 @@ +// @ts-nocheck +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const HOST_DOCKER_INTERNAL = "host.docker.internal"; +const OLLAMA_PROXY_URL = "http://host.openshell.internal:11435/v1"; + +function isHijackedDockerInternalUrl(url) { + try { + return new URL(String(url)).hostname === HOST_DOCKER_INTERNAL; + } catch { + return false; + } +} + +function getHostDockerInternalProbeFailure() { + return { + ok: false, + message: + `${HOST_DOCKER_INTERNAL} does not reach the host machine from inside the sandbox: ` + + `OpenShell k3s sandboxes do not provide it as a reliable host-service route. ` + + `It may fail DNS resolution or resolve to a gateway/bridge address where port ` + + `11434 is not forwarded. For local Ollama, use the auth-proxy URL ` + + `${OLLAMA_PROXY_URL} (the URL NemoClaw onboard configures automatically ` + + `when you pick "Local Ollama"). See issue #3136.`, + failures: [ + { + name: "host.docker.internal reachability", + httpStatus: 0, + curlStatus: 0, + message: + `${HOST_DOCKER_INTERNAL} is not a reliable host-service route from ` + + `OpenShell k3s sandboxes and cannot be used as an inference base URL.`, + body: "", + }, + ], + }; +} + +module.exports = { + HOST_DOCKER_INTERNAL, + OLLAMA_PROXY_URL, + isHijackedDockerInternalUrl, + getHostDockerInternalProbeFailure, +}; diff --git a/src/lib/inference/onboard-probes.test.ts b/src/lib/inference/onboard-probes.test.ts index ca624ca4256..20a4a0cf3e0 100644 --- a/src/lib/inference/onboard-probes.test.ts +++ b/src/lib/inference/onboard-probes.test.ts @@ -349,9 +349,8 @@ describe("OpenAI-compatible inference probes", () => { }); describe("sandbox-internal URL handling", () => { - it("identifies host.openshell.internal and host.docker.internal as sandbox-internal", () => { + it("identifies host.openshell.internal as sandbox-internal", () => { expect(isSandboxInternalUrl("http://host.openshell.internal:8001/v1")).toBe(true); - expect(isSandboxInternalUrl("http://host.docker.internal:11434/v1")).toBe(true); }); it("does not treat normal hostnames as sandbox-internal", () => { @@ -374,69 +373,6 @@ describe("OpenAI-compatible inference probes", () => { expect(result.note).toMatch(/only resolves inside the sandbox/); }); - it("skips the curl probe for host.docker.internal and returns ok with a note", () => { - const result = probeOpenAiLikeEndpoint( - "http://host.docker.internal:11434/v1", - "openai/nemotron-mini", - "", - ); - expect(result).toMatchObject({ ok: true, api: null }); - expect(result.note).toMatch(/host\.docker\.internal/); - }); - - it("probes host.docker.internal when strict chat-completions tool calling is required", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-host-docker-probe-")); - const fakeBin = path.join(tmpDir, "bin"); - const seenUrl = path.join(tmpDir, "url"); - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync( - path.join(fakeBin, "curl"), - `#!/usr/bin/env bash -outfile="" -url="" -while [ "$#" -gt 0 ]; do - case "$1" in - -o) outfile="$2"; shift 2 ;; - -w) shift 2 ;; - *) url="$1"; shift ;; - esac -done -printf '%s' "$url" > "${seenUrl}" -if [ -n "$outfile" ]; then - cat <<'JSON' > "$outfile" -{"choices":[{"message":{"tool_calls":[{"id":"call_1","type":"function","function":{"name":"sessions_send","arguments":"{\\"message\\":\\"hello\\"}"}}]}}]} -JSON -fi -printf '200' -exit 0 -`, - { mode: 0o755 }, - ); - - const originalPath = process.env.PATH; - process.env.PATH = `${fakeBin}:${originalPath || ""}`; - try { - const result = probeOpenAiLikeEndpoint( - "http://host.docker.internal:11434/v1", - "openai/nemotron-mini", - "", - { skipResponsesProbe: true, requireChatCompletionsToolCalling: true }, - ); - - expect(result).toMatchObject({ - ok: true, - api: "openai-completions", - label: "Chat Completions API", - }); - expect(fs.readFileSync(seenUrl, "utf8")).toBe( - "http://host.docker.internal:11434/v1/chat/completions", - ); - } finally { - process.env.PATH = originalPath; - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - it("fails closed for unprobeable sandbox-internal URLs when strict tool calling is required", () => { const result = probeOpenAiLikeEndpoint( "http://host.openshell.internal:8001/v1", diff --git a/src/lib/inference/onboard-probes.ts b/src/lib/inference/onboard-probes.ts index 641cd31a66f..ee3248312d1 100644 --- a/src/lib/inference/onboard-probes.ts +++ b/src/lib/inference/onboard-probes.ts @@ -8,6 +8,10 @@ const { getCredential, normalizeCredentialValue, resolveProviderCredential } = require("../credentials/store"); const { isWsl } = require("../platform"); const httpProbe = require("../adapters/http/probe"); +const { + getHostDockerInternalProbeFailure, + isHijackedDockerInternalUrl, +} = require("./onboard-host-docker-internal"); const { isNvcfFunctionNotFoundForAccount, nvcfFunctionNotFoundMessage, @@ -28,7 +32,7 @@ const { // so host-side validation cannot prove reachability for that URL. For ordinary // verification we still skip these endpoints, but strict tool-call validation // must fail closed unless the host is probeable from the onboard process. -const SANDBOX_INTERNAL_HOSTS = ["host.openshell.internal", "host.docker.internal"]; +const SANDBOX_INTERNAL_HOSTS = ["host.openshell.internal"]; function isSandboxInternalUrl(url) { try { @@ -484,6 +488,10 @@ function runChatCompletionsProbe({ authHeader, model, url, isWsl: isWslOverride } function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { + if (isHijackedDockerInternalUrl(endpointUrl) && options.allowHostDockerInternal !== true) { + return getHostDockerInternalProbeFailure(); + } + if (isSandboxInternalUrl(endpointUrl)) { const { hostname } = new URL(String(endpointUrl)); if (options.requireChatCompletionsToolCalling !== true) { @@ -494,21 +502,19 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { note: `${hostname} only resolves inside the sandbox — validation skipped. If the endpoint is unreachable at runtime, re-run onboard with a routable URL.`, }; } - if (hostname !== "host.docker.internal") { - return { - ok: false, - message: `${hostname} only resolves inside the sandbox and cannot be validated for required structured Chat Completions tool calls from the host. Use a routable endpoint URL and retry onboard.`, - failures: [ - { - name: "Chat Completions API with tool calling", - httpStatus: 0, - curlStatus: 0, - message: "sandbox-internal endpoint cannot be strictly validated from host", - body: "", - }, - ], - }; - } + return { + ok: false, + message: `${hostname} only resolves inside the sandbox and cannot be validated for required structured Chat Completions tool calls from the host. Use a routable endpoint URL and retry onboard.`, + failures: [ + { + name: "Chat Completions API with tool calling", + httpStatus: 0, + curlStatus: 0, + message: "sandbox-internal endpoint cannot be strictly validated from host", + body: "", + }, + ], + }; } const useQueryParam = options.authMode === "query-param"; @@ -787,6 +793,7 @@ function probeAnthropicEndpoint(endpointUrl, model, apiKey) { module.exports = { isSandboxInternalUrl, + isHijackedDockerInternalUrl, parseJsonObject, hasResponsesToolCall, hasChatCompletionsToolCall, diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index d7accfd73e9..4940e56d285 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2120,9 +2120,6 @@ function verifyWebSearchInsideSandbox( }); } -// getSandboxInferenceConfig — moved to onboard-providers.ts - -// Inference probes — moved to inference/onboard-probes.ts const { hasResponsesToolCall, hasChatCompletionsToolCall, @@ -2148,6 +2145,7 @@ async function validateOpenAiLikeSelection( requireChatCompletionsToolCalling?: boolean; skipResponsesProbe?: boolean; probeStreaming?: boolean; + allowHostDockerInternal?: boolean; } = {}, ): Promise { const apiKey = credentialEnv ? getCredential(credentialEnv) : ""; @@ -6025,6 +6023,8 @@ async function selectAndValidateOllamaModel( { skipResponsesProbe: true, requireChatCompletionsToolCalling: true, + allowHostDockerInternal: + localInference.getResolvedOllamaHost() === OLLAMA_HOST_DOCKER_INTERNAL, }, ); if (validation.retry === "selection") return { outcome: "back-to-selection" };