Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions docs/reference/troubleshooting.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -1048,6 +1048,38 @@ $ nemoclaw onboard
Docker Desktop, WSL, and hosts without the OpenShell Docker network use different routing models.
In those cases NemoClaw treats an unavailable sandbox-side probe as non-blocking and relies on the regular proxy health check.

### `host.docker.internal` does not reliably reach the host from the sandbox

Configuring an inference provider with a base URL like
`http://host.docker.internal:11434/v1` does not reliably reach a host Ollama
service from inside the OpenShell sandbox.
OpenShell runs sandboxes inside a k3s network, where `host.docker.internal` is
not a portable host-service route. Depending on the platform, it may fail DNS
resolution or resolve to an internal gateway/bridge address where the host's
port `11434` is not forwarded. The sandbox then sees a DNS failure or
`connection refused`:

```console
$ getent hosts host.docker.internal
172.17.0.1 host.docker.internal host.openshell.internal
$ no_proxy=host.docker.internal curl -v http://host.docker.internal:11434/api/tags
* connect to 172.17.0.1 port 11434 failed: Connection refused
```

For local Ollama, use the auth-proxy URL that NemoClaw's "Local Ollama" onboard
option configures automatically:

```text
http://host.openshell.internal:11435/v1
```

`host.openshell.internal` resolves to the same gateway IP, and the
[token-gated Ollama auth proxy](#ollama-auth-proxy-did-not-start) binds port
`11435` there and forwards requests to `127.0.0.1:11434` on the host.
If you need a different host service exposed to the sandbox, route it through
the OpenShell gateway rather than relying on `host.docker.internal`.
See issue [#3136](https://github.com/NVIDIA/NemoClaw/issues/3136).

### Local inference health check resolves to IPv6

Local inference health checks now use `127.0.0.1` instead of `localhost`.
Expand Down
105 changes: 105 additions & 0 deletions src/lib/inference/onboard-host-docker-internal.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it } from "vitest";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";

const {
isHijackedDockerInternalUrl,
} = require("../../../dist/lib/inference/onboard-host-docker-internal");
const { isSandboxInternalUrl, probeOpenAiLikeEndpoint } = require("../../../dist/lib/inference/onboard-probes");

describe("host.docker.internal onboarding inference policy", () => {
it("does not treat host.docker.internal as a usable sandbox URL", () => {
expect(isSandboxInternalUrl("http://host.docker.internal:11434/v1")).toBe(false);
expect(isHijackedDockerInternalUrl("http://host.docker.internal:11434/v1")).toBe(true);
expect(isHijackedDockerInternalUrl("http://host.openshell.internal:11435/v1")).toBe(false);
expect(isHijackedDockerInternalUrl("https://api.openai.com/v1")).toBe(false);
});

it("rejects host.docker.internal URLs with an actionable proxy hint (#3136)", () => {
const result = probeOpenAiLikeEndpoint(
"http://host.docker.internal:11434/v1",
"openai/nemotron-mini",
"",
);
expect(result.ok).toBe(false);
expect(result.message).toMatch(/host\.docker\.internal/);
expect(result.message).toMatch(/host\.openshell\.internal:11435/);
expect(result.failures).toEqual([
expect.objectContaining({ name: "host.docker.internal reachability" }),
]);
});

it("rejects host.docker.internal even when strict chat-completions tool calling is required", () => {
const result = probeOpenAiLikeEndpoint(
"http://host.docker.internal:11434/v1",
"openai/nemotron-mini",
"",
{ skipResponsesProbe: true, requireChatCompletionsToolCalling: true },
);

expect(result).toMatchObject({ ok: false });
expect(result.message).toMatch(/host\.docker\.internal/);
expect(result.message).toMatch(/host\.openshell\.internal:11435/);
});

it("allows explicit Windows-host Ollama validation to probe host.docker.internal", () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-host-docker-probe-"));
const fakeBin = path.join(tmpDir, "bin");
const seenUrl = path.join(tmpDir, "url");
fs.mkdirSync(fakeBin, { recursive: true });
fs.writeFileSync(
path.join(fakeBin, "curl"),
`#!/usr/bin/env bash
outfile=""
url=""
while [ "$#" -gt 0 ]; do
case "$1" in
-o) outfile="$2"; shift 2 ;;
-w) shift 2 ;;
*) url="$1"; shift ;;
esac
done
printf '%s' "$url" > "${seenUrl}"
if [ -n "$outfile" ]; then
cat <<'JSON' > "$outfile"
{"choices":[{"message":{"tool_calls":[{"id":"call_1","type":"function","function":{"name":"sessions_send","arguments":"{\\"message\\":\\"hello\\"}"}}]}}]}
JSON
fi
printf '200'
exit 0
`,
{ mode: 0o755 },
);

const originalPath = process.env.PATH;
process.env.PATH = `${fakeBin}:${originalPath || ""}`;
try {
const result = probeOpenAiLikeEndpoint(
"http://host.docker.internal:11434/v1",
"openai/nemotron-mini",
"",
{
skipResponsesProbe: true,
requireChatCompletionsToolCalling: true,
allowHostDockerInternal: true,
},
);

expect(result).toMatchObject({
ok: true,
api: "openai-completions",
label: "Chat Completions API",
});
expect(fs.readFileSync(seenUrl, "utf8")).toBe(
"http://host.docker.internal:11434/v1/chat/completions",
);
} finally {
process.env.PATH = originalPath;
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
});
45 changes: 45 additions & 0 deletions src/lib/inference/onboard-host-docker-internal.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
// @ts-nocheck
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

const HOST_DOCKER_INTERNAL = "host.docker.internal";
const OLLAMA_PROXY_URL = "http://host.openshell.internal:11435/v1";

function isHijackedDockerInternalUrl(url) {
try {
return new URL(String(url)).hostname === HOST_DOCKER_INTERNAL;
} catch {
return false;
}
}

function getHostDockerInternalProbeFailure() {
return {
ok: false,
message:
`${HOST_DOCKER_INTERNAL} does not reach the host machine from inside the sandbox: ` +
`OpenShell k3s sandboxes do not provide it as a reliable host-service route. ` +
`It may fail DNS resolution or resolve to a gateway/bridge address where port ` +
`11434 is not forwarded. For local Ollama, use the auth-proxy URL ` +
`${OLLAMA_PROXY_URL} (the URL NemoClaw onboard configures automatically ` +
`when you pick "Local Ollama"). See issue #3136.`,
failures: [
{
name: "host.docker.internal reachability",
httpStatus: 0,
curlStatus: 0,
message:
`${HOST_DOCKER_INTERNAL} is not a reliable host-service route from ` +
`OpenShell k3s sandboxes and cannot be used as an inference base URL.`,
body: "",
},
],
};
}

module.exports = {
HOST_DOCKER_INTERNAL,
OLLAMA_PROXY_URL,
isHijackedDockerInternalUrl,
getHostDockerInternalProbeFailure,
};
66 changes: 1 addition & 65 deletions src/lib/inference/onboard-probes.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -349,9 +349,8 @@ describe("OpenAI-compatible inference probes", () => {
});

describe("sandbox-internal URL handling", () => {
it("identifies host.openshell.internal and host.docker.internal as sandbox-internal", () => {
it("identifies host.openshell.internal as sandbox-internal", () => {
expect(isSandboxInternalUrl("http://host.openshell.internal:8001/v1")).toBe(true);
expect(isSandboxInternalUrl("http://host.docker.internal:11434/v1")).toBe(true);
});

it("does not treat normal hostnames as sandbox-internal", () => {
Expand All @@ -374,69 +373,6 @@ describe("OpenAI-compatible inference probes", () => {
expect(result.note).toMatch(/only resolves inside the sandbox/);
});

it("skips the curl probe for host.docker.internal and returns ok with a note", () => {
const result = probeOpenAiLikeEndpoint(
"http://host.docker.internal:11434/v1",
"openai/nemotron-mini",
"",
);
expect(result).toMatchObject({ ok: true, api: null });
expect(result.note).toMatch(/host\.docker\.internal/);
});

it("probes host.docker.internal when strict chat-completions tool calling is required", () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-host-docker-probe-"));
const fakeBin = path.join(tmpDir, "bin");
const seenUrl = path.join(tmpDir, "url");
fs.mkdirSync(fakeBin, { recursive: true });
fs.writeFileSync(
path.join(fakeBin, "curl"),
`#!/usr/bin/env bash
outfile=""
url=""
while [ "$#" -gt 0 ]; do
case "$1" in
-o) outfile="$2"; shift 2 ;;
-w) shift 2 ;;
*) url="$1"; shift ;;
esac
done
printf '%s' "$url" > "${seenUrl}"
if [ -n "$outfile" ]; then
cat <<'JSON' > "$outfile"
{"choices":[{"message":{"tool_calls":[{"id":"call_1","type":"function","function":{"name":"sessions_send","arguments":"{\\"message\\":\\"hello\\"}"}}]}}]}
JSON
fi
printf '200'
exit 0
`,
{ mode: 0o755 },
);

const originalPath = process.env.PATH;
process.env.PATH = `${fakeBin}:${originalPath || ""}`;
try {
const result = probeOpenAiLikeEndpoint(
"http://host.docker.internal:11434/v1",
"openai/nemotron-mini",
"",
{ skipResponsesProbe: true, requireChatCompletionsToolCalling: true },
);

expect(result).toMatchObject({
ok: true,
api: "openai-completions",
label: "Chat Completions API",
});
expect(fs.readFileSync(seenUrl, "utf8")).toBe(
"http://host.docker.internal:11434/v1/chat/completions",
);
} finally {
process.env.PATH = originalPath;
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});

it("fails closed for unprobeable sandbox-internal URLs when strict tool calling is required", () => {
const result = probeOpenAiLikeEndpoint(
"http://host.openshell.internal:8001/v1",
Expand Down
39 changes: 23 additions & 16 deletions src/lib/inference/onboard-probes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
const { getCredential, normalizeCredentialValue, resolveProviderCredential } = require("../credentials/store");
const { isWsl } = require("../platform");
const httpProbe = require("../adapters/http/probe");
const {
getHostDockerInternalProbeFailure,
isHijackedDockerInternalUrl,
} = require("./onboard-host-docker-internal");
const {
isNvcfFunctionNotFoundForAccount,
nvcfFunctionNotFoundMessage,
Expand All @@ -28,7 +32,7 @@ const {
// so host-side validation cannot prove reachability for that URL. For ordinary
// verification we still skip these endpoints, but strict tool-call validation
// must fail closed unless the host is probeable from the onboard process.
const SANDBOX_INTERNAL_HOSTS = ["host.openshell.internal", "host.docker.internal"];
const SANDBOX_INTERNAL_HOSTS = ["host.openshell.internal"];

function isSandboxInternalUrl(url) {
try {
Expand Down Expand Up @@ -484,6 +488,10 @@ function runChatCompletionsProbe({ authHeader, model, url, isWsl: isWslOverride
}

function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) {
if (isHijackedDockerInternalUrl(endpointUrl) && options.allowHostDockerInternal !== true) {
return getHostDockerInternalProbeFailure();
}

if (isSandboxInternalUrl(endpointUrl)) {
const { hostname } = new URL(String(endpointUrl));
if (options.requireChatCompletionsToolCalling !== true) {
Expand All @@ -494,21 +502,19 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) {
note: `${hostname} only resolves inside the sandbox — validation skipped. If the endpoint is unreachable at runtime, re-run onboard with a routable URL.`,
};
}
if (hostname !== "host.docker.internal") {
return {
ok: false,
message: `${hostname} only resolves inside the sandbox and cannot be validated for required structured Chat Completions tool calls from the host. Use a routable endpoint URL and retry onboard.`,
failures: [
{
name: "Chat Completions API with tool calling",
httpStatus: 0,
curlStatus: 0,
message: "sandbox-internal endpoint cannot be strictly validated from host",
body: "",
},
],
};
}
return {
ok: false,
message: `${hostname} only resolves inside the sandbox and cannot be validated for required structured Chat Completions tool calls from the host. Use a routable endpoint URL and retry onboard.`,
failures: [
{
name: "Chat Completions API with tool calling",
httpStatus: 0,
curlStatus: 0,
message: "sandbox-internal endpoint cannot be strictly validated from host",
body: "",
},
],
};
}

const useQueryParam = options.authMode === "query-param";
Expand Down Expand Up @@ -787,6 +793,7 @@ function probeAnthropicEndpoint(endpointUrl, model, apiKey) {

module.exports = {
isSandboxInternalUrl,
isHijackedDockerInternalUrl,
parseJsonObject,
hasResponsesToolCall,
hasChatCompletionsToolCall,
Expand Down
6 changes: 3 additions & 3 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2120,9 +2120,6 @@ function verifyWebSearchInsideSandbox(
});
}

// getSandboxInferenceConfig — moved to onboard-providers.ts

// Inference probes — moved to inference/onboard-probes.ts
const {
hasResponsesToolCall,
hasChatCompletionsToolCall,
Expand All @@ -2148,6 +2145,7 @@ async function validateOpenAiLikeSelection(
requireChatCompletionsToolCalling?: boolean;
skipResponsesProbe?: boolean;
probeStreaming?: boolean;
allowHostDockerInternal?: boolean;
} = {},
): Promise<EndpointValidationResult> {
const apiKey = credentialEnv ? getCredential(credentialEnv) : "";
Expand Down Expand Up @@ -6025,6 +6023,8 @@ async function selectAndValidateOllamaModel(
{
skipResponsesProbe: true,
requireChatCompletionsToolCalling: true,
allowHostDockerInternal:
localInference.getResolvedOllamaHost() === OLLAMA_HOST_DOCKER_INTERNAL,
},
);
if (validation.retry === "selection") return { outcome: "back-to-selection" };
Expand Down
Loading