Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions docs/inference/use-local-inference.md
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,45 @@ $ NEMOCLAW_EXPERIMENTAL=1 \
NemoClaw records the model returned by vLLM's `/v1/models` endpoint.
Start vLLM with the model you want before onboarding if you manage the server yourself.

### DGX Spark Host Example

On a DGX Spark host, run vLLM on the Docker bridge and, if desired, the LAN
address used by other machines on the same subnet.

```console
$ docker run -d --name nemoclaw-vllm --restart unless-stopped \
--gpus all \
-p 172.17.0.1:8000:8000 \
-p 172.16.253.231:8000:8000 \
vllm/vllm-openai:latest \
nvidia/Gemma-4-26B-A4B-NVFP4 \
--served-model-name gemma-4-26b-a4b-nvfp4 \
--host 0.0.0.0 \
--port 8000 \
--max-model-len 32768 \
--max-num-seqs 4 \
--max-num-batched-tokens 4096 \
--gpu-memory-utilization 0.88 \
--tool-call-parser gemma4 \
--reasoning-parser gemma4 \
--enable-auto-tool-choice \
--trust-remote-code
```

Then onboard with the local vLLM provider.

```console
$ NEMOCLAW_EXPERIMENTAL=1 \
NEMOCLAW_PROVIDER=vllm \
NEMOCLAW_MODEL=gemma-4-26b-a4b-nvfp4 \
nemoclaw onboard --non-interactive --yes
```

The sandbox reaches the host endpoint through `host.openshell.internal`.
Keep `host.openshell.internal` out of `HTTP_PROXY` by leaving
`NEMOCLAW_NO_PROXY_EXTRA` unset, or set it to a comma-separated replacement
list if your host aliases differ.

## NVIDIA NIM (Experimental)

NemoClaw can pull, start, and manage a NIM container on hosts with a NIM-capable NVIDIA GPU.
Expand Down
9 changes: 9 additions & 0 deletions scripts/nemoclaw-start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -901,11 +901,20 @@ PYAUTOPAIR
#
# NEMOCLAW_PROXY_HOST / NEMOCLAW_PROXY_PORT can be overridden at sandbox
# creation time if the gateway IP or port changes in a future OpenShell release.
# NEMOCLAW_NO_PROXY_EXTRA appends additional direct-connect hosts for local
# inference runtimes such as vLLM/Ollama on the Docker host. Set it to an empty
# string to disable the default host aliases.
# Ref: https://github.com/NVIDIA/NemoClaw/issues/626
PROXY_HOST="${NEMOCLAW_PROXY_HOST:-10.200.0.1}"
PROXY_PORT="${NEMOCLAW_PROXY_PORT:-3128}"
NO_PROXY_EXTRA="${NEMOCLAW_NO_PROXY_EXTRA-host.openshell.internal,host.docker.internal}"
NO_PROXY_EXTRA="${NO_PROXY_EXTRA#,}"
NO_PROXY_EXTRA="${NO_PROXY_EXTRA%,}"
_PROXY_URL="http://${PROXY_HOST}:${PROXY_PORT}"
_NO_PROXY_VAL="localhost,127.0.0.1,::1,${PROXY_HOST}"
if [ -n "$NO_PROXY_EXTRA" ]; then
_NO_PROXY_VAL="${_NO_PROXY_VAL},${NO_PROXY_EXTRA}"
fi
export HTTP_PROXY="$_PROXY_URL"
export HTTPS_PROXY="$_PROXY_URL"
export NO_PROXY="$_NO_PROXY_VAL"
Expand Down
6 changes: 5 additions & 1 deletion src/lib/inference-config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
DEFAULT_OLLAMA_MODEL,
DEFAULT_ROUTE_CREDENTIAL_ENV,
DEFAULT_ROUTE_PROFILE,
DEFAULT_VLLM_MODEL,
INFERENCE_ROUTE_URL,
MANAGED_PROVIDER_ID,
OLLAMA_LOCAL_CREDENTIAL_ENV,
Expand Down Expand Up @@ -166,7 +167,7 @@ describe("inference selection config", () => {
expect(getProviderSelectionConfig("compatible-anthropic-endpoint")?.model).toBe(
"custom-anthropic-model",
);
expect(getProviderSelectionConfig("vllm-local")?.model).toBe("vllm-local");
expect(getProviderSelectionConfig("vllm-local")?.model).toBe(DEFAULT_VLLM_MODEL);
});

it("builds a qualified OpenClaw primary model for ollama-local", () => {
Expand All @@ -182,6 +183,9 @@ describe("inference selection config", () => {
expect(getOpenClawPrimaryModel("ollama-local")).toBe(
`${MANAGED_PROVIDER_ID}/${DEFAULT_OLLAMA_MODEL}`,
);
expect(getOpenClawPrimaryModel("vllm-local")).toBe(
`${MANAGED_PROVIDER_ID}/${DEFAULT_VLLM_MODEL}`,
);
});
});

Expand Down
15 changes: 12 additions & 3 deletions src/lib/inference-config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { DEFAULT_OLLAMA_MODEL } from "./local-inference";

export const INFERENCE_ROUTE_URL = "https://inference.local/v1";
export const DEFAULT_CLOUD_MODEL = "nvidia/nemotron-3-super-120b-a12b";
export const DEFAULT_VLLM_MODEL = "vllm-local";
export const CLOUD_MODEL_OPTIONS = [
{ id: "nvidia/nemotron-3-super-120b-a12b", label: "Nemotron 3 Super 120B" },
{ id: "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", label: "Nemotron 3 Nano Omni 30B" },
Expand Down Expand Up @@ -104,7 +105,7 @@ export function getProviderSelectionConfig(
case "vllm-local":
return {
...base,
model: model || "vllm-local",
model: model || DEFAULT_VLLM_MODEL,
credentialEnv: VLLM_LOCAL_CREDENTIAL_ENV,
providerLabel: "Local vLLM",
};
Expand All @@ -121,8 +122,16 @@ export function getProviderSelectionConfig(
}

export function getOpenClawPrimaryModel(provider: string, model?: string): string {
const resolvedModel =
model || (provider === "ollama-local" ? DEFAULT_OLLAMA_MODEL : DEFAULT_CLOUD_MODEL);
let resolvedModel = model;
if (!resolvedModel) {
if (provider === "ollama-local") {
resolvedModel = DEFAULT_OLLAMA_MODEL;
} else if (provider === "vllm-local") {
resolvedModel = DEFAULT_VLLM_MODEL;
} else {
resolvedModel = DEFAULT_CLOUD_MODEL;
}
}
return `${MANAGED_PROVIDER_ID}/${resolvedModel}`;
}

Expand Down
12 changes: 11 additions & 1 deletion test/service-env.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -469,15 +469,25 @@ describe("service environment", () => {
expect(vars.HTTPS_PROXY).toBe("http://10.200.0.1:8080");
});

it("NO_PROXY includes loopback only, not inference.local", () => {
it("NO_PROXY includes loopback and local runtime host aliases, not inference.local", () => {
const vars = extractProxyVars();
const noProxy = vars.NO_PROXY.split(",");
expect(noProxy).toContain("localhost");
expect(noProxy).toContain("127.0.0.1");
expect(noProxy).toContain("::1");
expect(noProxy).toContain("host.openshell.internal");
expect(noProxy).toContain("host.docker.internal");
expect(noProxy).not.toContain("inference.local");
});

it("NEMOCLAW_NO_PROXY_EXTRA can override local runtime host aliases", () => {
const vars = extractProxyVars({ NEMOCLAW_NO_PROXY_EXTRA: "example.internal" });
const noProxy = vars.NO_PROXY.split(",");
expect(noProxy).toContain("example.internal");
expect(noProxy).not.toContain("host.openshell.internal");
expect(noProxy).not.toContain("host.docker.internal");
});

it("NO_PROXY includes OpenShell gateway IP", () => {
const vars = extractProxyVars();
expect(vars.NO_PROXY).toContain("10.200.0.1");
Expand Down