From cc2625d6eebf4c67c16ecdb34e4874a24b61451b Mon Sep 17 00:00:00 2001 From: David Byte Date: Tue, 5 May 2026 12:42:13 -0500 Subject: [PATCH] fix: support DGX Spark vLLM routing --- docs/inference/use-local-inference.md | 39 +++++++++++++++++++++++++++ scripts/nemoclaw-start.sh | 9 +++++++ src/lib/inference-config.test.ts | 6 ++++- src/lib/inference-config.ts | 15 ++++++++--- test/service-env.test.ts | 12 ++++++++- 5 files changed, 76 insertions(+), 5 deletions(-) diff --git a/docs/inference/use-local-inference.md b/docs/inference/use-local-inference.md index 2a643393ad8..bde0b279ce5 100644 --- a/docs/inference/use-local-inference.md +++ b/docs/inference/use-local-inference.md @@ -252,6 +252,45 @@ $ NEMOCLAW_EXPERIMENTAL=1 \ NemoClaw auto-detects the model from the running vLLM instance. To override the model, set `NEMOCLAW_MODEL`. +### DGX Spark Host Example + +On a DGX Spark host, run vLLM on the Docker bridge and, if desired, the LAN +address used by other machines on the same subnet. + +```console +$ docker run -d --name nemoclaw-vllm --restart unless-stopped \ + --gpus all \ + -p 172.17.0.1:8000:8000 \ + -p 172.16.253.231:8000:8000 \ + vllm/vllm-openai:latest \ + nvidia/Gemma-4-26B-A4B-NVFP4 \ + --served-model-name gemma-4-26b-a4b-nvfp4 \ + --host 0.0.0.0 \ + --port 8000 \ + --max-model-len 32768 \ + --max-num-seqs 4 \ + --max-num-batched-tokens 4096 \ + --gpu-memory-utilization 0.88 \ + --tool-call-parser gemma4 \ + --reasoning-parser gemma4 \ + --enable-auto-tool-choice \ + --trust-remote-code +``` + +Then onboard with the local vLLM provider. + +```console +$ NEMOCLAW_EXPERIMENTAL=1 \ + NEMOCLAW_PROVIDER=vllm \ + NEMOCLAW_MODEL=gemma-4-26b-a4b-nvfp4 \ + nemoclaw onboard --non-interactive --yes +``` + +The sandbox reaches the host endpoint through `host.openshell.internal`. +Keep `host.openshell.internal` out of `HTTP_PROXY` by leaving +`NEMOCLAW_NO_PROXY_EXTRA` unset, or set it to a comma-separated replacement +list if your host aliases differ. + ## NVIDIA NIM (Experimental) NemoClaw can pull, start, and manage a NIM container on hosts with a NIM-capable NVIDIA GPU. diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index f96669a88fb..12391fdc77c 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -1328,11 +1328,20 @@ PYAUTOPAIR # # NEMOCLAW_PROXY_HOST / NEMOCLAW_PROXY_PORT can be overridden at sandbox # creation time if the gateway IP or port changes in a future OpenShell release. +# NEMOCLAW_NO_PROXY_EXTRA appends additional direct-connect hosts for local +# inference runtimes such as vLLM/Ollama on the Docker host. Set it to an empty +# string to disable the default host aliases. # Ref: https://github.com/NVIDIA/NemoClaw/issues/626 PROXY_HOST="${NEMOCLAW_PROXY_HOST:-10.200.0.1}" PROXY_PORT="${NEMOCLAW_PROXY_PORT:-3128}" +NO_PROXY_EXTRA="${NEMOCLAW_NO_PROXY_EXTRA-host.openshell.internal,host.docker.internal}" +NO_PROXY_EXTRA="${NO_PROXY_EXTRA#,}" +NO_PROXY_EXTRA="${NO_PROXY_EXTRA%,}" _PROXY_URL="http://${PROXY_HOST}:${PROXY_PORT}" _NO_PROXY_VAL="localhost,127.0.0.1,::1,${PROXY_HOST}" +if [ -n "$NO_PROXY_EXTRA" ]; then + _NO_PROXY_VAL="${_NO_PROXY_VAL},${NO_PROXY_EXTRA}" +fi export HTTP_PROXY="$_PROXY_URL" export HTTPS_PROXY="$_PROXY_URL" export NO_PROXY="$_NO_PROXY_VAL" diff --git a/src/lib/inference-config.test.ts b/src/lib/inference-config.test.ts index cc2f04992b3..3acb8987daa 100644 --- a/src/lib/inference-config.test.ts +++ b/src/lib/inference-config.test.ts @@ -9,6 +9,7 @@ import { DEFAULT_OLLAMA_MODEL, DEFAULT_ROUTE_CREDENTIAL_ENV, DEFAULT_ROUTE_PROFILE, + DEFAULT_VLLM_MODEL, INFERENCE_ROUTE_URL, MANAGED_PROVIDER_ID, OLLAMA_LOCAL_CREDENTIAL_ENV, @@ -165,7 +166,7 @@ describe("inference selection config", () => { expect(getProviderSelectionConfig("compatible-anthropic-endpoint")?.model).toBe( "custom-anthropic-model", ); - expect(getProviderSelectionConfig("vllm-local")?.model).toBe("vllm-local"); + expect(getProviderSelectionConfig("vllm-local")?.model).toBe(DEFAULT_VLLM_MODEL); }); it("builds a qualified OpenClaw primary model for ollama-local", () => { @@ -181,6 +182,9 @@ describe("inference selection config", () => { expect(getOpenClawPrimaryModel("ollama-local")).toBe( `${MANAGED_PROVIDER_ID}/${DEFAULT_OLLAMA_MODEL}`, ); + expect(getOpenClawPrimaryModel("vllm-local")).toBe( + `${MANAGED_PROVIDER_ID}/${DEFAULT_VLLM_MODEL}`, + ); }); }); diff --git a/src/lib/inference-config.ts b/src/lib/inference-config.ts index c99ee33e1ca..2ded2013710 100644 --- a/src/lib/inference-config.ts +++ b/src/lib/inference-config.ts @@ -10,6 +10,7 @@ import { DEFAULT_OLLAMA_MODEL } from "./local-inference"; export const INFERENCE_ROUTE_URL = "https://inference.local/v1"; export const DEFAULT_CLOUD_MODEL = "nvidia/nemotron-3-super-120b-a12b"; +export const DEFAULT_VLLM_MODEL = "vllm-local"; export const CLOUD_MODEL_OPTIONS = [ { id: "nvidia/nemotron-3-super-120b-a12b", label: "Nemotron 3 Super 120B" }, { id: "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", label: "Nemotron 3 Nano Omni 30B" }, @@ -103,7 +104,7 @@ export function getProviderSelectionConfig( case "vllm-local": return { ...base, - model: model || "vllm-local", + model: model || DEFAULT_VLLM_MODEL, credentialEnv: VLLM_LOCAL_CREDENTIAL_ENV, providerLabel: "Local vLLM", }; @@ -120,8 +121,16 @@ export function getProviderSelectionConfig( } export function getOpenClawPrimaryModel(provider: string, model?: string): string { - const resolvedModel = - model || (provider === "ollama-local" ? DEFAULT_OLLAMA_MODEL : DEFAULT_CLOUD_MODEL); + let resolvedModel = model; + if (!resolvedModel) { + if (provider === "ollama-local") { + resolvedModel = DEFAULT_OLLAMA_MODEL; + } else if (provider === "vllm-local") { + resolvedModel = DEFAULT_VLLM_MODEL; + } else { + resolvedModel = DEFAULT_CLOUD_MODEL; + } + } return `${MANAGED_PROVIDER_ID}/${resolvedModel}`; } diff --git a/test/service-env.test.ts b/test/service-env.test.ts index d0b043ba48d..ec88b8507f6 100644 --- a/test/service-env.test.ts +++ b/test/service-env.test.ts @@ -469,15 +469,25 @@ describe("service environment", () => { expect(vars.HTTPS_PROXY).toBe("http://10.200.0.1:8080"); }); - it("NO_PROXY includes loopback only, not inference.local", () => { + it("NO_PROXY includes loopback and local runtime host aliases, not inference.local", () => { const vars = extractProxyVars(); const noProxy = vars.NO_PROXY.split(","); expect(noProxy).toContain("localhost"); expect(noProxy).toContain("127.0.0.1"); expect(noProxy).toContain("::1"); + expect(noProxy).toContain("host.openshell.internal"); + expect(noProxy).toContain("host.docker.internal"); expect(noProxy).not.toContain("inference.local"); }); + it("NEMOCLAW_NO_PROXY_EXTRA can override local runtime host aliases", () => { + const vars = extractProxyVars({ NEMOCLAW_NO_PROXY_EXTRA: "example.internal" }); + const noProxy = vars.NO_PROXY.split(","); + expect(noProxy).toContain("example.internal"); + expect(noProxy).not.toContain("host.openshell.internal"); + expect(noProxy).not.toContain("host.docker.internal"); + }); + it("NO_PROXY includes OpenShell gateway IP", () => { const vars = extractProxyVars(); expect(vars.NO_PROXY).toContain("10.200.0.1");