From 006b74c419626e2b83e3a2a811cb73065acf9e95 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 14 Jul 2026 11:54:00 -0700 Subject: [PATCH 01/21] feat(installer): add DGX Station express install Signed-off-by: Aaron Erickson --- ci/platform-matrix.json | 6 +- docs/get-started/quickstart.mdx | 9 ++- docs/inference/choose-inference-provider.mdx | 2 +- docs/inference/set-up-vllm.mdx | 24 +++++-- docs/reference/commands.mdx | 2 +- docs/reference/platform-support.mdx | 4 +- scripts/install.sh | 10 ++- src/lib/inference/vllm-models.test.ts | 32 +++++++++ src/lib/inference/vllm-models.ts | 76 +++++++++++++++++++- src/lib/inference/vllm.test.ts | 60 ++++++++++++++++ src/lib/inference/vllm.ts | 56 ++++++++++----- test/inference-options-docs.test.ts | 7 +- test/install-express-prompt.test.ts | 15 ++++ 13 files changed, 268 insertions(+), 35 deletions(-) diff --git a/ci/platform-matrix.json b/ci/platform-matrix.json index f903e2ffc8f..cb25204f80c 100644 --- a/ci/platform-matrix.json +++ b/ci/platform-matrix.json @@ -1,7 +1,7 @@ { "$comment": "Single source of truth for NemoClaw launch claims and platform support. Covers platforms, inference providers, supported agents, messaging integrations, and deployment paths. Scripts read this to generate README and docs tables. QA/CI update platform/provider rows; the engineering owner reviews other rows. Docs are derived.", "version": "1.1", - "updated": "2026-07-12", + "updated": "2026-07-14", "project_status": { "stage": "alpha", @@ -64,7 +64,7 @@ "status": "deferred", "prd_priority": "P1", "ci_tested": false, - "notes": "The PRD marks this platform as P1. Workstation form-factor with NVIDIA GPUs and the same Docker + NVIDIA Container Toolkit + CDI requirements as DGX Spark. Onboard path not yet validated end-to-end on the hardware; vLLM defaults to `deepseek-ai/DeepSeek-V4-Flash` for this host class and will move out of `deferred` once the hardware run is signed off." + "notes": "The PRD marks this platform as P1. Workstation form-factor with NVIDIA GPUs and the same Docker + NVIDIA Container Toolkit + CDI requirements as DGX Spark. The installer detects DGX Station and offers express install with the pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` recipe, including an approximately 352 GB model download, without follow-up provider, model, policy, or sandbox-name choices. Direct managed-vLLM onboarding still defaults to `deepseek-ai/DeepSeek-V4-Flash` when no model override is set. The full NemoClaw onboarding path, including the express recipe, has not been validated end-to-end on physical DGX Station hardware and remains `deferred` until that run is signed off." }, { "name": "NVIDIA RTX (consumer and Pro workstation GPUs)", @@ -147,7 +147,7 @@ "name": "Local vLLM (managed install/start)", "status": "caveated", "endpoint_type": "Local OpenAI-compatible", - "notes": "Appears by default on DGX Spark and DGX Station. DGX Station remains deferred until its managed-vLLM onboarding path is validated end-to-end on physical hardware. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pins each host profile to an immutable NGC platform digest. DGX Spark and DGX Station use the `linux/arm64` digest published under `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use the matching `linux/arm64` or `linux/amd64` digest published under `nvcr.io/nvidia/vllm:26.03.post1-py3`. Validated defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing." + "notes": "Appears by default on DGX Spark and DGX Station. DGX Station remains deferred until its managed-vLLM onboarding path is validated end-to-end on physical hardware. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pins runtime images to immutable digests. DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest published under `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use the matching `linux/arm64` or `linux/amd64` digest published under `nvcr.io/nvidia/vllm:26.03.post1-py3`. The DGX Station express installer selects `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` with a pinned Hugging Face revision and the multi-platform index digest published under `vllm/vllm-openai:v0.22.0`; that index resolves to the `linux/arm64` manifest on Station. Direct managed-vLLM profile defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls from `nvcr.io` require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing." } ], diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 14cf91c3667..cb95b99323a 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -125,6 +125,7 @@ Use these details when your first-run path needs more control. On macOS, start Docker Desktop or Colima first. DGX Spark, DGX Station, and Windows WSL offer an interactive express-install path that chooses a managed local inference option for the platform. + On DGX Station, accepting the express prompt selects the pinned `nemotron-3-ultra-550b-a55b` managed-vLLM recipe and completes onboarding without more provider, model, policy, or sandbox-name choices. Refer to [Platform Support](../reference/platform-support) and [Choose an Inference Provider](../inference/learn-and-choose/choose-inference-provider) for the current platform behavior. If `nemoclaw` is not found after installation and you use nvm or fnm, open a new terminal or reload your shell profile. @@ -191,12 +192,18 @@ Use these details when your first-run path needs more control. On DGX Spark, DGX Station, and Windows WSL, interactive installation offers express install after you accept the third-party software notice. Express install switches onboarding to non-interactive mode, allows `sudo` password prompts for required host changes, and selects the managed local inference path for that platform. - DGX Spark uses managed vLLM with `qwen3.6-35b-a3b-nvfp4` by default, and DGX Station uses `deepseek-v4-flash` by default. + DGX Spark uses managed vLLM with `qwen3.6-35b-a3b-nvfp4` by default. + DGX Station express install explicitly selects `nemotron-3-ultra-550b-a55b` instead of the Station managed-vLLM profile default, `deepseek-v4-flash`, and discloses the approximately `352 GB` model download before confirmation. Unless `NEMOCLAW_POLICY_TIER` is set, express install applies policy in `suggested` mode with the `balanced` tier, including the base sandbox policy and supported package, model, web-search, and local-inference presets. Express install uses `my-assistant` as the sandbox name across all platforms unless `NEMOCLAW_SANDBOX_NAME` is set. Windows WSL selects the Windows-host Ollama setup path. Set `NEMOCLAW_NO_EXPRESS=1` to skip the express prompt, or set `NEMOCLAW_PROVIDER` before launching the installer to choose a provider yourself. + + Express install automates the configuration but does not change DGX Station's Deferred support status. + Full NemoClaw onboarding with this recipe has not completed end-to-end validation on physical DGX Station hardware. + + The installer auto-launches `nemoclaw onboard` when it can find the new binary. If it cannot find the binary or blocking host preflight checks fail, it prints diagnostics and a `To finish setup, run:` block with the explicit `nemoclaw onboard` command. diff --git a/docs/inference/choose-inference-provider.mdx b/docs/inference/choose-inference-provider.mdx index a7521fc8b2e..a87cc050137 100644 --- a/docs/inference/choose-inference-provider.mdx +++ b/docs/inference/choose-inference-provider.mdx @@ -32,7 +32,7 @@ Use this status table to distinguish validated provider integrations from adapte | Local Ollama | Tested with limitations | Local Ollama API | Available when Ollama is installed or running on the host. Validated default models: `qwen3.6:35b` (high VRAM), `nemotron-3-nano:30b` (medium VRAM), `qwen3.5:9b` (low VRAM fallback). | | Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard/fatal-runtime-preflight.ts`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. On Linux arm64 DGX Spark and DGX Station hosts, onboarding warns that some NIM images may not publish a `linux/arm64` manifest; the warning is advisory, and the selected image pull can still fail when the registry has no matching platform manifest. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`. | | Local vLLM (already running) | Tested with limitations | Local OpenAI-compatible | Appears in the onboarding menu when NemoClaw detects a server already on `localhost:8000`. No flag required. Model is whatever the existing server serves. | -| Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and DGX Station. DGX Station remains deferred until its managed-vLLM onboarding path is validated end-to-end on physical hardware. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pins each host profile to an immutable NGC platform digest. DGX Spark and DGX Station use the `linux/arm64` digest published under `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use the matching `linux/arm64` or `linux/amd64` digest published under `nvcr.io/nvidia/vllm:26.03.post1-py3`. Validated defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing. | +| Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and DGX Station. DGX Station remains deferred until its managed-vLLM onboarding path is validated end-to-end on physical hardware. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pins runtime images to immutable digests. DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest published under `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use the matching `linux/arm64` or `linux/amd64` digest published under `nvcr.io/nvidia/vllm:26.03.post1-py3`. The DGX Station express installer selects `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` with a pinned Hugging Face revision and the multi-platform index digest published under `vllm/vllm-openai:v0.22.0`; that index resolves to the `linux/arm64` manifest on Station. Direct managed-vLLM profile defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls from `nvcr.io` require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing. | {/* provider-status:end */} ## Hosted Providers diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 8f0357c445f..4679f2f7a93 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -79,16 +79,18 @@ NemoClaw pulls the vLLM image, downloads model weights into `~/.cache/huggingfac The container carries a NemoClaw ownership label so later onboarding runs can distinguish it from a bring-your-own server without trusting the container name alone. If an existing `nemoclaw-vllm` container does not carry that label, NemoClaw preserves it and asks you to remove or rename it before installing managed vLLM. If another vLLM server already occupies port `8000`, the managed path stops and asks you to select the existing server or stop it first. -Managed profiles use immutable NGC platform digests: +Managed profiles and model-specific recipes use immutable image digests: -- DGX Spark and DGX Station use the `linux/arm64` digest `sha256:9204569b17ee4c0eff75194b8e6e458479c8aee18953b5ab9cf359fcdac659e2` with a compressed layer size of `9.60 GB` under `nvcr.io/nvidia/vllm:26.05.post1-py3`. - DGX Station remains deferred until this path is validated end-to-end on physical hardware. +- DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest `sha256:9204569b17ee4c0eff75194b8e6e458479c8aee18953b5ab9cf359fcdac659e2` with a compressed layer size of `9.60 GB` under `nvcr.io/nvidia/vllm:26.05.post1-py3`. +- The DGX Station Nemotron 3 Ultra express recipe uses the multi-platform index digest `sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416` under `vllm/vllm-openai:v0.22.0`; on DGX Station, that index selects a `linux/arm64` manifest with a compressed layer size of `10.67 GB`. + It also pins Hugging Face revision `183968f87ae4cedce3039313cac1fd43d112c578` for the approximately `352.38 GB` model download. - Generic Linux `arm64` hosts use `sha256:447995cbb57e6c7cf792cab95e9852e5f62b5fb6d2f39e030fa4eda9a54eadb4` with a compressed layer size of `9.28 GB` under `nvcr.io/nvidia/vllm:26.03.post1-py3`. - Generic Linux `amd64` hosts use `sha256:7be6c2f676c36059a494fe17254e69ae5c677535ba6191044e5fc8e42a91c773` with a compressed layer size of `8.93 GB` under `nvcr.io/nvidia/vllm:26.03.post1-py3`. Image pulls from `nvcr.io` require NGC registry login. +The Station Nemotron 3 Ultra recipe pulls its vLLM image from Docker Hub instead. -The first run can take 10 to 30 minutes. +The first run duration depends on image and model size, storage performance, and network throughput. Later runs reuse the cached image and model weights. The managed container uses Docker's `--restart unless-stopped` policy, so Docker restarts it unless an operator explicitly stops it. @@ -131,7 +133,7 @@ Before the first managed-vLLM run, verify that the filesystem backing your model If image pull output stops making progress, a watchdog stops the stalled pull without imposing a fixed wall-clock limit on slow but active downloads. If vLLM does not become ready, NemoClaw prints a short tail of the container logs before exiting. -Managed vLLM uses these profile defaults: +When you start managed vLLM outside the installer express flow, NemoClaw uses these profile defaults: | Host profile | Default model | |---|---| @@ -139,6 +141,14 @@ Managed vLLM uses these profile defaults: | DGX Station | `deepseek-ai/DeepSeek-V4-Flash` | | Linux with an NVIDIA GPU | `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8` | +On DGX Station, accepting the installer express prompt sets `NEMOCLAW_VLLM_MODEL=nemotron-3-ultra-550b-a55b` and overrides the profile default. +The registered recipe configures the pinned model revision, CPU offload, host networking, `16 GB` of shared memory, and the Nemotron reasoning and tool-call parsers. +Because the service uses host networking and listens on port `8000`, apply the firewall guidance at the top of this page. + + +DGX Station and its managed-vLLM paths remain Deferred until NemoClaw completes end-to-end validation on physical DGX Station hardware. + + ## Run Non-Interactive Onboarding Use an already-running server. @@ -158,6 +168,7 @@ NEMOCLAW_PROVIDER=install-vllm \ On DGX Spark and DGX Station, `NEMOCLAW_PROVIDER=install-vllm` is sufficient for a non-interactive run. Add `NEMOCLAW_EXPERIMENTAL=1` on a generic Linux NVIDIA GPU host. Non-interactive runs use the profile default unless you set `NEMOCLAW_VLLM_MODEL`. +On DGX Station, a direct provider-only run therefore selects `deepseek-v4-flash`; the installer express flow sets the Nemotron 3 Ultra override for you. ## Select a Managed Model @@ -169,7 +180,8 @@ NemoClaw applies the registered `vllm serve` arguments, including the reasoning | `qwen3.6-27b` | `Qwen/Qwen3.6-27B-FP8` | Supported override. | | `qwen3.6-35b-a3b-nvfp4` | `nvidia/Qwen3.6-35B-A3B-NVFP4` | DGX Spark default. | | `nemotron-3-nano-4b` | `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8` | Generic Linux NVIDIA GPU default. | -| `deepseek-v4-flash` | `deepseek-ai/DeepSeek-V4-Flash` | DGX Station default. | +| `deepseek-v4-flash` | `deepseek-ai/DeepSeek-V4-Flash` | DGX Station profile default outside express install. | +| `nemotron-3-ultra-550b-a55b` | `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` | DGX Station express-install selection with a pinned model revision and model-specific vLLM image. | | `deepseek-r1-distill-70b` | `deepseek-ai/DeepSeek-R1-Distill-Llama-70B` | Gated and requires license acceptance. | Slugs are case-insensitive, and NemoClaw also accepts the full Hugging Face model ID. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 12145b2dfcb..e8d7c60f57a 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3248,7 +3248,7 @@ Set them before running `$$nemoclaw onboard`. | `SANDBOX_NAME` | sandbox name | Compatibility spelling used after `NEMOCLAW_SANDBOX_NAME` and `NEMOCLAW_SANDBOX`. | | `NEMOCLAW_INSTALL_REF` | git ref | For internal installer commands: the git ref to install from. Overridden by the `--install-ref` flag. | | `NEMOCLAW_INSTALL_TAG` | release tag | For internal installer commands: the release tag to install. Defaults to the admin-promoted `lkg` tag when unset. Overridden by the `--install-tag` flag. | -| `NEMOCLAW_VLLM_MODEL` | registry slug or Hugging Face model id | Selects the model the managed-vLLM install path serves. Recognised slugs: `qwen3.6-27b`, `qwen3.6-35b-a3b-nvfp4`, `nemotron-3-nano-4b`, `deepseek-v4-flash`, `deepseek-r1-distill-70b`. Unset uses the per-platform profile default. Gated models (e.g. `deepseek-r1-distill-70b`) require `HF_TOKEN` or `HUGGING_FACE_HUB_TOKEN`. | +| `NEMOCLAW_VLLM_MODEL` | registry slug or Hugging Face model id | Selects the model the managed-vLLM install path serves. Recognised slugs: `qwen3.6-27b`, `qwen3.6-35b-a3b-nvfp4`, `nemotron-3-nano-4b`, `deepseek-v4-flash`, `nemotron-3-ultra-550b-a55b`, `deepseek-r1-distill-70b`. Unset uses the per-platform profile default. The DGX Station express installer sets `nemotron-3-ultra-550b-a55b` explicitly. Gated models (e.g. `deepseek-r1-distill-70b`) require `HF_TOKEN` or `HUGGING_FACE_HUB_TOKEN`. | | `NEMOCLAW_VLLM_EXTRA_ARGS_JSON` | JSON array of non-blank strings | Appends advanced operator-owned tokens to the managed `vllm serve` command after NemoClaw's registry defaults. Example: `["--max-num-seqs","2"]`. Malformed JSON, non-string tokens, or blank tokens fail before Docker work starts. | | `NEMOCLAW_IGNORE_VLLM_DISK_SPACE` | exactly `1` | Allows managed vLLM to continue when verified Docker image-storage capacity is insufficient; other values do not bypass the check. Inconclusive image-storage checks report diagnostics and continue automatically. The check does not cover Hugging Face model-cache capacity. | diff --git a/docs/reference/platform-support.mdx b/docs/reference/platform-support.mdx index 039e1c92b05..ae9f07ffb12 100644 --- a/docs/reference/platform-support.mdx +++ b/docs/reference/platform-support.mdx @@ -82,7 +82,7 @@ For the onboarding-time supported set without deferred rows, refer to [Prerequis | macOS (Apple Silicon) | Colima, Docker Desktop | Tested with limitations | P0 | Yes | Start the container runtime (Colima or Docker Desktop) before running the installer. Homebrew Colima users must install both Colima and the Docker CLI (`brew install colima docker`) before `docker info` can work. Xcode Command Line Tools (`xcode-select --install`) are typically required for Node native modules during install. NemoClaw recommends them but does not enforce them during preflight. | | DGX Spark | Docker | Tested | P1 | Yes | Use the standard installer and `$$nemoclaw onboard`. For an end-to-end walkthrough with local inference, see the [NVIDIA Spark playbook](https://build.nvidia.com/spark/nemoclaw). | | Windows WSL2 | Docker Desktop (WSL backend) | Tested with limitations | P1 | No | Requires WSL2 with Docker Desktop backend. | -| DGX Station | Docker | Deferred | P1 | No | The PRD marks this platform as P1. Workstation form-factor with NVIDIA GPUs and the same Docker + NVIDIA Container Toolkit + CDI requirements as DGX Spark. Onboard path not yet validated end-to-end on the hardware; vLLM defaults to `deepseek-ai/DeepSeek-V4-Flash` for this host class and will move out of `deferred` once the hardware run is signed off. | +| DGX Station | Docker | Deferred | P1 | No | The PRD marks this platform as P1. Workstation form-factor with NVIDIA GPUs and the same Docker + NVIDIA Container Toolkit + CDI requirements as DGX Spark. The installer detects DGX Station and offers express install with the pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` recipe, including an approximately 352 GB model download, without follow-up provider, model, policy, or sandbox-name choices. Direct managed-vLLM onboarding still defaults to `deepseek-ai/DeepSeek-V4-Flash` when no model override is set. The full NemoClaw onboarding path, including the express recipe, has not been validated end-to-end on physical DGX Station hardware and remains `deferred` until that run is signed off. | | NVIDIA RTX (consumer and Pro workstation GPUs) | Docker | Deferred | P1 | No | The PRD marks this platform as P1. Covers RTX consumer cards and RTX Pro workstation cards on Linux hosts that meet the generic-Linux-GPU requirements (NVIDIA Container Toolkit + CDI present). The provider menu emits managed vLLM behind `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm` for this host class today; the end-to-end onboard path on this hardware is not yet validated in CI. | {/* platform-matrix-full:end */} @@ -104,7 +104,7 @@ NemoClaw routes inference through the OpenShell gateway. Each row below is a pro | Local Ollama | Tested with limitations | Local Ollama API | Available when Ollama is installed or running on the host. Validated default models: `qwen3.6:35b` (high VRAM), `nemotron-3-nano:30b` (medium VRAM), `qwen3.5:9b` (low VRAM fallback). | | Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard/fatal-runtime-preflight.ts`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. On Linux arm64 DGX Spark and DGX Station hosts, onboarding warns that some NIM images may not publish a `linux/arm64` manifest; the warning is advisory, and the selected image pull can still fail when the registry has no matching platform manifest. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`. | | Local vLLM (already running) | Tested with limitations | Local OpenAI-compatible | Appears in the onboarding menu when NemoClaw detects a server already on `localhost:8000`. No flag required. Model is whatever the existing server serves. | -| Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and DGX Station. DGX Station remains deferred until its managed-vLLM onboarding path is validated end-to-end on physical hardware. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pins each host profile to an immutable NGC platform digest. DGX Spark and DGX Station use the `linux/arm64` digest published under `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use the matching `linux/arm64` or `linux/amd64` digest published under `nvcr.io/nvidia/vllm:26.03.post1-py3`. Validated defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing. | +| Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and DGX Station. DGX Station remains deferred until its managed-vLLM onboarding path is validated end-to-end on physical hardware. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pins runtime images to immutable digests. DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest published under `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use the matching `linux/arm64` or `linux/amd64` digest published under `nvcr.io/nvidia/vllm:26.03.post1-py3`. The DGX Station express installer selects `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` with a pinned Hugging Face revision and the multi-platform index digest published under `vllm/vllm-openai:v0.22.0`; that index resolves to the `linux/arm64` manifest on Station. Direct managed-vLLM profile defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls from `nvcr.io` require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing. | {/* provider-status-full:end */} ## Messaging Integrations diff --git a/scripts/install.sh b/scripts/install.sh index f487398830e..f769c1ac063 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2683,8 +2683,12 @@ describe_express_install() { sandbox_summary="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" ;; "DGX Station") - inference_summary="managed local vLLM" - inference_disclosure="Managed vLLM pulls the configured vLLM image/model and runs a local vLLM inference container." + if [ -n "${NEMOCLAW_VLLM_MODEL:-}" ]; then + inference_summary="managed local vLLM with model ${NEMOCLAW_VLLM_MODEL}" + else + inference_summary="managed local vLLM with NVIDIA Nemotron 3 Ultra 550B" + fi + inference_disclosure="Managed vLLM pulls the pinned Station image and approximately 352 GB model, then runs a local inference container." sandbox_summary="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" ;; "Windows WSL") @@ -2786,7 +2790,9 @@ maybe_offer_express_install() { fi ;; "DGX Station") + export NEMOCLAW_SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" export NEMOCLAW_PROVIDER=install-vllm + export NEMOCLAW_VLLM_MODEL="${NEMOCLAW_VLLM_MODEL:-nemotron-3-ultra-550b-a55b}" ;; "Windows WSL") export NEMOCLAW_PROVIDER=install-windows-ollama diff --git a/src/lib/inference/vllm-models.test.ts b/src/lib/inference/vllm-models.test.ts index bbe843206e2..49aff591699 100644 --- a/src/lib/inference/vllm-models.test.ts +++ b/src/lib/inference/vllm-models.test.ts @@ -57,6 +57,37 @@ describe("vllm model registry", () => { ).toEqual(deepseek); }); + it("pins the DGX Station Nemotron Ultra serving recipe", () => { + const ultra = VLLM_MODELS.find((m) => m.envValue === "nemotron-3-ultra-550b-a55b"); + expect(ultra).toBeDefined(); + expect(ultra!.id).toBe("nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4"); + expect(ultra!.revision).toBe("183968f87ae4cedce3039313cac1fd43d112c578"); + expect(ultra!.servedModelId).toBe("nvidia/nemotron-3-ultra-550b-a55b"); + expect(ultra!.runtime).toEqual({ + image: + "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416", + imageDownloadSizeBytes: 10_670_087_425, + dockerRunArgs: ["--network", "host", "--shm-size", "16g"], + publishPort: false, + }); + + const cmd = buildVllmServeCommand(ultra!); + expect(cmd).toContain("vllm serve nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4"); + expect(cmd).toContain("--revision 183968f87ae4cedce3039313cac1fd43d112c578"); + expect(cmd).toContain("--served-model-name nvidia/nemotron-3-ultra-550b-a55b"); + expect(cmd).toContain("--cpu-offload-gb 150 --cpu-offload-params experts"); + expect(cmd).toContain(`--kernel_config '{"enable_flashinfer_autotune": false}'`); + expect(cmd).toContain("--max-num-seqs 256 --gpu-memory-utilization 0.9"); + expect(cmd).toContain("--reasoning-parser nemotron_v3"); + expect(cmd).toContain("--enable-auto-tool-choice --tool-call-parser qwen3_coder"); + expect(cmd).toContain( + `--default-chat-template-kwargs '{"enable_thinking":true,"force_nonempty_content":true}'`, + ); + expect(cmd).toMatch( + /^export VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY=1 && export VLLM_NVFP4_GEMM_BACKEND=flashinfer-trtllm && /, + ); + }); + it("rejects an unknown NEMOCLAW_VLLM_MODEL with a helpful message", () => { expect(() => selectVllmModelFromEnv({ NEMOCLAW_VLLM_MODEL: "made-up-model" } as NodeJS.ProcessEnv), @@ -267,6 +298,7 @@ describe("modelsForPlatform", () => { expect(slugs).toContain("nemotron-3-nano-4b"); expect(slugs).toContain("deepseek-r1-distill-70b"); expect(slugs).toContain("deepseek-v4-flash"); + expect(slugs).toContain("nemotron-3-ultra-550b-a55b"); expect(slugs).not.toContain("qwen3.6-35b-a3b-nvfp4"); }); diff --git a/src/lib/inference/vllm-models.ts b/src/lib/inference/vllm-models.ts index 7359fc9bfd0..48193909fe9 100644 --- a/src/lib/inference/vllm-models.ts +++ b/src/lib/inference/vllm-models.ts @@ -5,8 +5,9 @@ * Registry of models the express vLLM install path knows how to serve. * * Each entry pins the model-specific `vllm serve` flags (reasoning parser, - * tool-call parser, max model length, load format) so the express path can - * swap models without leaving the wrong flags behind. + * tool-call parser, max model length, load format) and any required runtime + * image/container overrides so the express path can swap models without + * leaving the wrong recipe behind. * * Selection precedence in `installVllm`: * 1. `NEMOCLAW_VLLM_MODEL=` for automation overrides. @@ -28,6 +29,25 @@ export type VllmPlatform = "spark" | "station" | "linux"; +export interface VllmRuntimeOverride { + /** Model-specific runtime image, pinned by digest. */ + image: string; + /** Compressed size of the selected platform manifest. */ + imageDownloadSizeBytes: number; + /** Additional `docker run` arguments required by this recipe. */ + dockerRunArgs?: readonly string[]; + /** Whether the managed container should publish localhost:8000. */ + publishPort?: boolean; +} + +export const NEMOTRON_ULTRA_STATION_IMAGE = { + tag: "vllm/vllm-openai:v0.22.0", + arm64: { + ref: "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416", + downloadSizeBytes: 10_670_087_425, + }, +} as const; + export interface VllmModelDef { /** Hugging Face model id (also passed to `vllm serve`). */ id: string; @@ -37,6 +57,10 @@ export interface VllmModelDef { envValue: string; /** `--max-model-len` flag value. */ maxModelLen: number; + /** Immutable Hugging Face revision used for download and serving. */ + revision?: string; + /** Stable model name exposed by the local OpenAI-compatible endpoint. */ + servedModelId?: string; /** Model-specific flags appended after the shared serving flags. */ modelArgs: string[]; /** True when the upstream HF repo requires accepting a licence. */ @@ -56,6 +80,8 @@ export interface VllmModelDef { * container shell. */ serveEnv?: Record; + /** Runtime overrides for recipes that cannot use the platform image. */ + runtime?: VllmRuntimeOverride; } export const VLLM_MODELS: readonly VllmModelDef[] = [ @@ -169,6 +195,50 @@ export const VLLM_MODELS: readonly VllmModelDef[] = [ gated: false, platforms: ["station"], }, + { + id: "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", + label: "NVIDIA Nemotron 3 Ultra 550B NVFP4", + envValue: "nemotron-3-ultra-550b-a55b", + downloadSizeBytes: 352_381_245_521, + maxModelLen: 262144, + revision: "183968f87ae4cedce3039313cac1fd43d112c578", + // Keep the route identity aligned with NemoClaw's existing managed + // Nemotron Ultra compatibility manifests and request adapters. + servedModelId: "nvidia/nemotron-3-ultra-550b-a55b", + modelArgs: [ + "--host", + "0.0.0.0", + "--cpu-offload-gb", + "150", + "--cpu-offload-params", + "experts", + "--kernel_config", + `'{"enable_flashinfer_autotune": false}'`, + "--max-num-seqs", + "256", + "--gpu-memory-utilization", + "0.9", + "--reasoning-parser", + "nemotron_v3", + "--enable-auto-tool-choice", + "--tool-call-parser", + "qwen3_coder", + "--default-chat-template-kwargs", + `'{"enable_thinking":true,"force_nonempty_content":true}'`, + ], + gated: false, + platforms: ["station"], + serveEnv: { + VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: "1", + VLLM_NVFP4_GEMM_BACKEND: "flashinfer-trtllm", + }, + runtime: { + image: NEMOTRON_ULTRA_STATION_IMAGE.arm64.ref, + imageDownloadSizeBytes: NEMOTRON_ULTRA_STATION_IMAGE.arm64.downloadSizeBytes, + dockerRunArgs: ["--network", "host", "--shm-size", "16g"], + publishPort: false, + }, + }, { id: "nvidia/Qwen3.6-35B-A3B-NVFP4", label: "Qwen3.6 35B-A3B NVFP4", @@ -397,6 +467,8 @@ export function buildVllmServeCommand( ...SHARED_VLLM_ARGS, "--max-model-len", String(model.maxModelLen), + ...(model.revision ? ["--revision", model.revision] : []), + ...(model.servedModelId ? ["--served-model-name", model.servedModelId] : []), ...model.modelArgs, ]; const extraArgs = parseVllmExtraServeArgs(env).map(shellQuote); diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index a31b3bee5ac..b4cc572b3ab 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -53,8 +53,10 @@ import { NEMOCLAW_VLLM_CONTAINER_NAME, NEMOCLAW_VLLM_MANAGED_LABEL, pullImage, + resolveVllmRuntimeProfile, resolveVllmServedModelId, } from "./vllm"; +import { VLLM_MODELS } from "./vllm-models"; beforeEach(() => { mocks.dockerImageInspectFormat.mockReturnValue(""); @@ -158,6 +160,27 @@ describe("vLLM profile detection", () => { expect(profile!.defaultModel.envValue).toBe("deepseek-v4-flash"); }); + it("resolves Nemotron Ultra to the pinned Station runtime without Docker port publishing", () => { + mocks.getGpuIndicesByName.mockReturnValue([0]); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + const ultra = VLLM_MODELS.find((model) => model.envValue === "nemotron-3-ultra-550b-a55b"); + + expect(profile).not.toBeNull(); + expect(ultra).toBeDefined(); + const runtime = resolveVllmRuntimeProfile(profile!, ultra!); + expect(runtime.image).toBe( + "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416", + ); + expect(runtime.imageDownloadSizeBytes).toBe(10_670_087_425); + expect(runtime.buildDockerRunFlags!()).toEqual( + expect.arrayContaining(["--gpus", "device=0", "--network", "host", "--shm-size", "16g"]), + ); + + const args = buildVllmRunArgs(runtime, ultra!, runtime.buildDockerRunFlags!()); + expect(args).not.toContain("-p"); + expect(args).toContain(runtime.image); + }); + it("keeps DGX Spark on the Qwen3.6 35B NVFP4 default", () => { const profile = detectVllmProfile({ platform: "spark", type: "nvidia" }); @@ -473,6 +496,43 @@ describe("installVllm model resolution", () => { expect(summary).toContain("Model: Qwen/Qwen3.6-27B-FP8 (NEMOCLAW_VLLM_MODEL override)"); }); + it("installs the complete Nemotron Ultra Station recipe without another selection", async () => { + process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + const beforeInstall = vi.fn(); + const promptFn = vi.fn<(q: string) => Promise>(); + mockSuccessfulVllmInstall(profile.containerName); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn, + beforeInstall, + }); + + expect(result).toEqual({ ok: true }); + expect(promptFn).not.toHaveBeenCalled(); + expect(beforeInstall).toHaveBeenCalledWith("nvidia/nemotron-3-ultra-550b-a55b"); + expect(mocks.dockerPullWithProgressWatchdog).toHaveBeenCalledWith( + "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416", + expect.any(Object), + ); + const [downloadArgs] = mocks.dockerSpawn.mock.calls[0] as [string[]]; + expect(downloadArgs).toEqual( + expect.arrayContaining([ + "download", + "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", + "--revision", + "183968f87ae4cedce3039313cac1fd43d112c578", + ]), + ); + const [runArgs] = mocks.dockerRunDetached.mock.calls[0] as [string[]]; + expect(runArgs).toEqual(expect.arrayContaining(["--network", "host", "--shm-size", "16g"])); + expect(runArgs).not.toContain("-p"); + expect(runArgs.at(-1)).toContain("--cpu-offload-gb 150"); + expect(runArgs.at(-1)).toContain("--reasoning-parser nemotron_v3"); + }); + it("offers the interactive picker when no env override is set", async () => { const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; const queue = ["", "n"]; diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 32a5e3023f7..000ac4ee044 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -25,6 +25,7 @@ import { getGpuIndicesByName } from "./nim"; import { buildVllmDockerEnv } from "./vllm-docker-env"; import { buildVllmServeCommand, + NEMOTRON_ULTRA_STATION_IMAGE, parseVllmExtraServeArgs, VLLM_EXTRA_ARGS_ENV, VLLM_MODELS, @@ -66,6 +67,9 @@ export interface VllmProfile { // dockerRunFlags at install time. Used by Station to pick the GB300 GPU // out of a mixed-GPU host instead of using `--gpus all`. buildDockerRunFlags?: () => string[]; + // Host-network recipes already bind the service port directly and must not + // also add Docker's `-p` publishing flag. + publishPort?: boolean; // Maximum wall-clock safety budget for image pulls. The Docker adapter uses // a shorter progress watchdog for stalls, so slow-but-moving pulls can keep // going until this last-ditch cap. @@ -78,6 +82,7 @@ export interface VllmProfile { // named release tags. Pinning the digest makes a cache hit authoritative: an // explicit pull cannot begin downloading different same-tag layers. export const VLLM_IMAGES = { + vllm022: NEMOTRON_ULTRA_STATION_IMAGE, ngc2603Post1: { tag: "nvcr.io/nvidia/vllm:26.03.post1-py3", amd64: { @@ -335,6 +340,7 @@ function downloadModel( profile.image, "download", model.id, + ...(model.revision ? ["--revision", model.revision] : []), ], { env: buildVllmDockerEnv(buildHfTokenForwardEnv()), @@ -445,8 +451,7 @@ export function buildVllmRunArgs( ...safeRunFlags, "--label", `${NEMOCLAW_VLLM_MANAGED_LABEL}=true`, - "-p", - `${String(VLLM_PORT)}:8000`, + ...(profile.publishPort === false ? [] : ["-p", `${String(VLLM_PORT)}:8000`]), "--name", containerName, "--entrypoint", @@ -457,6 +462,22 @@ export function buildVllmRunArgs( ]; } +export function resolveVllmRuntimeProfile(profile: VllmProfile, model: VllmModelDef): VllmProfile { + const runtime = model.runtime; + if (!runtime) return profile; + const extraRunArgs = [...(runtime.dockerRunArgs ?? [])]; + return { + ...profile, + image: runtime.image, + imageDownloadSizeBytes: runtime.imageDownloadSizeBytes, + dockerRunFlags: [...profile.dockerRunFlags, ...extraRunArgs], + buildDockerRunFlags: profile.buildDockerRunFlags + ? () => [...profile.buildDockerRunFlags!(), ...extraRunArgs] + : undefined, + publishPort: runtime.publishPort ?? profile.publishPort, + }; +} + type VllmContainerOwnership = | { kind: "absent" } | { kind: "foreign" } @@ -780,12 +801,13 @@ export async function installVllm( }); if (!resolved) return { ok: false }; const { model, source: modelSource } = resolved; + const runtimeProfile = resolveVllmRuntimeProfile(profile, model); let extraServeArgs: string[]; let servedModelId: string; try { extraServeArgs = parseVllmExtraServeArgs(); - servedModelId = resolveVllmServedModelId(model.id, extraServeArgs); + servedModelId = resolveVllmServedModelId(model.servedModelId ?? model.id, extraServeArgs); } catch (err) { console.error(` vLLM install failed: ${(err as Error).message}`); return { ok: false }; @@ -793,8 +815,8 @@ export async function installVllm( opts.beforeInstall?.(servedModelId); console.log(""); - console.log(` vLLM (${profile.name}):`); - console.log(` Image: ${profile.image}`); + console.log(` vLLM (${runtimeProfile.name}):`); + console.log(` Image: ${runtimeProfile.image}`); console.log( ` Model: ${model.id}${modelSource === "env" ? " (NEMOCLAW_VLLM_MODEL override)" : ""}`, ); @@ -823,42 +845,44 @@ export async function installVllm( // Fail before large downloads when the fixed name belongs to another // operator. startContainer repeats this check to close the teardown race. - const replacement = vllmContainerReplacementTarget(profile.containerName); + const replacement = vllmContainerReplacementTarget(runtimeProfile.containerName); if (!replacement.ok) { console.error(` vLLM install failed: ${replacement.reason}`); return { ok: false }; } - const hasImage = imageIsCached(profile); - if (!hasImage && !(await imageStorageAccepted(profile, opts))) { + const hasImage = imageIsCached(runtimeProfile); + if (!hasImage && !(await imageStorageAccepted(runtimeProfile, opts))) { return { ok: false }; } - const pull = await pullImage(profile); + const pull = await pullImage(runtimeProfile); if (!pull.ok) { console.error(` vLLM install failed: ${String(pull.reason)}`); return { ok: false }; } - const modelDownload = await downloadModel(profile, model); + const modelDownload = await downloadModel(runtimeProfile, model); if (!modelDownload.ok) { console.error(` vLLM install failed: ${String(modelDownload.reason)}`); return { ok: false }; } - const start = startContainer(profile, model); + const start = startContainer(runtimeProfile, model); if (!start.ok) { console.error(` vLLM install failed: ${String(start.reason)}`); return { ok: false }; } emit("Launching vLLM"); - emit(`Launch can take 5 minutes to ${String(Math.ceil(profile.loadTimeoutSec / 60))} minutes`); + emit( + `Launch can take 5 minutes to ${String(Math.ceil(runtimeProfile.loadTimeoutSec / 60))} minutes`, + ); - const ready = await waitForVllmReady(profile); + const ready = await waitForVllmReady(runtimeProfile); if (!ready.ok) { - printContainerLogTail(profile); - dockerStop(profile.containerName, { + printContainerLogTail(runtimeProfile); + dockerStop(runtimeProfile.containerName, { env: buildVllmDockerEnv(), ignoreError: true, suppressOutput: true, @@ -867,7 +891,7 @@ export async function installVllm( return { ok: false }; } - if (!containerStillRunning(profile)) { + if (!containerStillRunning(runtimeProfile)) { console.error(" vLLM container exited unexpectedly after readiness"); return { ok: false }; } diff --git a/test/inference-options-docs.test.ts b/test/inference-options-docs.test.ts index 3521a46df7d..3663edabc68 100644 --- a/test/inference-options-docs.test.ts +++ b/test/inference-options-docs.test.ts @@ -294,10 +294,15 @@ describe("inference setup navigation", () => { const markdown = fs.readFileSync(vllmSetupPath, "utf8"); const entries = [ { - prefix: "- DGX Spark and DGX Station", + prefix: "- DGX Spark and DGX Station models without a model-specific runtime", image: VLLM_IMAGES.ngc2605Post1.arm64, tag: VLLM_IMAGES.ngc2605Post1.tag, }, + { + prefix: "- The DGX Station Nemotron 3 Ultra express recipe", + image: VLLM_IMAGES.vllm022.arm64, + tag: VLLM_IMAGES.vllm022.tag, + }, { prefix: "- Generic Linux `arm64` hosts", image: VLLM_IMAGES.ngc2603Post1.arm64, diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index 57873166cec..bb8acce1a98 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -196,6 +196,21 @@ detect_express_platform ); }); + it("uses the Nemotron Ultra recipe without follow-up choices on DGX Station", () => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Station"); + const output = `${result.stdout}${result.stderr}`; + expect(result.status, output).toBe(0); + expect(output).toMatch(/Detected DGX Station/); + expect(output).toMatch( + /Express install will configure managed local vLLM with NVIDIA Nemotron 3 Ultra 550B/, + ); + expect(output).toMatch(/approximately 352 GB model/); + expect(output).toMatch(/Using express install for DGX Station/); + expect(output).toMatch( + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL=nemotron-3-ultra-550b-a55b POLICY=suggested YES=1 SANDBOX=my-assistant/, + ); + }); + it("detects Windows WSL as an express install platform", () => { const result = spawnSync( "bash", From dd31881018ff9c313903b01114839d6dea045691 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 14 Jul 2026 12:07:38 -0700 Subject: [PATCH 02/21] fix(inference): preserve vLLM bridge network Signed-off-by: Aaron Erickson --- docs/inference/set-up-vllm.mdx | 5 +++-- src/lib/inference/vllm-models.test.ts | 3 +-- src/lib/inference/vllm-models.ts | 7 +++---- src/lib/inference/vllm.test.ts | 11 ++++++----- src/lib/inference/vllm.ts | 7 ++----- 5 files changed, 15 insertions(+), 18 deletions(-) diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 4679f2f7a93..685798c0bdf 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -142,8 +142,9 @@ When you start managed vLLM outside the installer express flow, NemoClaw uses th | Linux with an NVIDIA GPU | `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8` | On DGX Station, accepting the installer express prompt sets `NEMOCLAW_VLLM_MODEL=nemotron-3-ultra-550b-a55b` and overrides the profile default. -The registered recipe configures the pinned model revision, CPU offload, host networking, `16 GB` of shared memory, and the Nemotron reasoning and tool-call parsers. -Because the service uses host networking and listens on port `8000`, apply the firewall guidance at the top of this page. +The registered recipe configures the pinned model revision, CPU offload, `16 GB` of shared memory, and the Nemotron reasoning and tool-call parsers. +NemoClaw intentionally keeps its existing bridge-networked managed-inference topology instead of importing the playbook's host-network setting. +The container publishes port `8000` through Docker, so apply the firewall guidance at the top of this page. DGX Station and its managed-vLLM paths remain Deferred until NemoClaw completes end-to-end validation on physical DGX Station hardware. diff --git a/src/lib/inference/vllm-models.test.ts b/src/lib/inference/vllm-models.test.ts index 49aff591699..433f1242365 100644 --- a/src/lib/inference/vllm-models.test.ts +++ b/src/lib/inference/vllm-models.test.ts @@ -67,8 +67,7 @@ describe("vllm model registry", () => { image: "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416", imageDownloadSizeBytes: 10_670_087_425, - dockerRunArgs: ["--network", "host", "--shm-size", "16g"], - publishPort: false, + dockerRunArgs: ["--shm-size", "16g"], }); const cmd = buildVllmServeCommand(ultra!); diff --git a/src/lib/inference/vllm-models.ts b/src/lib/inference/vllm-models.ts index 48193909fe9..6ecb2358f58 100644 --- a/src/lib/inference/vllm-models.ts +++ b/src/lib/inference/vllm-models.ts @@ -36,8 +36,6 @@ export interface VllmRuntimeOverride { imageDownloadSizeBytes: number; /** Additional `docker run` arguments required by this recipe. */ dockerRunArgs?: readonly string[]; - /** Whether the managed container should publish localhost:8000. */ - publishPort?: boolean; } export const NEMOTRON_ULTRA_STATION_IMAGE = { @@ -235,8 +233,9 @@ export const VLLM_MODELS: readonly VllmModelDef[] = [ runtime: { image: NEMOTRON_ULTRA_STATION_IMAGE.arm64.ref, imageDownloadSizeBytes: NEMOTRON_ULTRA_STATION_IMAGE.arm64.downloadSizeBytes, - dockerRunArgs: ["--network", "host", "--shm-size", "16g"], - publishPort: false, + // Keep NemoClaw's bridge-networked local-inference boundary instead of + // importing the playbook's host-network setting. + dockerRunArgs: ["--shm-size", "16g"], }, }, { diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index b4cc572b3ab..de17710d771 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -160,7 +160,7 @@ describe("vLLM profile detection", () => { expect(profile!.defaultModel.envValue).toBe("deepseek-v4-flash"); }); - it("resolves Nemotron Ultra to the pinned Station runtime without Docker port publishing", () => { + it("resolves Nemotron Ultra to the pinned Station runtime on the bridge network", () => { mocks.getGpuIndicesByName.mockReturnValue([0]); const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); const ultra = VLLM_MODELS.find((model) => model.envValue === "nemotron-3-ultra-550b-a55b"); @@ -173,11 +173,12 @@ describe("vLLM profile detection", () => { ); expect(runtime.imageDownloadSizeBytes).toBe(10_670_087_425); expect(runtime.buildDockerRunFlags!()).toEqual( - expect.arrayContaining(["--gpus", "device=0", "--network", "host", "--shm-size", "16g"]), + expect.arrayContaining(["--gpus", "device=0", "--shm-size", "16g"]), ); const args = buildVllmRunArgs(runtime, ultra!, runtime.buildDockerRunFlags!()); - expect(args).not.toContain("-p"); + expect(args).not.toContain("--network"); + expect(args).toEqual(expect.arrayContaining(["-p", "8000:8000"])); expect(args).toContain(runtime.image); }); @@ -527,8 +528,8 @@ describe("installVllm model resolution", () => { ]), ); const [runArgs] = mocks.dockerRunDetached.mock.calls[0] as [string[]]; - expect(runArgs).toEqual(expect.arrayContaining(["--network", "host", "--shm-size", "16g"])); - expect(runArgs).not.toContain("-p"); + expect(runArgs).toEqual(expect.arrayContaining(["--shm-size", "16g", "-p", "8000:8000"])); + expect(runArgs).not.toContain("--network"); expect(runArgs.at(-1)).toContain("--cpu-offload-gb 150"); expect(runArgs.at(-1)).toContain("--reasoning-parser nemotron_v3"); }); diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 000ac4ee044..ce70aeb26c2 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -67,9 +67,6 @@ export interface VllmProfile { // dockerRunFlags at install time. Used by Station to pick the GB300 GPU // out of a mixed-GPU host instead of using `--gpus all`. buildDockerRunFlags?: () => string[]; - // Host-network recipes already bind the service port directly and must not - // also add Docker's `-p` publishing flag. - publishPort?: boolean; // Maximum wall-clock safety budget for image pulls. The Docker adapter uses // a shorter progress watchdog for stalls, so slow-but-moving pulls can keep // going until this last-ditch cap. @@ -451,7 +448,8 @@ export function buildVllmRunArgs( ...safeRunFlags, "--label", `${NEMOCLAW_VLLM_MANAGED_LABEL}=true`, - ...(profile.publishPort === false ? [] : ["-p", `${String(VLLM_PORT)}:8000`]), + "-p", + `${String(VLLM_PORT)}:8000`, "--name", containerName, "--entrypoint", @@ -474,7 +472,6 @@ export function resolveVllmRuntimeProfile(profile: VllmProfile, model: VllmModel buildDockerRunFlags: profile.buildDockerRunFlags ? () => [...profile.buildDockerRunFlags!(), ...extraRunArgs] : undefined, - publishPort: runtime.publishPort ?? profile.publishPort, }; } From ffd3da437b92019262649f0db7b034429f224100 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 14 Jul 2026 12:12:43 -0700 Subject: [PATCH 03/21] test(onboard): cover Station express FSM handoff Signed-off-by: Aaron Erickson --- src/lib/onboard/setup-nim-flow.test.ts | 71 ++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/src/lib/onboard/setup-nim-flow.test.ts b/src/lib/onboard/setup-nim-flow.test.ts index 125d507d1ee..910a84107cb 100644 --- a/src/lib/onboard/setup-nim-flow.test.ts +++ b/src/lib/onboard/setup-nim-flow.test.ts @@ -696,6 +696,77 @@ describe("createSetupNim", () => { }); }); + it("threads the DGX Station express model through the standard managed-vLLM selection contract", async () => { + const profile = { name: "DGX Station", platform: "station" } as VllmProfile; + const servedModel = "nvidia/nemotron-3-ultra-550b-a55b"; + const prompt = vi.fn(async () => unexpected("provider prompt")); + const detectInferenceProviderHostState = vi.fn(() => + makeHostState({ + vllmProfile: profile, + hasVllmImage: false, + vllmEntries: [{ key: "install-vllm", label: "Start vLLM (DGX Station)" }], + }), + ); + const installVllm = vi.fn(async (_profile, options) => { + options.beforeInstall?.(servedModel); + return { ok: true }; + }); + const routeGuard = vi.fn(() => ({ + requiredModel: null, + requiredEndpointUrl: null, + requiredInferenceApi: null, + })); + const handleVllmSelection = vi.fn(async (state) => { + expect(state).toMatchObject({ + provider: "vllm-local", + model: servedModel, + endpointUrl: null, + credentialEnv: null, + preferredInferenceApi: "openai-completions", + }); + state.provider = "vllm"; + state.endpointUrl = "http://127.0.0.1:8000/v1"; + return "selected"; + }); + const setupNim = createSetupNim( + makeDeps({ + isNonInteractive: () => true, + getNonInteractiveProvider: () => "install-vllm", + prompt, + detectInferenceProviderHostState, + installVllm, + handleVllmSelection, + }), + ); + + const result = await setupNim(null, null, null, true, null, "nemoclaw", routeGuard); + + expect(prompt).not.toHaveBeenCalled(); + expect(installVllm).toHaveBeenCalledWith(profile, { + hasImage: false, + nonInteractive: true, + promptFn: prompt, + beforeInstall: expect.any(Function), + }); + expect(routeGuard).toHaveBeenCalledWith({ + provider: "vllm-local", + model: servedModel, + endpointUrl: null, + preferredInferenceApi: "openai-completions", + credentialEnv: null, + }); + expect(handleVllmSelection).toHaveBeenCalledWith( + expect.objectContaining({ model: servedModel }), + { managedInstall: true, sparkHost: false }, + ); + expect(result).toMatchObject({ + model: servedModel, + provider: "vllm", + endpointUrl: "http://127.0.0.1:8000/v1", + preferredInferenceApi: "openai-completions", + }); + }); + it("passes sparkHost:true when detected GPU is Spark (firmware-unknown GB10 threading)", async () => { const handleVllmSelection = vi.fn(async (state) => { state.provider = "vllm-local"; From 70e5bb8ecd26a621e65c686bc07df2c10bdcc755 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 14 Jul 2026 12:29:27 -0700 Subject: [PATCH 04/21] fix(installer): normalize Station model overrides Signed-off-by: Aaron Erickson --- scripts/install.sh | 10 +++++++--- test/install-express-prompt.test.ts | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index f769c1ac063..67ec7a01a38 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2683,12 +2683,13 @@ describe_express_install() { sandbox_summary="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" ;; "DGX Station") - if [ -n "${NEMOCLAW_VLLM_MODEL:-}" ]; then + if [ -n "$(printf "%s" "${NEMOCLAW_VLLM_MODEL:-}" | tr -d '[:space:]')" ]; then inference_summary="managed local vLLM with model ${NEMOCLAW_VLLM_MODEL}" + inference_disclosure="Managed vLLM pulls the configured vLLM image/model and runs a local inference container." else inference_summary="managed local vLLM with NVIDIA Nemotron 3 Ultra 550B" + inference_disclosure="Managed vLLM pulls the pinned Station image and approximately 352 GB model, then runs a local inference container." fi - inference_disclosure="Managed vLLM pulls the pinned Station image and approximately 352 GB model, then runs a local inference container." sandbox_summary="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" ;; "Windows WSL") @@ -2792,7 +2793,10 @@ maybe_offer_express_install() { "DGX Station") export NEMOCLAW_SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" export NEMOCLAW_PROVIDER=install-vllm - export NEMOCLAW_VLLM_MODEL="${NEMOCLAW_VLLM_MODEL:-nemotron-3-ultra-550b-a55b}" + if [ -z "$(printf "%s" "${NEMOCLAW_VLLM_MODEL:-}" | tr -d '[:space:]')" ]; then + NEMOCLAW_VLLM_MODEL="nemotron-3-ultra-550b-a55b" + fi + export NEMOCLAW_VLLM_MODEL ;; "Windows WSL") export NEMOCLAW_PROVIDER=install-windows-ollama diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index bb8acce1a98..716c1e1d7ee 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -211,6 +211,33 @@ detect_express_platform ); }); + it("describes and preserves an explicit DGX Station model override", () => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { + NEMOCLAW_VLLM_MODEL: "custom-station-model", + }); + const output = `${result.stdout}${result.stderr}`; + expect(result.status, output).toBe(0); + expect(output).toMatch(/managed local vLLM with model custom-station-model/); + expect(output).toMatch(/pulls the configured vLLM image\/model/); + expect(output).not.toMatch(/approximately 352 GB model/); + expect(output).toMatch( + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL=custom-station-model POLICY=suggested YES=1 SANDBOX=my-assistant/, + ); + }); + + it("treats a whitespace-only DGX Station model override as unset", () => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { + NEMOCLAW_VLLM_MODEL: " \t ", + }); + const output = `${result.stdout}${result.stderr}`; + expect(result.status, output).toBe(0); + expect(output).toMatch(/managed local vLLM with NVIDIA Nemotron 3 Ultra 550B/); + expect(output).toMatch(/approximately 352 GB model/); + expect(output).toMatch( + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL=nemotron-3-ultra-550b-a55b POLICY=suggested YES=1 SANDBOX=my-assistant/, + ); + }); + it("detects Windows WSL as an express install platform", () => { const result = spawnSync( "bash", From f9f5416564cdb7b3f6c8c9854751d7825ae5d558 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 14 Jul 2026 12:35:45 -0700 Subject: [PATCH 05/21] fix(inference): align Ultra with storage contract Signed-off-by: Aaron Erickson --- src/lib/inference/vllm-models.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/inference/vllm-models.ts b/src/lib/inference/vllm-models.ts index 6ecb2358f58..c1abc1e42bb 100644 --- a/src/lib/inference/vllm-models.ts +++ b/src/lib/inference/vllm-models.ts @@ -197,7 +197,6 @@ export const VLLM_MODELS: readonly VllmModelDef[] = [ id: "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", label: "NVIDIA Nemotron 3 Ultra 550B NVFP4", envValue: "nemotron-3-ultra-550b-a55b", - downloadSizeBytes: 352_381_245_521, maxModelLen: 262144, revision: "183968f87ae4cedce3039313cac1fd43d112c578", // Keep the route identity aligned with NemoClaw's existing managed From 6e0b01b1b395768e5e1789c237d5a79fd78c0cd5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 14 Jul 2026 13:05:28 -0700 Subject: [PATCH 06/21] fix(inference): guard Station runtime boundary Signed-off-by: Aaron Erickson --- src/lib/inference/vllm-models.test.ts | 1 + src/lib/inference/vllm-models.ts | 26 +++++++++++++++++--------- src/lib/inference/vllm.test.ts | 23 +++++++++++++++++++++++ src/lib/inference/vllm.ts | 4 ++++ 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/src/lib/inference/vllm-models.test.ts b/src/lib/inference/vllm-models.test.ts index 433f1242365..d6ed302335f 100644 --- a/src/lib/inference/vllm-models.test.ts +++ b/src/lib/inference/vllm-models.test.ts @@ -72,6 +72,7 @@ describe("vllm model registry", () => { const cmd = buildVllmServeCommand(ultra!); expect(cmd).toContain("vllm serve nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4"); + expect(cmd).not.toContain("pip install"); expect(cmd).toContain("--revision 183968f87ae4cedce3039313cac1fd43d112c578"); expect(cmd).toContain("--served-model-name nvidia/nemotron-3-ultra-550b-a55b"); expect(cmd).toContain("--cpu-offload-gb 150 --cpu-offload-params experts"); diff --git a/src/lib/inference/vllm-models.ts b/src/lib/inference/vllm-models.ts index c1abc1e42bb..085acead90f 100644 --- a/src/lib/inference/vllm-models.ts +++ b/src/lib/inference/vllm-models.ts @@ -67,8 +67,9 @@ export interface VllmModelDef { * Platforms whose interactive picker should offer this entry. Models with * platform-specific flags (the NVFP4 MoE checkpoint targets `sm_121a` only, * the very large V4 Flash recipe wants Station-class VRAM) appear only on - * profiles they can actually run on. Non-interactive callers and direct - * `NEMOCLAW_VLLM_MODEL` overrides bypass the filter. + * profiles they can actually run on. Direct `NEMOCLAW_VLLM_MODEL` + * overrides normally bypass the picker filter, but a model-specific runtime + * is rejected outside this list so an incompatible image is never pulled. */ platforms: readonly VllmPlatform[]; /** @@ -80,6 +81,8 @@ export interface VllmModelDef { serveEnv?: Record; /** Runtime overrides for recipes that cannot use the platform image. */ runtime?: VllmRuntimeOverride; + /** Whether startup must install vLLM's fastsafetensors extra. Defaults to true. */ + installFastSafetensors?: boolean; } export const VLLM_MODELS: readonly VllmModelDef[] = [ @@ -236,6 +239,10 @@ export const VLLM_MODELS: readonly VllmModelDef[] = [ // importing the playbook's host-network setting. dockerRunArgs: ["--shm-size", "16g"], }, + // The digest-pinned vLLM image already contains the serving package, and + // this recipe does not use the fastsafetensors load format. Avoid mutating + // the reviewed runtime from the package index when the container starts. + installFastSafetensors: false, }, { id: "nvidia/Qwen3.6-35B-A3B-NVFP4", @@ -448,9 +455,11 @@ function shellQuote(value: string): string { * Build the `vllm serve` command line for the supplied model: the shared * serving flags merged with the model-specific args from the registry. * - * The command is prefixed with the `pip install` that pulls the - * `fastsafetensors` extra so existing express scripts keep working; a model - * may prepend env exports via `serveEnv`. + * By default the command is prefixed with the `pip install` that pulls the + * `fastsafetensors` extra so existing express scripts keep working. A pinned + * runtime that already contains everything its recipe needs may disable that + * mutation with `installFastSafetensors: false`; a model may also prepend env + * exports via `serveEnv`. */ export function buildVllmServeCommand( model: VllmModelDef, @@ -470,8 +479,7 @@ export function buildVllmServeCommand( ...model.modelArgs, ]; const extraArgs = parseVllmExtraServeArgs(env).map(shellQuote); - return `${envPrefix}pip install vllm[fastsafetensors] && vllm serve ${model.id} ${[ - ...args, - ...extraArgs, - ].join(" ")}`; + const setup = + model.installFastSafetensors === false ? "" : "pip install vllm[fastsafetensors] && "; + return `${envPrefix}${setup}vllm serve ${model.id} ${[...args, ...extraArgs].join(" ")}`; } diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index de17710d771..a5685b1e206 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -497,6 +497,29 @@ describe("installVllm model resolution", () => { expect(summary).toContain("Model: Qwen/Qwen3.6-27B-FP8 (NEMOCLAW_VLLM_MODEL override)"); }); + it("rejects a Station-only runtime override before side effects on generic Linux", async () => { + process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + const profile = detectVllmProfile({ platform: "linux", type: "nvidia" })!; + const beforeInstall = vi.fn(); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + beforeInstall, + }); + + expect(result).toEqual({ ok: false }); + expect(beforeInstall).not.toHaveBeenCalled(); + expect(mocks.runCapture).not.toHaveBeenCalled(); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + const errors = errSpy.mock.calls.map((c: unknown[]) => String(c[0])).join("\n"); + expect(errors).toContain( + "NVIDIA Nemotron 3 Ultra 550B NVFP4 is not supported on Linux + NVIDIA GPU", + ); + }); + it("installs the complete Nemotron Ultra Station recipe without another selection", async () => { process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index ce70aeb26c2..1c8e46ccdf2 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -798,6 +798,10 @@ export async function installVllm( }); if (!resolved) return { ok: false }; const { model, source: modelSource } = resolved; + if (model.runtime && !model.platforms.includes(profile.platform)) { + console.error(` vLLM install failed: ${model.label} is not supported on ${profile.name}`); + return { ok: false }; + } const runtimeProfile = resolveVllmRuntimeProfile(profile, model); let extraServeArgs: string[]; From 35f9c105a4d71690f17011f45b2eb8a4dc837694 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 07:22:29 -0700 Subject: [PATCH 07/21] fix(installer): harden Station express recipes Signed-off-by: Aaron Erickson --- ci/platform-matrix.json | 4 +- docs/get-started/quickstart.mdx | 2 + docs/inference/set-up-vllm.mdx | 22 +- docs/reference/commands.mdx | 2 +- docs/reference/platform-support.mdx | 2 +- install.sh | 1 + scripts/install.sh | 76 +++++- src/lib/inference/vllm-models.test.ts | 7 +- src/lib/inference/vllm-models.ts | 10 +- src/lib/inference/vllm-storage.test.ts | 155 +++++++++++++ src/lib/inference/vllm-storage.ts | 185 ++++++++++++++- src/lib/inference/vllm.test.ts | 305 +++++++++++++++++++++++-- src/lib/inference/vllm.ts | 194 ++++++++++++++-- src/lib/onboard/setup-nim-vllm.test.ts | 14 ++ src/lib/onboard/setup-nim-vllm.ts | 8 + test/inference-options-docs.test.ts | 10 + test/install-express-prompt.test.ts | 90 +++++++- test/onboard-selection-vllm.test.ts | 78 +++++++ 18 files changed, 1106 insertions(+), 59 deletions(-) diff --git a/ci/platform-matrix.json b/ci/platform-matrix.json index cb25204f80c..71387c2530c 100644 --- a/ci/platform-matrix.json +++ b/ci/platform-matrix.json @@ -1,7 +1,7 @@ { "$comment": "Single source of truth for NemoClaw launch claims and platform support. Covers platforms, inference providers, supported agents, messaging integrations, and deployment paths. Scripts read this to generate README and docs tables. QA/CI update platform/provider rows; the engineering owner reviews other rows. Docs are derived.", "version": "1.1", - "updated": "2026-07-14", + "updated": "2026-07-15", "project_status": { "stage": "alpha", @@ -64,7 +64,7 @@ "status": "deferred", "prd_priority": "P1", "ci_tested": false, - "notes": "The PRD marks this platform as P1. Workstation form-factor with NVIDIA GPUs and the same Docker + NVIDIA Container Toolkit + CDI requirements as DGX Spark. The installer detects DGX Station and offers express install with the pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` recipe, including an approximately 352 GB model download, without follow-up provider, model, policy, or sandbox-name choices. Direct managed-vLLM onboarding still defaults to `deepseek-ai/DeepSeek-V4-Flash` when no model override is set. The full NemoClaw onboarding path, including the express recipe, has not been validated end-to-end on physical DGX Station hardware and remains `deferred` until that run is signed off." + "notes": "The PRD marks this platform as P1. Workstation form-factor with NVIDIA GPUs and the same Docker + NVIDIA Container Toolkit + CDI requirements as DGX Spark. The installer detects DGX Station and offers express install with the pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` recipe, including an approximately 352 GB model download, without follow-up provider, model, policy, or sandbox-name choices. Pass `--station-deepseek` to use `deepseek-ai/DeepSeek-V4-Flash` for a Station demo while retaining the one-confirmation express flow. Direct managed-vLLM onboarding still defaults to `deepseek-ai/DeepSeek-V4-Flash` when no model override is set. The full NemoClaw onboarding path, including the express recipe, has not been validated end-to-end on physical DGX Station hardware and remains `deferred` until that run is signed off." }, { "name": "NVIDIA RTX (consumer and Pro workstation GPUs)", diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index cb95b99323a..59980dee75e 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -126,6 +126,7 @@ Use these details when your first-run path needs more control. DGX Spark, DGX Station, and Windows WSL offer an interactive express-install path that chooses a managed local inference option for the platform. On DGX Station, accepting the express prompt selects the pinned `nemotron-3-ultra-550b-a55b` managed-vLLM recipe and completes onboarding without more provider, model, policy, or sandbox-name choices. + Pass `--station-deepseek` to use DeepSeek V4 Flash for a Station demo instead. Refer to [Platform Support](../reference/platform-support) and [Choose an Inference Provider](../inference/learn-and-choose/choose-inference-provider) for the current platform behavior. If `nemoclaw` is not found after installation and you use nvm or fnm, open a new terminal or reload your shell profile. @@ -194,6 +195,7 @@ Use these details when your first-run path needs more control. Express install switches onboarding to non-interactive mode, allows `sudo` password prompts for required host changes, and selects the managed local inference path for that platform. DGX Spark uses managed vLLM with `qwen3.6-35b-a3b-nvfp4` by default. DGX Station express install explicitly selects `nemotron-3-ultra-550b-a55b` instead of the Station managed-vLLM profile default, `deepseek-v4-flash`, and discloses the approximately `352 GB` model download before confirmation. + To select DeepSeek V4 Flash while retaining the one-confirmation Station express flow, run `curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-deepseek`. Unless `NEMOCLAW_POLICY_TIER` is set, express install applies policy in `suggested` mode with the `balanced` tier, including the base sandbox policy and supported package, model, web-search, and local-inference presets. Express install uses `my-assistant` as the sandbox name across all platforms unless `NEMOCLAW_SANDBOX_NAME` is set. Windows WSL selects the Windows-host Ollama setup path. diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 685798c0bdf..0e6cd29bde3 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -110,24 +110,26 @@ The model-download helper and the long-lived vLLM container use `--pull=never`, NemoClaw treats `/var/run/docker.sock` and `/run/docker.sock` as the local default Docker socket paths. It cannot inspect daemon storage through a remote endpoint, a named non-default Docker context, or another Unix socket. When both `DOCKER_CONTEXT` and `DOCKER_HOST` are set, NemoClaw follows Docker CLI precedence and uses `DOCKER_CONTEXT`. -When the check is inconclusive, NemoClaw prints the reason, approximate image requirement, and diagnostic commands, then continues with the pull without claiming that capacity is sufficient. +Before a registered model with pinned size metadata is downloaded, NemoClaw also checks the filesystem that contains the host Hugging Face cache. +When either check is inconclusive, NemoClaw prints the reason, approximate requirement, and diagnostic commands, then continues without claiming that capacity is sufficient. When verified capacity is insufficient, the warning identifies the image, available capacity, approximate requirement, checked path and source, remediation, and diagnostic commands. Interactive setup continues only when you enter `y` or `yes` explicitly. -Pressing **Enter**, entering `n` or `no`, or entering any other response stops before the guarded image pull starts. +Pressing **Enter**, entering `n` or `no`, or entering any other response stops before the guarded image or model download starts. `--yes` and `NEMOCLAW_YES=1` do not accept a verified insufficient-storage warning. Non-interactive setup stops unless you set `NEMOCLAW_IGNORE_VLLM_DISK_SPACE=1` exactly. -The override applies only when NemoClaw verifies that Docker image storage is insufficient. -Use it only after you independently verify that Docker storage can complete the image pull. +The override applies only when NemoClaw verifies that Docker image storage or model-cache storage is insufficient. +Use it only after you independently verify that the relevant filesystem can complete the download. Inconclusive checks continue automatically and do not require the override. -This preflight checks only the pinned vLLM container image. -It does not estimate or gate the Hugging Face model-weight download into `~/.cache/huggingface`. -Before the first managed-vLLM run, verify that the filesystem backing your model cache has enough free space for the selected model. +The model-cache guard applies when the selected registered recipe declares a pinned snapshot size. +NemoClaw creates `~/.cache/huggingface` as the host user and runs the one-shot download container with the host UID and GID, avoiding root-owned cache paths. +Hugging Face credentials are forwarded only to that temporary downloader, not retained by the long-lived vLLM container. +A legacy root-owned or otherwise unwritable cache stops before the image pull and prints a safe ownership-repair command; NemoClaw never changes its ownership automatically. If image pull output stops making progress, a watchdog stops the stalled pull without imposing a fixed wall-clock limit on slow but active downloads. @@ -142,7 +144,8 @@ When you start managed vLLM outside the installer express flow, NemoClaw uses th | Linux with an NVIDIA GPU | `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8` | On DGX Station, accepting the installer express prompt sets `NEMOCLAW_VLLM_MODEL=nemotron-3-ultra-550b-a55b` and overrides the profile default. -The registered recipe configures the pinned model revision, CPU offload, `16 GB` of shared memory, and the Nemotron reasoning and tool-call parsers. +Pass `--station-deepseek` to select the existing `deepseek-v4-flash` recipe while retaining the same one-confirmation express flow. +The registered Ultra recipe tracks the [official DGX Station deployment guide](https://github.com/NVIDIA-NeMo/Nemotron/blob/287ae845639d2ce998998cb8fd1f70a3fa943c0b/usage-cookbook/Nemotron-3-Ultra/StationDeploymentGuide/README.md) and configures the pinned model revision, CPU offload, `16 GB` of shared memory, memory/stack ulimits, MTP speculative decoding, and the Nemotron reasoning and tool-call parsers. NemoClaw intentionally keeps its existing bridge-networked managed-inference topology instead of importing the playbook's host-network setting. The container publishes port `8000` through Docker, so apply the firewall guidance at the top of this page. @@ -189,7 +192,8 @@ Slugs are case-insensitive, and NemoClaw also accepts the full Hugging Face mode An unrecognized value fails before image or model downloads and prints the valid slugs. Gated models require a Hugging Face token. -Export `HF_TOKEN` before onboarding so NemoClaw can forward it to the managed container. +Export `HF_TOKEN` before onboarding so NemoClaw can forward it temporarily to the one-shot model downloader. +The token is not retained in the long-lived managed vLLM container after the explicit download completes. ```bash export HF_TOKEN="" diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index e8d7c60f57a..3bdfc29328e 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3250,7 +3250,7 @@ Set them before running `$$nemoclaw onboard`. | `NEMOCLAW_INSTALL_TAG` | release tag | For internal installer commands: the release tag to install. Defaults to the admin-promoted `lkg` tag when unset. Overridden by the `--install-tag` flag. | | `NEMOCLAW_VLLM_MODEL` | registry slug or Hugging Face model id | Selects the model the managed-vLLM install path serves. Recognised slugs: `qwen3.6-27b`, `qwen3.6-35b-a3b-nvfp4`, `nemotron-3-nano-4b`, `deepseek-v4-flash`, `nemotron-3-ultra-550b-a55b`, `deepseek-r1-distill-70b`. Unset uses the per-platform profile default. The DGX Station express installer sets `nemotron-3-ultra-550b-a55b` explicitly. Gated models (e.g. `deepseek-r1-distill-70b`) require `HF_TOKEN` or `HUGGING_FACE_HUB_TOKEN`. | | `NEMOCLAW_VLLM_EXTRA_ARGS_JSON` | JSON array of non-blank strings | Appends advanced operator-owned tokens to the managed `vllm serve` command after NemoClaw's registry defaults. Example: `["--max-num-seqs","2"]`. Malformed JSON, non-string tokens, or blank tokens fail before Docker work starts. | -| `NEMOCLAW_IGNORE_VLLM_DISK_SPACE` | exactly `1` | Allows managed vLLM to continue when verified Docker image-storage capacity is insufficient; other values do not bypass the check. Inconclusive image-storage checks report diagnostics and continue automatically. The check does not cover Hugging Face model-cache capacity. | +| `NEMOCLAW_IGNORE_VLLM_DISK_SPACE` | exactly `1` | Allows managed vLLM to continue when verified Docker image-storage or registered model-cache capacity is insufficient; other values do not bypass either check. Inconclusive checks report diagnostics and continue automatically. | | `NEMOCLAW_MINIMAL_BOOTSTRAP` | `1` to enable | Skips default OpenClaw workspace-template seeding for new pristine workspaces. Existing files are not deleted; refer to [Understand Runtime Changes](../manage-sandboxes/configure-sandboxes/understand-runtime-changes). | diff --git a/docs/reference/platform-support.mdx b/docs/reference/platform-support.mdx index ae9f07ffb12..0131ee518b2 100644 --- a/docs/reference/platform-support.mdx +++ b/docs/reference/platform-support.mdx @@ -82,7 +82,7 @@ For the onboarding-time supported set without deferred rows, refer to [Prerequis | macOS (Apple Silicon) | Colima, Docker Desktop | Tested with limitations | P0 | Yes | Start the container runtime (Colima or Docker Desktop) before running the installer. Homebrew Colima users must install both Colima and the Docker CLI (`brew install colima docker`) before `docker info` can work. Xcode Command Line Tools (`xcode-select --install`) are typically required for Node native modules during install. NemoClaw recommends them but does not enforce them during preflight. | | DGX Spark | Docker | Tested | P1 | Yes | Use the standard installer and `$$nemoclaw onboard`. For an end-to-end walkthrough with local inference, see the [NVIDIA Spark playbook](https://build.nvidia.com/spark/nemoclaw). | | Windows WSL2 | Docker Desktop (WSL backend) | Tested with limitations | P1 | No | Requires WSL2 with Docker Desktop backend. | -| DGX Station | Docker | Deferred | P1 | No | The PRD marks this platform as P1. Workstation form-factor with NVIDIA GPUs and the same Docker + NVIDIA Container Toolkit + CDI requirements as DGX Spark. The installer detects DGX Station and offers express install with the pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` recipe, including an approximately 352 GB model download, without follow-up provider, model, policy, or sandbox-name choices. Direct managed-vLLM onboarding still defaults to `deepseek-ai/DeepSeek-V4-Flash` when no model override is set. The full NemoClaw onboarding path, including the express recipe, has not been validated end-to-end on physical DGX Station hardware and remains `deferred` until that run is signed off. | +| DGX Station | Docker | Deferred | P1 | No | The PRD marks this platform as P1. Workstation form-factor with NVIDIA GPUs and the same Docker + NVIDIA Container Toolkit + CDI requirements as DGX Spark. The installer detects DGX Station and offers express install with the pinned `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` recipe, including an approximately 352 GB model download, without follow-up provider, model, policy, or sandbox-name choices. Pass `--station-deepseek` to use `deepseek-ai/DeepSeek-V4-Flash` for a Station demo while retaining the one-confirmation express flow. Direct managed-vLLM onboarding still defaults to `deepseek-ai/DeepSeek-V4-Flash` when no model override is set. The full NemoClaw onboarding path, including the express recipe, has not been validated end-to-end on physical DGX Station hardware and remains `deferred` until that run is signed off. | | NVIDIA RTX (consumer and Pro workstation GPUs) | Docker | Deferred | P1 | No | The PRD marks this platform as P1. Covers RTX consumer cards and RTX Pro workstation cards on Linux hosts that meet the generic-Linux-GPU requirements (NVIDIA Container Toolkit + CDI present). The provider menu emits managed vLLM behind `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm` for this host class today; the end-to-end onboard path on this hardware is not yet validated in CI. | {/* platform-matrix-full:end */} diff --git a/install.sh b/install.sh index 44619f21d4c..ab867b4c891 100755 --- a/install.sh +++ b/install.sh @@ -113,6 +113,7 @@ bootstrap_usage() { printf " curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- [options]\n\n" printf " Options:\n" printf " --non-interactive Skip prompts (uses env vars / defaults)\n" + printf " --station-deepseek Use DeepSeek V4 Flash for DGX Station express install\n" printf " --yes-i-accept-third-party-software Accept the third-party software notice without prompting\n" printf " --fresh Discard any failed/interrupted onboarding session and start over\n" printf " --version, -v Print installer version and exit\n" diff --git a/scripts/install.sh b/scripts/install.sh index 67ec7a01a38..1688cb737cb 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -635,6 +635,7 @@ usage() { printf " --non-interactive Skip prompts (uses env vars / defaults)\n" printf " --yes-i-accept-third-party-software Accept the third-party software notice without prompting\n" printf " --fresh Discard any failed/interrupted onboarding session and start over\n" + printf " --station-deepseek Use DeepSeek V4 Flash for DGX Station express install\n" printf " --version, -v Print installer version and exit\n" printf " --help, -h Show this help message and exit\n\n" printf " ${C_DIM}Environment:${C_RESET}\n" @@ -2660,6 +2661,68 @@ detect_express_platform() { esac } +STATION_ULTRA_VLLM_MODEL="nemotron-3-ultra-550b-a55b" +STATION_ULTRA_SERVED_MODEL="nvidia/nemotron-3-ultra-550b-a55b" +STATION_DEEPSEEK_VLLM_MODEL="deepseek-v4-flash" +STATION_DEEPSEEK_SERVED_MODEL="deepseek-ai/DeepSeek-V4-Flash" + +normalize_station_vllm_model() { + printf "%s" "${1:-}" | tr '[:upper:]' '[:lower:]' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' +} + +validate_station_deepseek_override() { + local platform="$1" + if [ "${STATION_DEEPSEEK:-}" != "1" ]; then + return 0 + fi + if [ "$platform" != "DGX Station" ]; then + error "--station-deepseek requires a detected DGX Station (detected: ${platform:-unsupported platform})." + fi + if [ "${NEMOCLAW_NO_EXPRESS:-}" = "1" ]; then + error "--station-deepseek cannot be combined with NEMOCLAW_NO_EXPRESS=1. Remove one override." + fi + if [ "${NON_INTERACTIVE:-}" = "1" ]; then + error "--station-deepseek selects the DGX Station express prompt and cannot be combined with --non-interactive." + fi + if [ -n "${NEMOCLAW_PROVIDER:-}" ]; then + error "--station-deepseek conflicts with NEMOCLAW_PROVIDER=${NEMOCLAW_PROVIDER}. Remove the provider override to use Station express install." + fi + + local requested_model + requested_model="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_MODEL:-}")" + case "$requested_model" in + "" | "$STATION_DEEPSEEK_VLLM_MODEL" | "deepseek-ai/deepseek-v4-flash") ;; + *) + error "--station-deepseek conflicts with NEMOCLAW_VLLM_MODEL='${NEMOCLAW_VLLM_MODEL}'. Remove one override or set NEMOCLAW_VLLM_MODEL=${STATION_DEEPSEEK_VLLM_MODEL}." + ;; + esac +} + +configure_station_express_model() { + local selected_model + selected_model="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_MODEL:-}")" + if [ "${STATION_DEEPSEEK:-}" = "1" ]; then + NEMOCLAW_VLLM_MODEL="$STATION_DEEPSEEK_VLLM_MODEL" + NEMOCLAW_MODEL="$STATION_DEEPSEEK_SERVED_MODEL" + elif [ -z "$selected_model" ]; then + NEMOCLAW_VLLM_MODEL="$STATION_ULTRA_VLLM_MODEL" + NEMOCLAW_MODEL="$STATION_ULTRA_SERVED_MODEL" + else + case "$selected_model" in + "$STATION_ULTRA_VLLM_MODEL" | "nvidia/nvidia-nemotron-3-ultra-550b-a55b-nvfp4") + NEMOCLAW_MODEL="$STATION_ULTRA_SERVED_MODEL" + ;; + "$STATION_DEEPSEEK_VLLM_MODEL" | "deepseek-ai/deepseek-v4-flash") + NEMOCLAW_MODEL="$STATION_DEEPSEEK_SERVED_MODEL" + ;; + esac + fi + export NEMOCLAW_VLLM_MODEL + if [ -n "${NEMOCLAW_MODEL:-}" ]; then + export NEMOCLAW_MODEL + fi +} + # Prompt the user to opt into express install on supported platforms. Sets the # non-interactive + provider/model env vars when accepted. Skipped when # the user already passed --non-interactive, set NEMOCLAW_PROVIDER, or has @@ -2683,7 +2746,10 @@ describe_express_install() { sandbox_summary="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" ;; "DGX Station") - if [ -n "$(printf "%s" "${NEMOCLAW_VLLM_MODEL:-}" | tr -d '[:space:]')" ]; then + if [ "${STATION_DEEPSEEK:-}" = "1" ]; then + inference_summary="managed local vLLM with DeepSeek V4 Flash" + inference_disclosure="Managed vLLM pulls the configured Station image/model and runs a local inference container." + elif [ -n "$(printf "%s" "${NEMOCLAW_VLLM_MODEL:-}" | tr -d '[:space:]')" ]; then inference_summary="managed local vLLM with model ${NEMOCLAW_VLLM_MODEL}" inference_disclosure="Managed vLLM pulls the configured vLLM image/model and runs a local inference container." else @@ -2732,6 +2798,7 @@ describe_express_install() { maybe_offer_express_install() { local platform platform="$(detect_express_platform)" + validate_station_deepseek_override "$platform" # Not on a platform we have an express recipe for — say nothing. if [ -z "$platform" ]; then return 0 @@ -2793,10 +2860,7 @@ maybe_offer_express_install() { "DGX Station") export NEMOCLAW_SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" export NEMOCLAW_PROVIDER=install-vllm - if [ -z "$(printf "%s" "${NEMOCLAW_VLLM_MODEL:-}" | tr -d '[:space:]')" ]; then - NEMOCLAW_VLLM_MODEL="nemotron-3-ultra-550b-a55b" - fi - export NEMOCLAW_VLLM_MODEL + configure_station_express_model ;; "Windows WSL") export NEMOCLAW_PROVIDER=install-windows-ollama @@ -2821,11 +2885,13 @@ main() { NON_INTERACTIVE="" ACCEPT_THIRD_PARTY_SOFTWARE="" FRESH="" + STATION_DEEPSEEK="" for arg in "$@"; do case "$arg" in --non-interactive) NON_INTERACTIVE=1 ;; --yes-i-accept-third-party-software) ACCEPT_THIRD_PARTY_SOFTWARE=1 ;; --fresh) FRESH=1 ;; + --station-deepseek) STATION_DEEPSEEK=1 ;; --version | -v) local version_suffix version_suffix="$(installer_version_for_display)" diff --git a/src/lib/inference/vllm-models.test.ts b/src/lib/inference/vllm-models.test.ts index d6ed302335f..0b3e81bbe38 100644 --- a/src/lib/inference/vllm-models.test.ts +++ b/src/lib/inference/vllm-models.test.ts @@ -67,7 +67,9 @@ describe("vllm model registry", () => { image: "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416", imageDownloadSizeBytes: 10_670_087_425, - dockerRunArgs: ["--shm-size", "16g"], + modelDownloadSizeBytes: 352_381_245_521, + loadTimeoutSec: 3600, + dockerRunArgs: ["--shm-size", "16g", "--ulimit", "memlock=-1", "--ulimit", "stack=67108864"], }); const cmd = buildVllmServeCommand(ultra!); @@ -77,6 +79,9 @@ describe("vllm model registry", () => { expect(cmd).toContain("--served-model-name nvidia/nemotron-3-ultra-550b-a55b"); expect(cmd).toContain("--cpu-offload-gb 150 --cpu-offload-params experts"); expect(cmd).toContain(`--kernel_config '{"enable_flashinfer_autotune": false}'`); + expect(cmd).toContain( + `--speculative-config '{"method":"nemotron_h_mtp","num_speculative_tokens":3}'`, + ); expect(cmd).toContain("--max-num-seqs 256 --gpu-memory-utilization 0.9"); expect(cmd).toContain("--reasoning-parser nemotron_v3"); expect(cmd).toContain("--enable-auto-tool-choice --tool-call-parser qwen3_coder"); diff --git a/src/lib/inference/vllm-models.ts b/src/lib/inference/vllm-models.ts index 085acead90f..e0b927e61e9 100644 --- a/src/lib/inference/vllm-models.ts +++ b/src/lib/inference/vllm-models.ts @@ -34,6 +34,10 @@ export interface VllmRuntimeOverride { image: string; /** Compressed size of the selected platform manifest. */ imageDownloadSizeBytes: number; + /** Size of the pinned Hugging Face snapshot used for cache preflight. */ + modelDownloadSizeBytes?: number; + /** Maximum time to wait for this model to become ready after launch. */ + loadTimeoutSec?: number; /** Additional `docker run` arguments required by this recipe. */ dockerRunArgs?: readonly string[]; } @@ -214,6 +218,8 @@ export const VLLM_MODELS: readonly VllmModelDef[] = [ "experts", "--kernel_config", `'{"enable_flashinfer_autotune": false}'`, + "--speculative-config", + `'{"method":"nemotron_h_mtp","num_speculative_tokens":3}'`, "--max-num-seqs", "256", "--gpu-memory-utilization", @@ -235,9 +241,11 @@ export const VLLM_MODELS: readonly VllmModelDef[] = [ runtime: { image: NEMOTRON_ULTRA_STATION_IMAGE.arm64.ref, imageDownloadSizeBytes: NEMOTRON_ULTRA_STATION_IMAGE.arm64.downloadSizeBytes, + modelDownloadSizeBytes: 352_381_245_521, + loadTimeoutSec: 3600, // Keep NemoClaw's bridge-networked local-inference boundary instead of // importing the playbook's host-network setting. - dockerRunArgs: ["--shm-size", "16g"], + dockerRunArgs: ["--shm-size", "16g", "--ulimit", "memlock=-1", "--ulimit", "stack=67108864"], }, // The digest-pinned vLLM image already contains the serving package, and // this recipe does not use the fastsafetensors load format. Avoid mutating diff --git a/src/lib/inference/vllm-storage.test.ts b/src/lib/inference/vllm-storage.test.ts index c9b56d23c31..07a3c578865 100644 --- a/src/lib/inference/vllm-storage.test.ts +++ b/src/lib/inference/vllm-storage.test.ts @@ -3,9 +3,13 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { + findUnwritableTreePath, formatStorageBytes, imageStorageRequirementBytes, + measureDirectorySizeBytes, + modelStorageRequirementBytes, probeDockerStorage, + probeHostStorage, resolveDockerStorageLocations, } from "./vllm-storage"; @@ -42,12 +46,163 @@ describe("managed vLLM image-storage requirements", () => { ); }); + it("reserves the pinned snapshot plus staging space for a model download", () => { + expect(modelStorageRequirementBytes(352_381_245_521)).toBe(352_381_245_521n + 3n * GIB); + expect(() => modelStorageRequirementBytes(0)).toThrow( + "vLLM model download size must be a positive finite byte count", + ); + }); + it("formats available and required bytes as rounded GiB values", () => { expect(formatStorageBytes(2n * GIB)).toBe("2 GiB"); expect(formatStorageBytes((23n * GIB) / 10n)).toBe("2.3 GiB"); }); }); +describe("host cache-storage detection", () => { + it("finds an unwritable root-created cache directory", () => { + expect( + findUnwritableTreePath("/home/user/.cache/huggingface", { + canWrite: (target) => target !== "/home/user/.cache/huggingface", + list: () => [], + }), + ).toBe("/home/user/.cache/huggingface"); + }); + + it("finds root-owned descendants before a host-UID download", () => { + const entries = new Map([ + [ + "/cache", + [ + { kind: "directory" as const, name: "hub" }, + { kind: "file" as const, name: "version.txt" }, + ], + ], + ["/cache/hub", [{ kind: "file" as const, name: "root-owned-blob" }]], + ]); + const checked: string[] = []; + + expect( + findUnwritableTreePath("/cache", { + canWrite: (target) => { + checked.push(target); + return target !== "/cache/hub/root-owned-blob"; + }, + list: (target) => entries.get(target) ?? [], + }), + ).toBe("/cache/hub/root-owned-blob"); + expect(checked).toEqual(expect.arrayContaining(["/cache", "/cache/version.txt", "/cache/hub"])); + }); + + it("accepts a writable cache tree", () => { + expect( + findUnwritableTreePath("/cache", { + canWrite: () => true, + list: () => [], + }), + ).toBeNull(); + }); + + it("counts complete and partial snapshot files without following directory symlinks", () => { + const entries = new Map([ + [ + "/cache/snapshot", + [ + { kind: "file" as const, name: "config.json" }, + { kind: "directory" as const, name: "weights" }, + { kind: "symlink" as const, name: "tokenizer.json" }, + { kind: "symlink" as const, name: "directory-link" }, + ], + ], + ["/cache/snapshot/weights", [{ kind: "symlink" as const, name: "model-00001.safetensors" }]], + ]); + const sizes = new Map([ + ["/cache/snapshot/config.json", 100n], + ["/cache/snapshot/tokenizer.json", 200n], + ["/cache/snapshot/weights/model-00001.safetensors", 1_000n], + ["/cache/snapshot/directory-link", null], + ]); + + expect( + measureDirectorySizeBytes("/cache/snapshot", { + exists: () => true, + list: (target) => entries.get(target) ?? [], + statFileSize: (target) => sizes.get(target) ?? null, + }), + ).toBe(1_300n); + }); + + it("treats a missing or unreadable snapshot as uncached", () => { + expect( + measureDirectorySizeBytes("/cache/missing", { + exists: () => false, + }), + ).toBe(0n); + expect( + measureDirectorySizeBytes("/cache/interrupted", { + exists: () => true, + list: () => { + throw new Error("broken link"); + }, + }), + ).toBe(0n); + }); + + it("measures the filesystem containing an existing cache directory", () => { + const statfs = vi.fn(() => ({ bavail: 400n, bsize: GIB })); + + expect( + probeHostStorage("/home/user/.cache/huggingface", "Hugging Face cache", { + exists: () => true, + statfs, + }), + ).toEqual({ + ok: true, + capacity: { + availableBytes: 400n * GIB, + path: "/home/user/.cache/huggingface", + source: "Hugging Face cache", + }, + }); + expect(statfs).toHaveBeenCalledWith("/home/user/.cache/huggingface"); + }); + + it("measures the nearest existing ancestor before a first cache download", () => { + const statfs = vi.fn(() => ({ bavail: 500n, bsize: GIB })); + + expect( + probeHostStorage("/home/user/.cache/huggingface", "Hugging Face cache", { + exists: (target) => target === "/home/user", + statfs, + }), + ).toEqual({ + ok: true, + capacity: { + availableBytes: 500n * GIB, + path: "/home/user/.cache/huggingface", + source: "Hugging Face cache", + }, + }); + expect(statfs).toHaveBeenCalledWith("/home/user"); + }); + + it("reports an inconclusive host probe without creating the cache path", () => { + expect( + probeHostStorage("/mnt/models/huggingface", "Hugging Face cache", { + exists: (target) => target === "/mnt", + statfs: () => { + throw new Error("mount unavailable"); + }, + }), + ).toEqual({ + ok: false, + reason: "could not inspect /mnt: mount unavailable", + path: "/mnt/models/huggingface", + source: "Hugging Face cache", + }); + }); +}); + describe("Docker image-storage detection", () => { it("measures DockerRootDir for the classic image store (#6757)", () => { const statfs = vi.fn(() => ({ bavail: 7n, bsize: GIB })); diff --git a/src/lib/inference/vllm-storage.ts b/src/lib/inference/vllm-storage.ts index f197a0d78cc..4b65bab227c 100644 --- a/src/lib/inference/vllm-storage.ts +++ b/src/lib/inference/vllm-storage.ts @@ -10,7 +10,7 @@ import { buildVllmDockerEnv } from "./vllm-docker-env"; export const VLLM_STORAGE_OVERRIDE_ENV = "NEMOCLAW_IGNORE_VLLM_DISK_SPACE"; const GIB_BYTES = 1024n ** 3n; -const IMAGE_PULL_TEMP_HEADROOM_BYTES = 3n * GIB_BYTES; +const DOWNLOAD_TEMP_HEADROOM_BYTES = 3n * GIB_BYTES; const DEFAULT_CONTAINERD_ROOT = "/var/lib/containerd"; const DEFAULT_CONTAINERD_CONFIG = "/etc/containerd/config.toml"; const DEFAULT_DOCKER_SOCKET_PATHS = new Set(["/var/run/docker.sock", "/run/docker.sock"]); @@ -82,8 +82,18 @@ function positiveBytes(value: number, label: string): bigint { */ export function imageStorageRequirementBytes(downloadSizeBytes: number): bigint { return ( - positiveBytes(downloadSizeBytes, "vLLM image download size") * 3n + - IMAGE_PULL_TEMP_HEADROOM_BYTES + positiveBytes(downloadSizeBytes, "vLLM image download size") * 3n + DOWNLOAD_TEMP_HEADROOM_BYTES + ); +} + +/** + * Hugging Face downloads stream into the target cache before atomically + * promoting the completed blobs. Reserve the pinned snapshot size plus the + * same fixed staging allowance used by the managed-image guard. + */ +export function modelStorageRequirementBytes(downloadSizeBytes: number): bigint { + return ( + positiveBytes(downloadSizeBytes, "vLLM model download size") + DOWNLOAD_TEMP_HEADROOM_BYTES ); } @@ -286,3 +296,172 @@ export function probeDockerStorage(overrides: Partial = {}): S ? { ok: true, capacity: limiting } : { ok: false, reason: "docker info did not report a usable image-storage path" }; } + +interface HostStorageProbeDeps { + exists: (target: string) => boolean; + statfs: (target: string) => { bavail: bigint; bsize: bigint }; +} + +interface DirectorySizeEntry { + kind: "directory" | "file" | "symlink" | "other"; + name: string; +} + +interface WritableTreeDeps { + canWrite: (target: string, directory: boolean) => boolean; + list: (target: string) => DirectorySizeEntry[]; +} + +function defaultWritableTreeDeps(): WritableTreeDeps { + return { + canWrite: (target, directory) => { + try { + const mode = fs.constants.R_OK | fs.constants.W_OK | (directory ? fs.constants.X_OK : 0); + fs.accessSync(target, mode); + return true; + } catch { + return false; + } + }, + list: defaultDirectorySizeDeps().list, + }; +} + +/** + * Return the first path in a cache tree that the current host user cannot + * read and write. Directories additionally require search permission. This + * catches legacy cache trees created by a root-run Docker downloader before + * a host-UID downloader is started. Symlinks are not followed because their + * targets are checked separately under the cache's blobs directory. + */ +export function findUnwritableTreePath( + targetPath: string, + overrides: Partial = {}, +): string | null { + const deps = { ...defaultWritableTreeDeps(), ...overrides }; + const pending = [targetPath]; + while (pending.length > 0) { + const directory = pending.pop()!; + if (!deps.canWrite(directory, true)) return directory; + let entries: DirectorySizeEntry[]; + try { + entries = deps.list(directory); + } catch { + return directory; + } + for (const entry of entries) { + const entryPath = path.join(directory, entry.name); + if (entry.kind === "directory") { + pending.push(entryPath); + } else if (entry.kind === "file" && !deps.canWrite(entryPath, false)) { + return entryPath; + } + } + } + return null; +} + +interface DirectorySizeDeps { + exists: (target: string) => boolean; + list: (target: string) => DirectorySizeEntry[]; + statFileSize: (target: string) => bigint | null; +} + +function defaultDirectorySizeDeps(): DirectorySizeDeps { + return { + exists: fs.existsSync, + list: (target) => + fs.readdirSync(target, { withFileTypes: true }).map((entry) => ({ + kind: entry.isDirectory() + ? "directory" + : entry.isFile() + ? "file" + : entry.isSymbolicLink() + ? "symlink" + : "other", + name: entry.name, + })), + statFileSize: (target) => { + const stat = fs.statSync(target, { bigint: true }); + return stat.isFile() ? stat.size : null; + }, + }; +} + +/** + * Return the logical bytes represented by regular files in a directory tree. + * Symlinks are followed only when they resolve to files; directory symlinks + * are never traversed, avoiding cycles. Any unreadable or malformed tree + * returns zero so callers conservatively plan for a full download. + */ +export function measureDirectorySizeBytes( + targetPath: string, + overrides: Partial = {}, +): bigint { + const deps = { ...defaultDirectorySizeDeps(), ...overrides }; + if (!deps.exists(targetPath)) return 0n; + function walk(directory: string): bigint { + let total = 0n; + for (const entry of deps.list(directory)) { + const entryPath = path.join(directory, entry.name); + if (entry.kind === "directory") { + total += walk(entryPath); + } else if (entry.kind === "file" || entry.kind === "symlink") { + total += deps.statFileSize(entryPath) ?? 0n; + } + } + return total; + } + try { + return walk(targetPath); + } catch { + return 0n; + } +} + +/** + * Measure the filesystem that will contain a host cache path. On a fresh + * install the cache directory may not exist yet, so walk to its nearest + * existing ancestor without creating anything before the capacity decision. + */ +export function probeHostStorage( + targetPath: string, + source: string, + overrides: Partial = {}, +): StorageProbeResult { + if (!path.isAbsolute(targetPath)) { + return { + ok: false, + reason: `storage path is not absolute: ${targetPath}`, + path: targetPath, + source, + }; + } + const deps: HostStorageProbeDeps = { + exists: fs.existsSync, + statfs: (target) => fs.statfsSync(target, { bigint: true }), + ...overrides, + }; + let probePath = path.normalize(targetPath); + while (!deps.exists(probePath)) { + const parent = path.dirname(probePath); + if (parent === probePath) break; + probePath = parent; + } + try { + const stats = deps.statfs(probePath); + const availableBytes = stats.bavail * stats.bsize; + if (availableBytes < 0n) throw new Error("filesystem reported negative available space"); + return { + ok: true, + capacity: { availableBytes, path: targetPath, source }, + }; + } catch (err) { + return { + ok: false, + reason: `could not inspect ${probePath}: ${(err as Error).message}`, + path: targetPath, + source, + }; + } +} diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index a5685b1e206..a86f7944be0 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { EventEmitter } from "node:events"; +import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -14,8 +15,11 @@ const mocks = vi.hoisted(() => ({ dockerRunDetached: vi.fn(), dockerSpawn: vi.fn(), dockerStop: vi.fn(), + findUnwritableTreePath: vi.fn(), getGpuIndicesByName: vi.fn<(_pattern: RegExp) => number[]>(() => []), + measureDirectorySizeBytes: vi.fn(), probeDockerStorage: vi.fn(), + probeHostStorage: vi.fn(), runCapture: vi.fn(), })); @@ -41,7 +45,10 @@ vi.mock("./vllm-storage", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, + findUnwritableTreePath: mocks.findUnwritableTreePath, + measureDirectorySizeBytes: mocks.measureDirectorySizeBytes, probeDockerStorage: mocks.probeDockerStorage, + probeHostStorage: mocks.probeHostStorage, }; }); @@ -60,10 +67,20 @@ import { VLLM_MODELS } from "./vllm-models"; beforeEach(() => { mocks.dockerImageInspectFormat.mockReturnValue(""); + mocks.findUnwritableTreePath.mockReturnValue(null); + mocks.measureDirectorySizeBytes.mockReturnValue(0n); mocks.probeDockerStorage.mockReturnValue({ ok: true, capacity: { availableBytes: 1_000_000_000_000n, path: "/docker", source: "Docker" }, }); + mocks.probeHostStorage.mockReturnValue({ + ok: true, + capacity: { + availableBytes: 1_000_000_000_000n, + path: path.join(os.homedir(), ".cache", "huggingface"), + source: "Hugging Face cache", + }, + }); }); function mockDockerSpawnSuccess(): EventEmitter & { @@ -172,6 +189,8 @@ describe("vLLM profile detection", () => { "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416", ); expect(runtime.imageDownloadSizeBytes).toBe(10_670_087_425); + expect(runtime.modelDownloadSizeBytes).toBe(352_381_245_521); + expect(runtime.loadTimeoutSec).toBe(3600); expect(runtime.buildDockerRunFlags!()).toEqual( expect.arrayContaining(["--gpus", "device=0", "--shm-size", "16g"]), ); @@ -348,9 +367,10 @@ describe("vLLM run command", () => { HF_TOKEN: "hf_test", } as NodeJS.ProcessEnv); - expect(args).toEqual(expect.arrayContaining(["--label", labelValue, "-e", "HF_TOKEN"])); + expect(args).toEqual(expect.arrayContaining(["--label", labelValue])); expect(args).not.toContain(`--label ${labelValue}`); expect(args).not.toContain("-e HF_TOKEN"); + expect(args).not.toContain("HF_TOKEN"); expect(args.join(" ")).not.toContain("hf_test"); }); @@ -437,6 +457,7 @@ describe("managed vLLM ownership", () => { describe("installVllm model resolution", () => { let logSpy: ReturnType; let errSpy: ReturnType; + let mkdirSpy: ReturnType; let stdoutWrite: ReturnType; const originalEnv = { ...process.env }; @@ -444,6 +465,7 @@ describe("installVllm model resolution", () => { vi.clearAllMocks(); logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + mkdirSpy = vi.spyOn(fs, "mkdirSync").mockImplementation(() => undefined); stdoutWrite = vi.spyOn(process.stdout, "write").mockImplementation(() => true); delete process.env.NEMOCLAW_VLLM_MODEL; delete process.env.NEMOCLAW_VLLM_EXTRA_ARGS_JSON; @@ -459,6 +481,7 @@ describe("installVllm model resolution", () => { afterEach(() => { logSpy.mockRestore(); errSpy.mockRestore(); + mkdirSpy.mockRestore(); stdoutWrite.mockRestore(); process.env = { ...originalEnv }; }); @@ -537,6 +560,31 @@ describe("installVllm model resolution", () => { expect(result).toEqual({ ok: true }); expect(promptFn).not.toHaveBeenCalled(); expect(beforeInstall).toHaveBeenCalledWith("nvidia/nemotron-3-ultra-550b-a55b"); + expect(mocks.measureDirectorySizeBytes).toHaveBeenCalledWith( + path.join( + os.homedir(), + ".cache", + "huggingface", + "hub", + "models--nvidia--NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", + "snapshots", + "183968f87ae4cedce3039313cac1fd43d112c578", + ), + ); + expect(mocks.probeHostStorage).toHaveBeenCalledWith( + path.join(os.homedir(), ".cache", "huggingface"), + "Hugging Face cache", + ); + expect(mocks.probeHostStorage.mock.invocationCallOrder[0]).toBeLessThan( + mocks.dockerPullWithProgressWatchdog.mock.invocationCallOrder[0], + ); + expect(mocks.probeHostStorage).toHaveBeenCalledTimes(2); + expect(mocks.dockerPullWithProgressWatchdog.mock.invocationCallOrder[0]).toBeLessThan( + mocks.probeHostStorage.mock.invocationCallOrder[1], + ); + expect(mocks.probeHostStorage.mock.invocationCallOrder[1]).toBeLessThan( + mocks.dockerSpawn.mock.invocationCallOrder[0], + ); expect(mocks.dockerPullWithProgressWatchdog).toHaveBeenCalledWith( "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416", expect.any(Object), @@ -544,12 +592,29 @@ describe("installVllm model resolution", () => { const [downloadArgs] = mocks.dockerSpawn.mock.calls[0] as [string[]]; expect(downloadArgs).toEqual( expect.arrayContaining([ + "-v", + `${path.join(os.homedir(), ".cache", "huggingface")}:/tmp/nemoclaw-huggingface`, + "-e", + "HF_HOME=/tmp/nemoclaw-huggingface", "download", "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", "--revision", "183968f87ae4cedce3039313cac1fd43d112c578", ]), ); + if (typeof process.getuid === "function" && typeof process.getgid === "function") { + expect(downloadArgs).toEqual( + expect.arrayContaining([ + "--user", + `${String(process.getuid())}:${String(process.getgid())}`, + ]), + ); + } else { + expect(downloadArgs).not.toContain("--user"); + } + expect(mkdirSpy).toHaveBeenCalledWith(path.join(os.homedir(), ".cache", "huggingface"), { + recursive: true, + }); const [runArgs] = mocks.dockerRunDetached.mock.calls[0] as [string[]]; expect(runArgs).toEqual(expect.arrayContaining(["--shm-size", "16g", "-p", "8000:8000"])); expect(runArgs).not.toContain("--network"); @@ -664,6 +729,166 @@ describe("installVllm model resolution", () => { expect(errors).toContain("docker system df"); }); + it("stops an absent or interrupted Ultra download when the HF cache is too small", async () => { + process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName); + mocks.probeHostStorage.mockReturnValue({ + ok: true, + capacity: { + availableBytes: 1n, + path: path.join(os.homedir(), ".cache", "huggingface"), + source: "Hugging Face cache", + }, + }); + + const result = await installVllm(profile, { + hasImage: false, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: false }); + expect(mocks.probeHostStorage).toHaveBeenCalledTimes(1); + expect(mocks.dockerImageInspectFormat).not.toHaveBeenCalled(); + expect(mocks.probeDockerStorage).not.toHaveBeenCalled(); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); + expect(mkdirSpy).not.toHaveBeenCalled(); + const errors = errSpy.mock.calls.map((call: unknown[]) => String(call[0])).join("\n"); + expect(errors).toContain("Insufficient storage for the managed vLLM model cache"); + expect(errors).toContain("nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4"); + expect(errors).toContain("Hugging Face cache"); + expect(errors).toContain("Non-interactive setup stops before the guarded download"); + }); + + it("re-probes after a cold image pull to prevent shared-filesystem overcommit", async () => { + process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName); + mocks.probeHostStorage + // The model and image checks can each pass against this initial free + // space even though their combined cold-download footprint cannot. + .mockReturnValueOnce({ + ok: true, + capacity: { + availableBytes: 360_000_000_000n, + path: path.join(os.homedir(), ".cache", "huggingface"), + source: "Hugging Face cache", + }, + }) + // Account for image layers consumed on the same filesystem. + .mockReturnValueOnce({ + ok: true, + capacity: { + availableBytes: 340_000_000_000n, + path: path.join(os.homedir(), ".cache", "huggingface"), + source: "Hugging Face cache", + }, + }); + + const result = await installVllm(profile, { + hasImage: false, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: false }); + expect(mocks.probeHostStorage).toHaveBeenCalledTimes(2); + expect(mocks.dockerPullWithProgressWatchdog).toHaveBeenCalledTimes(1); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); + expect(errSpy).toHaveBeenCalledWith( + expect.stringContaining("Insufficient storage for the managed vLLM model cache"), + ); + }); + + it("requires only missing snapshot bytes plus headroom for a partial Ultra cache", async () => { + process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName); + mocks.dockerImageInspectFormat.mockReturnValue("sha256:cached-image"); + mocks.measureDirectorySizeBytes.mockReturnValue(342_381_245_521n); + mocks.probeHostStorage.mockReturnValue({ + ok: true, + capacity: { + // 10 GB remains, plus the shared 3 GiB staging allowance. This is + // enough for the remainder but intentionally far below 352 GB. + availableBytes: 14_000_000_000n, + path: path.join(os.homedir(), ".cache", "huggingface"), + source: "Hugging Face cache", + }, + }); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: true }); + expect(mocks.probeHostStorage).toHaveBeenCalledTimes(1); + expect(mocks.dockerPullWithProgressWatchdog).toHaveBeenCalledTimes(1); + expect(mocks.dockerSpawn).toHaveBeenCalledTimes(1); + }); + + it("skips the capacity gate for a complete pinned Ultra snapshot", async () => { + process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName); + mocks.dockerImageInspectFormat.mockReturnValue("sha256:cached-image"); + mocks.measureDirectorySizeBytes.mockReturnValue(352_381_245_521n); + mocks.probeHostStorage.mockReturnValue({ + ok: true, + capacity: { + availableBytes: 1n, + path: path.join(os.homedir(), ".cache", "huggingface"), + source: "Hugging Face cache", + }, + }); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: true }); + expect(mocks.measureDirectorySizeBytes).toHaveBeenCalledTimes(1); + expect(mocks.probeHostStorage).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).toHaveBeenCalledTimes(1); + }); + + it("uses the dedicated disk-space override for a low Ultra model cache", async () => { + process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + process.env.NEMOCLAW_IGNORE_VLLM_DISK_SPACE = "1"; + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName); + mocks.dockerImageInspectFormat.mockReturnValue("sha256:cached-image"); + mocks.probeHostStorage.mockReturnValue({ + ok: true, + capacity: { + availableBytes: 1n, + path: path.join(os.homedir(), ".cache", "huggingface"), + source: "Hugging Face cache", + }, + }); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: true }); + expect(mocks.dockerPullWithProgressWatchdog).toHaveBeenCalledTimes(1); + expect(mocks.dockerSpawn).toHaveBeenCalledTimes(1); + expect(errSpy).toHaveBeenCalledWith( + expect.stringContaining("NEMOCLAW_IGNORE_VLLM_DISK_SPACE=1"), + ); + }); + it("continues an uncached image pull only after an explicit storage yes (#6757)", async () => { const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; process.env.NEMOCLAW_VLLM_MODEL = profile.defaultModel.envValue; @@ -776,6 +1001,7 @@ describe("installVllm model resolution", () => { expect.objectContaining({ env: expect.any(Object), ignoreError: true, timeout: 10_000 }), ); expect(mocks.probeDockerStorage).not.toHaveBeenCalled(); + expect(mocks.probeHostStorage).not.toHaveBeenCalled(); expect(mocks.dockerPullWithProgressWatchdog).toHaveBeenCalledTimes(1); expect(mocks.dockerSpawn).toHaveBeenCalledTimes(1); const [downloadArgs] = mocks.dockerSpawn.mock.calls[0] as [string[]]; @@ -838,7 +1064,59 @@ describe("installVllm model resolution", () => { } }); - it("starts the long-lived vLLM container through Docker argv, not a shell command", async () => { + it("fails before image pull when the host Hugging Face cache cannot be created", async () => { + const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName); + mkdirSpy.mockImplementation(() => { + throw new Error("permission denied"); + }); + + const result = await installVllm(profile, { + hasImage: false, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: false }); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); + expect(errSpy).toHaveBeenCalledWith( + expect.stringContaining("could not create Hugging Face cache directory"), + ); + }); + + it("fails before image pull with a safe repair command for a root-owned cache", async () => { + const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName); + mocks.dockerImageInspectFormat.mockReturnValue("sha256:cached-image"); + const cacheDir = path.join(os.homedir(), ".cache", "huggingface"); + const rootOwnedPath = path.join(cacheDir, "hub", ".locks"); + mocks.findUnwritableTreePath.mockReturnValue(rootOwnedPath); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: false }); + expect(mocks.findUnwritableTreePath).toHaveBeenCalledWith(cacheDir); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); + const errors = errSpy.mock.calls.map((call: unknown[]) => String(call[0])).join("\n"); + expect(errors).toContain(rootOwnedPath); + expect(errors).toContain("not writable by host user"); + expect(errors).toContain("NemoClaw did not modify it"); + expect(errors).toContain("sudo chown -R"); + expect(errors).toContain(`'${cacheDir}'`); + if (typeof process.getuid === "function" && typeof process.getgid === "function") { + expect(errors).toContain(`${String(process.getuid())}:${String(process.getgid())}`); + } + }); + + it("limits the Hugging Face token to the one-shot download container", async () => { process.env.HF_TOKEN = "hf_test"; const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; mockSuccessfulVllmInstall(profile.containerName); @@ -852,27 +1130,28 @@ describe("installVllm model resolution", () => { expect(result).toEqual({ ok: true }); expect(mocks.dockerForceRm).not.toHaveBeenCalled(); + const [downloadArgs, downloadOpts] = mocks.dockerSpawn.mock.calls[0] as [ + string[], + { env?: Record }, + ]; + expect(downloadArgs).toEqual(expect.arrayContaining(["-e", "HF_TOKEN"])); + expect(downloadArgs.join(" ")).not.toContain("hf_test"); + expect(downloadOpts).toEqual( + expect.objectContaining({ env: expect.objectContaining({ HF_TOKEN: "hf_test" }) }), + ); expect(mocks.dockerRunDetached).toHaveBeenCalledTimes(1); const [args, opts] = mocks.dockerRunDetached.mock.calls[0] as [ string[], { env?: Record }, ]; expect(args).toEqual( - expect.arrayContaining([ - "--pull=never", - "--restart", - "unless-stopped", - "-e", - "HF_TOKEN", - profile.image, - ]), + expect.arrayContaining(["--pull=never", "--restart", "unless-stopped", profile.image]), ); + expect(args).not.toContain("HF_TOKEN"); expect(args.join(" ")).not.toContain("hf_test"); expect(args.some((arg) => arg.includes("docker run"))).toBe(false); expect(args[args.indexOf("-lc") + 1]).toContain("vllm serve"); - expect(opts).toEqual( - expect.objectContaining({ env: expect.objectContaining({ HF_TOKEN: "hf_test" }) }), - ); + expect(opts.env).not.toHaveProperty("HF_TOKEN"); }); it("replaces only an existing managed container by its inspected ID", async () => { diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 1c8e46ccdf2..fe98350f528 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -5,6 +5,7 @@ // offer vLLM at all" lives in onboard.ts; this module owns picking the // right profile per platform and running the install. +import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { @@ -18,6 +19,7 @@ import { } from "../adapters/docker"; import { buildValidatedCurlCommandArgs } from "../adapters/http/curl-args"; import { VLLM_PORT } from "../core/ports"; +import { shellQuote } from "../core/shell-quote"; import { isAffirmativeAnswer } from "../onboard/prompt-helpers"; import { runCapture } from "../runner"; import { isSafeModelId } from "../validation"; @@ -34,9 +36,13 @@ import { } from "./vllm-models"; import { resolveVllmInstallModel } from "./vllm-prompt"; import { + findUnwritableTreePath, formatStorageBytes, imageStorageRequirementBytes, + measureDirectorySizeBytes, + modelStorageRequirementBytes, probeDockerStorage, + probeHostStorage, type StorageProbeResult, VLLM_STORAGE_OVERRIDE_ENV, } from "./vllm-storage"; @@ -73,6 +79,9 @@ export interface VllmProfile { pullTimeoutSec: number; // Wall-clock budget for the load phase (after pull, before ready). loadTimeoutSec: number; + // Optional pinned model snapshot size. Model-specific runtime overrides use + // this to guard the host Hugging Face cache before a cold download. + modelDownloadSizeBytes?: number; } // Platform manifests and decimal compressed sizes published by NGC for the @@ -122,6 +131,8 @@ const HF_TOKEN_ENV_KEYS = ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"] as const; const MODEL_DOWNLOAD_HEARTBEAT_MS = 30_000; const VLLM_LAUNCH_HEARTBEAT_MS = 30_000; const HF_CACHE_CONTAINER_DIR = "/root/.cache/huggingface"; +const HF_DOWNLOAD_CACHE_CONTAINER_DIR = "/tmp/nemoclaw-huggingface"; +const HF_CACHE_COMPONENT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/; export const NEMOCLAW_VLLM_CONTAINER_NAME = "nemoclaw-vllm"; export const NEMOCLAW_VLLM_MANAGED_LABEL = "com.nvidia.nemoclaw.managed-vllm"; const DOCKER_CONTAINER_ID_PATTERN = /^[a-f0-9]{12,64}$/; @@ -134,6 +145,39 @@ function hfCacheMount(): string { return `${hostHfCacheDir()}:${HF_CACHE_CONTAINER_DIR}`; } +function hfDownloadCacheMount(): string { + return `${hostHfCacheDir()}:${HF_DOWNLOAD_CACHE_CONTAINER_DIR}`; +} + +function hfModelSnapshotDir(model: VllmModelDef): string | null { + const revision = model.revision; + const modelParts = model.id.split("/"); + if ( + !revision || + !HF_CACHE_COMPONENT_PATTERN.test(revision) || + modelParts.some((part) => !HF_CACHE_COMPONENT_PATTERN.test(part)) + ) { + return null; + } + return path.join( + hostHfCacheDir(), + "hub", + `models--${modelParts.join("--")}`, + "snapshots", + revision, + ); +} + +function hostUserIdentity(): string | null { + if (typeof process.getuid !== "function" || typeof process.getgid !== "function") return null; + return `${String(process.getuid())}:${String(process.getgid())}`; +} + +function hostUserDockerArgs(): string[] { + const identity = hostUserIdentity(); + return identity ? ["--user", identity] : []; +} + function vllmDockerRunFlags(gpuFlag = "all"): string[] { return [ "--gpus", @@ -157,16 +201,15 @@ function pickHfTokenEntry( } /** - * Forward a Hugging Face token from the host into the vLLM/hf container so - * `hf download` and `vllm serve` can pull weights for gated models. + * Forward a Hugging Face token from the host into the one-shot `hf download` + * container so gated model weights can be fetched. * * Returns the bare `-e KEY` form (no `=value`) so the token never lands in * the host process list. Docker reads the actual value from its own * environment, which the caller is responsible for populating via * `buildHfTokenForwardEnv` when spawning through the runner allowlist. - * The `hf download` container can live for several minutes during a cold - * pull and `vllm serve` runs for the lifetime of the sandbox; argv-embedded - * secrets would be visible via `ps` for that whole window. + * The download container can live for several minutes during a cold pull; + * argv-embedded secrets would be visible via `ps` for that whole window. */ export function buildHfTokenDockerArgs(env: NodeJS.ProcessEnv = process.env): string[] { const entry = pickHfTokenEntry(env); @@ -327,12 +370,13 @@ function downloadModel( "-t", "--rm", "--pull=never", + ...hostUserDockerArgs(), "--entrypoint", "hf", "-v", - hfCacheMount(), + hfDownloadCacheMount(), "-e", - `HF_HOME=${HF_CACHE_CONTAINER_DIR}`, + `HF_HOME=${HF_DOWNLOAD_CACHE_CONTAINER_DIR}`, ...buildHfTokenDockerArgs(), profile.image, "download", @@ -437,10 +481,7 @@ export function buildVllmRunArgs( ): string[] { const image = validateDockerArg(profile.image, "vLLM image"); const containerName = validateDockerArg(profile.containerName, "vLLM container name"); - const safeRunFlags = validateDockerArgs( - [...runFlags, ...buildHfTokenDockerArgs(env)], - "vLLM docker run flags", - ); + const safeRunFlags = validateDockerArgs(runFlags, "vLLM docker run flags"); return [ "--pull=never", "--restart", @@ -468,6 +509,8 @@ export function resolveVllmRuntimeProfile(profile: VllmProfile, model: VllmModel ...profile, image: runtime.image, imageDownloadSizeBytes: runtime.imageDownloadSizeBytes, + modelDownloadSizeBytes: runtime.modelDownloadSizeBytes ?? profile.modelDownloadSizeBytes, + loadTimeoutSec: runtime.loadTimeoutSec ?? profile.loadTimeoutSec, dockerRunFlags: [...profile.dockerRunFlags, ...extraRunArgs], buildDockerRunFlags: profile.buildDockerRunFlags ? () => [...profile.buildDockerRunFlags!(), ...extraRunArgs] @@ -548,12 +591,8 @@ function startContainer( const resolvedFlags = profile.buildDockerRunFlags ? profile.buildDockerRunFlags() : profile.dockerRunFlags; - // Forward HF_TOKEN/HUGGING_FACE_HUB_TOKEN so the long-lived `vllm serve` - // pull can authenticate against gated repos when the model weights are - // not already in the mounted cache. The runner allowlist strips the - // token from the docker subprocess env by default, so we have to put it - // back via the `env:` option; the docker argv only carries `-e KEY` so - // the value stays out of /proc//cmdline. + // The explicit download completed before this long-lived container starts, + // so do not retain the host Hugging Face token in the serving process. let runArgs: string[]; try { runArgs = buildVllmRunArgs(profile, model, resolvedFlags); @@ -572,7 +611,7 @@ function startContainer( }); } const result = dockerRunDetached(runArgs, { - env: buildVllmDockerEnv(buildHfTokenForwardEnv()), + env: buildVllmDockerEnv(), ignoreError: true, suppressOutput: true, }); @@ -716,6 +755,43 @@ function printImageStorageWarning( console.error(" docker info --format '{{.DockerRootDir}}'"); } +function printModelStorageWarning( + model: VllmModelDef, + probe: StorageProbeResult, + requiredBytes: bigint, + cachedBytes: bigint, + snapshotBytes: bigint, +): void { + const insufficient = probe.ok && probe.capacity.availableBytes < requiredBytes; + console.error(""); + console.error( + ` ${insufficient ? "Insufficient" : "Unable to verify"} storage for the managed vLLM model cache.`, + ); + console.error(""); + console.error(` Model: ${model.id}`); + if (cachedBytes > 0n) { + console.error( + ` Cached: ${formatStorageBytes(cachedBytes)} of ${formatStorageBytes(snapshotBytes)}`, + ); + } + console.error( + ` Available: ${ + probe.ok ? formatStorageBytes(probe.capacity.availableBytes) : `unknown (${probe.reason})` + }`, + ); + console.error(` Required: approximately ${formatStorageBytes(requiredBytes)}`); + if (probe.ok) { + console.error(` Storage: ${probe.capacity.source} (${probe.capacity.path})`); + } else if (probe.path) { + console.error(` Storage: ${probe.source ?? "filesystem"} (${probe.path})`); + } + console.error(""); + if (insufficient) console.error(" Free or expand the model-cache storage before continuing."); + console.error(" Useful diagnostics:"); + console.error(` df -h ${hostHfCacheDir()}`); + console.error(` du -sh ${hostHfCacheDir()} 2>/dev/null`); +} + async function imageStorageAccepted( profile: VllmProfile, opts: InstallVllmOptions, @@ -744,6 +820,68 @@ async function imageStorageAccepted( return isAffirmativeAnswer(await opts.promptFn(" Continue with the pull anyway? [y/N]: ")); } +async function modelStorageAccepted( + profile: VllmProfile, + model: VllmModelDef, + opts: InstallVllmOptions, + env: NodeJS.ProcessEnv = process.env, +): Promise { + if (profile.modelDownloadSizeBytes === undefined) return true; + if (!Number.isFinite(profile.modelDownloadSizeBytes) || profile.modelDownloadSizeBytes <= 0) { + throw new Error("vLLM model download size must be a positive finite byte count"); + } + const snapshotBytes = BigInt(Math.ceil(profile.modelDownloadSizeBytes)); + const snapshotDir = hfModelSnapshotDir(model); + const cachedBytes = snapshotDir ? measureDirectorySizeBytes(snapshotDir) : 0n; + if (cachedBytes >= snapshotBytes) return true; + const remainingBytes = snapshotBytes - cachedBytes; + const probe = probeHostStorage(hostHfCacheDir(), "Hugging Face cache"); + const requiredBytes = modelStorageRequirementBytes(Number(remainingBytes)); + if (probe.ok && probe.capacity.availableBytes >= requiredBytes) return true; + printModelStorageWarning(model, probe, requiredBytes, cachedBytes, snapshotBytes); + if (!probe.ok) { + console.error(" Continuing because model-cache capacity could not be verified."); + return true; + } + if (env[VLLM_STORAGE_OVERRIDE_ENV] === "1") { + console.error(` Continuing because ${VLLM_STORAGE_OVERRIDE_ENV}=1.`); + return true; + } + if (opts.nonInteractive) { + console.error( + ` Non-interactive setup stops before the guarded download. Set ${VLLM_STORAGE_OVERRIDE_ENV}=1 to override.`, + ); + return false; + } + return isAffirmativeAnswer( + await opts.promptFn(" Continue with the model download anyway? [y/N]: "), + ); +} + +function ensureHfCacheDir(): { ok: true } | { ok: false; reason: string } { + const cacheDir = hostHfCacheDir(); + try { + fs.mkdirSync(cacheDir, { recursive: true }); + } catch (err) { + return { + ok: false, + reason: `could not create Hugging Face cache directory ${cacheDir}: ${(err as Error).message}`, + }; + } + const unwritablePath = findUnwritableTreePath(cacheDir); + if (unwritablePath) { + const identity = hostUserIdentity() ?? "$(id -u):$(id -g)"; + return { + ok: false, + reason: + `Hugging Face cache path ${unwritablePath} is not writable by host user ${identity}. ` + + "It may have been created by an earlier root-run downloader; NemoClaw did not modify it. " + + `Repair ownership, then retry: sudo chown -R ${identity} ${shellQuote(cacheDir)}`, + }; + } + return { ok: true }; +} + interface InstallVllmOptions { hasImage: boolean; nonInteractive: boolean; @@ -852,17 +990,37 @@ export async function installVllm( return { ok: false }; } + // Guard the host filesystem before an image pull or model-download + // container can start. The cache path itself is created only after both + // storage decisions pass, so Docker never creates it as root. + if (!(await modelStorageAccepted(runtimeProfile, model, opts))) { + return { ok: false }; + } + const hasImage = imageIsCached(runtimeProfile); if (!hasImage && !(await imageStorageAccepted(runtimeProfile, opts))) { return { ok: false }; } + const cacheDir = ensureHfCacheDir(); + if (!cacheDir.ok) { + console.error(` vLLM install failed: ${cacheDir.reason}`); + return { ok: false }; + } + const pull = await pullImage(runtimeProfile); if (!pull.ok) { console.error(` vLLM install failed: ${String(pull.reason)}`); return { ok: false }; } + // A cold image pull can consume the same host filesystem that backs the + // Hugging Face cache. Re-probe after the pull so two independently passing + // capacity checks cannot overcommit shared storage before `hf download`. + if (!hasImage && !(await modelStorageAccepted(runtimeProfile, model, opts))) { + return { ok: false }; + } + const modelDownload = await downloadModel(runtimeProfile, model); if (!modelDownload.ok) { console.error(` vLLM install failed: ${String(modelDownload.reason)}`); diff --git a/src/lib/onboard/setup-nim-vllm.test.ts b/src/lib/onboard/setup-nim-vllm.test.ts index 80ec767f98f..9f45d705e73 100644 --- a/src/lib/onboard/setup-nim-vllm.test.ts +++ b/src/lib/onboard/setup-nim-vllm.test.ts @@ -88,6 +88,20 @@ describe("setupNim vLLM route containment", () => { await expect(handler(selection)).rejects.toThrow("exit 1"); expect(validate).not.toHaveBeenCalled(); + expect(console.error).toHaveBeenCalledWith( + " Detected vLLM model 'served/model' does not match the shared gateway route 'required/model'.", + ); + expect(console.error).toHaveBeenCalledWith( + " To install 'required/model', stop the existing vLLM server on localhost:8000, then rerun the original install/onboard command.", + ); + expect(console.error).toHaveBeenCalledWith( + " To keep 'served/model' instead, start detailed setup:", + ); + expect(console.error).toHaveBeenCalledWith( + " unset NEMOCLAW_PROVIDER NEMOCLAW_MODEL NEMOCLAW_VLLM_MODEL", + ); + expect(console.error).toHaveBeenCalledWith(" nemoclaw onboard --fresh"); + expect(console.error).toHaveBeenCalledWith(" Then select Local vLLM when prompted."); }); it("warns on DGX Spark identified via GPU name (firmware-unknown GB10 host)", async () => { diff --git a/src/lib/onboard/setup-nim-vllm.ts b/src/lib/onboard/setup-nim-vllm.ts index 2d586040a79..ead86ae7133 100644 --- a/src/lib/onboard/setup-nim-vllm.ts +++ b/src/lib/onboard/setup-nim-vllm.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { cliName } from "./branding"; import type { SetupNimSelectionResult, SetupNimSelectionState } from "./setup-nim-flow"; type VllmModelEntry = { @@ -194,6 +195,13 @@ export function createSetupNimVllmHandler( console.error( ` Detected vLLM model '${detectedModel}' does not match the shared gateway route '${requiredModel}'.`, ); + console.error( + ` To install '${requiredModel}', stop the existing vLLM server on localhost:${deps.VLLM_PORT}, then rerun the original install/onboard command.`, + ); + console.error(` To keep '${detectedModel}' instead, start detailed setup:`); + console.error(" unset NEMOCLAW_PROVIDER NEMOCLAW_MODEL NEMOCLAW_VLLM_MODEL"); + console.error(` ${cliName()} onboard --fresh`); + console.error(" Then select Local vLLM when prompted."); deps.exitProcess(1); } state.model = detectedModel; diff --git a/test/inference-options-docs.test.ts b/test/inference-options-docs.test.ts index 3663edabc68..ef6bdd29248 100644 --- a/test/inference-options-docs.test.ts +++ b/test/inference-options-docs.test.ts @@ -324,6 +324,16 @@ describe("inference setup navigation", () => { } }); + it("documents the canonical Station Ultra recipe and DeepSeek demo override", () => { + const markdown = fs.readFileSync(vllmSetupPath, "utf8"); + + expect(markdown).toContain("--station-deepseek"); + expect(markdown).toContain("memory/stack ulimits"); + expect(markdown).toContain("MTP speculative decoding"); + expect(markdown).toContain("model-cache storage is insufficient"); + expect(markdown).toContain("not retained by the long-lived vLLM container"); + }); + it("keeps tool-calling remediation canonical in troubleshooting", () => { const markdown = fs.readFileSync(troubleshootingPath, "utf8"); const start = markdown.indexOf("### Tool calls appear as assistant text"); diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index 716c1e1d7ee..77b24278f96 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -35,9 +35,9 @@ platform = sys.argv[4] script = r''' source "$INSTALLER_UNDER_TEST" >/dev/null detect_express_platform() { printf "$EXPRESS_PLATFORM"; } -NON_INTERACTIVE="" -NEMOCLAW_PROVIDER="" -NEMOCLAW_NO_EXPRESS="" +NON_INTERACTIVE="\${NON_INTERACTIVE:-}" +NEMOCLAW_PROVIDER="\${NEMOCLAW_PROVIDER:-}" +NEMOCLAW_NO_EXPRESS="\${NEMOCLAW_NO_EXPRESS:-}" maybe_offer_express_install printf "RESULT NON_INTERACTIVE=%s SUDO_MODE=%s PROVIDER=%s MODEL=%s VLLM_MODEL=%s POLICY=%s YES=%s SANDBOX=%s\\n" \\ "\${NON_INTERACTIVE:-}" "\${NEMOCLAW_NON_INTERACTIVE_SUDO_MODE:-}" "\${NEMOCLAW_PROVIDER:-}" "\${NEMOCLAW_MODEL:-}" \\ @@ -145,6 +145,19 @@ detect_express_platform ); } + it("parses and documents the DGX Station DeepSeek override", () => { + const result = spawnSync("bash", [INSTALLER_PAYLOAD, "--station-deepseek", "--help"], { + cwd: path.join(import.meta.dirname, ".."), + encoding: "utf-8", + }); + const output = `${result.stdout}${result.stderr}`; + + expect(result.status, output).toBe(0); + expect(output).toMatch( + /--station-deepseek\s+Use DeepSeek V4 Flash for DGX Station express install/, + ); + }); + it("offers express install when curl-piped stdin still has a controlling TTY", () => { const result = runExpressPromptWithTty("y\n", "pipe"); const output = `${result.stdout}${result.stderr}`; @@ -207,10 +220,77 @@ detect_express_platform expect(output).toMatch(/approximately 352 GB model/); expect(output).toMatch(/Using express install for DGX Station/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL=nemotron-3-ultra-550b-a55b POLICY=suggested YES=1 SANDBOX=my-assistant/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b POLICY=suggested YES=1 SANDBOX=my-assistant/, ); }); + it("uses DeepSeek V4 Flash for the Station demo override with one confirmation", () => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { + STATION_DEEPSEEK: "1", + }); + const output = `${result.stdout}${result.stderr}`; + expect(result.status, output).toBe(0); + expect(output).toMatch( + /Express install will configure managed local vLLM with DeepSeek V4 Flash/, + ); + expect(output.match(/Run express install with these settings\?/g)).toHaveLength(1); + expect(output).toMatch(/Using express install for DGX Station/); + expect(output).toMatch( + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=deepseek-ai\/DeepSeek-V4-Flash VLLM_MODEL=deepseek-v4-flash POLICY=suggested YES=1 SANDBOX=my-assistant/, + ); + }); + + it("allows a matching explicit DeepSeek model with the Station demo override", () => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { + STATION_DEEPSEEK: "1", + NEMOCLAW_VLLM_MODEL: "deepseek-ai/DeepSeek-V4-Flash", + }); + const output = `${result.stdout}${result.stderr}`; + expect(result.status, output).toBe(0); + expect(output).toMatch(/managed local vLLM with DeepSeek V4 Flash/); + expect(output).toMatch(/MODEL=deepseek-ai\/DeepSeek-V4-Flash VLLM_MODEL=deepseek-v4-flash/); + }); + + it("rejects a conflicting explicit model with the Station demo override", () => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { + STATION_DEEPSEEK: "1", + NEMOCLAW_VLLM_MODEL: "nemotron-3-ultra-550b-a55b", + }); + const output = `${result.stdout}${result.stderr}`; + expect(result.status, output).not.toBe(0); + expect(output).toMatch( + /--station-deepseek conflicts with NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b'/, + ); + expect(output).not.toMatch(/Run express install/); + }); + + it("rejects the Station demo override on non-Station platforms", () => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Spark", { + STATION_DEEPSEEK: "1", + }); + const output = `${result.stdout}${result.stderr}`; + expect(result.status, output).not.toBe(0); + expect(output).toMatch( + /--station-deepseek requires a detected DGX Station \(detected: DGX Spark\)/, + ); + expect(output).not.toMatch(/Run express install/); + }); + + it.each([ + ["NEMOCLAW_NO_EXPRESS", "1", /cannot be combined with NEMOCLAW_NO_EXPRESS=1/], + ["NON_INTERACTIVE", "1", /cannot be combined with --non-interactive/], + ["NEMOCLAW_PROVIDER", "install-vllm", /conflicts with NEMOCLAW_PROVIDER=install-vllm/], + ])("rejects %s when the Station demo override would otherwise be ignored", (name, value, message) => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { + STATION_DEEPSEEK: "1", + [name]: value, + }); + const output = `${result.stdout}${result.stderr}`; + expect(result.status, output).not.toBe(0); + expect(output).toMatch(message); + expect(output).not.toMatch(/Run express install/); + }); + it("describes and preserves an explicit DGX Station model override", () => { const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { NEMOCLAW_VLLM_MODEL: "custom-station-model", @@ -234,7 +314,7 @@ detect_express_platform expect(output).toMatch(/managed local vLLM with NVIDIA Nemotron 3 Ultra 550B/); expect(output).toMatch(/approximately 352 GB model/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL=nemotron-3-ultra-550b-a55b POLICY=suggested YES=1 SANDBOX=my-assistant/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b POLICY=suggested YES=1 SANDBOX=my-assistant/, ); }); diff --git a/test/onboard-selection-vllm.test.ts b/test/onboard-selection-vllm.test.ts index 378eb542d8a..f6beebf0e6f 100644 --- a/test/onboard-selection-vllm.test.ts +++ b/test/onboard-selection-vllm.test.ts @@ -628,4 +628,82 @@ const { setupNim } = require(${onboardPath}); /NEMOCLAW_PROVIDER=install-vllm requested, but vLLM is already running on localhost:8000 — selecting the running instance\./, ); }); + + it("rejects an existing DeepSeek server when Station express requires Ultra", () => { + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-ultra-mismatch-")); + const scriptPath = path.join(tmpDir, "station-ultra-mismatch-check.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + + const script = String.raw` +const credentials = require(${credentialsPath}); +const runner = require(${runnerPath}); + +credentials.prompt = async () => { + throw new Error("Unexpected prompt in non-interactive Station express test"); +}; +credentials.ensureApiKey = async () => { + throw new Error("Unexpected ensureApiKey call in non-interactive Station express test"); +}; +runner.runCapture = (command) => { + const cmd = Array.isArray(command) ? command.join(" ") : command; + if (cmd.includes("command -v ollama")) return ""; + if (cmd.includes("127.0.0.1:11434/api/tags")) return ""; + if (cmd.includes("127.0.0.1:8000/v1/models")) { + return JSON.stringify({ data: [{ id: "deepseek-ai/DeepSeek-V4-Flash" }] }); + } + if (cmd.includes("docker images")) return ""; + return ""; +}; + +const { setupNim } = require(${onboardPath}); + +(async () => { + await setupNim({ type: "nvidia", platform: "station" }, null); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_PROVIDER: "install-vllm", + NEMOCLAW_VLLM_MODEL: "nemotron-3-ultra-550b-a55b", + NEMOCLAW_MODEL: "nvidia/nemotron-3-ultra-550b-a55b", + }, + }); + + assert.equal(result.status, 1); + assert.match( + result.stdout, + /NEMOCLAW_PROVIDER=install-vllm requested, but vLLM is already running on localhost:8000 — selecting the running instance\./, + ); + assert.match( + result.stderr, + /Detected vLLM model 'deepseek-ai\/DeepSeek-V4-Flash' does not match the shared gateway route 'nvidia\/nemotron-3-ultra-550b-a55b'\./, + ); + assert.match( + result.stderr, + /To install 'nvidia\/nemotron-3-ultra-550b-a55b', stop the existing vLLM server on localhost:8000, then rerun the original install\/onboard command\./, + ); + assert.match( + result.stderr, + /To keep 'deepseek-ai\/DeepSeek-V4-Flash' instead, start detailed setup:/, + ); + assert.match( + result.stderr, + /unset NEMOCLAW_PROVIDER NEMOCLAW_MODEL NEMOCLAW_VLLM_MODEL\s+nemoclaw onboard --fresh\s+Then select Local vLLM when prompted\./, + ); + }); }); From d348ed885c28508b819679a1b0f4bf47049b5509 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 07:27:16 -0700 Subject: [PATCH 08/21] test(inference): keep host identity assertions linear Signed-off-by: Aaron Erickson --- src/lib/inference/vllm.test.ts | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index a86f7944be0..0fdfe606d6a 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -83,6 +83,12 @@ beforeEach(() => { }); }); +function currentHostIdentity(): string | null { + const uid = process.getuid?.(); + const gid = process.getgid?.(); + return uid === undefined || gid === undefined ? null : `${String(uid)}:${String(gid)}`; +} + function mockDockerSpawnSuccess(): EventEmitter & { stdout: EventEmitter; stderr: EventEmitter; @@ -602,16 +608,11 @@ describe("installVllm model resolution", () => { "183968f87ae4cedce3039313cac1fd43d112c578", ]), ); - if (typeof process.getuid === "function" && typeof process.getgid === "function") { - expect(downloadArgs).toEqual( - expect.arrayContaining([ - "--user", - `${String(process.getuid())}:${String(process.getgid())}`, - ]), - ); - } else { - expect(downloadArgs).not.toContain("--user"); - } + const hostIdentity = currentHostIdentity(); + expect(downloadArgs.includes("--user")).toBe(hostIdentity !== null); + expect(downloadArgs).toEqual( + expect.arrayContaining(hostIdentity === null ? [] : ["--user", hostIdentity]), + ); expect(mkdirSpy).toHaveBeenCalledWith(path.join(os.homedir(), ".cache", "huggingface"), { recursive: true, }); @@ -1111,9 +1112,7 @@ describe("installVllm model resolution", () => { expect(errors).toContain("NemoClaw did not modify it"); expect(errors).toContain("sudo chown -R"); expect(errors).toContain(`'${cacheDir}'`); - if (typeof process.getuid === "function" && typeof process.getgid === "function") { - expect(errors).toContain(`${String(process.getuid())}:${String(process.getgid())}`); - } + expect(errors).toContain(currentHostIdentity() ?? "$(id -u):$(id -g)"); }); it("limits the Hugging Face token to the one-shot download container", async () => { From 3170c00ff7d4f92d092a8ec763803598a9c14007 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 09:40:15 -0700 Subject: [PATCH 09/21] feat(installer): add experimental Station Ultra profile Signed-off-by: Aaron Erickson --- docs/get-started/quickstart.mdx | 5 + docs/inference/choose-inference-provider.mdx | 7 + docs/inference/set-up-vllm.mdx | 96 +++- docs/reference/commands.mdx | 18 + docs/reference/platform-support.mdx | 4 + install.sh | 2 + scripts/install.sh | 81 ++- .../actions/sandbox/connect-vllm-preflight.ts | 10 +- .../sandbox/rebuild-env-isolation.test.ts | 3 + .../actions/sandbox/rebuild-env-isolation.ts | 3 +- .../vllm-experimental-single-user.test.ts | 469 ++++++++++++++++++ src/lib/inference/vllm-models.test.ts | 263 +++++++++- src/lib/inference/vllm-models.ts | 286 ++++++++++- src/lib/inference/vllm.test.ts | 57 ++- src/lib/inference/vllm.ts | 280 +++++++++-- src/lib/onboard/setup-nim-vllm.test.ts | 30 +- src/lib/onboard/setup-nim-vllm.ts | 4 +- src/lib/onboard/vllm-model-preflight.ts | 4 +- test/cli/dispatch-basics.test.ts | 4 +- test/generate-hermes-config.test.ts | 25 + test/inference-options-docs.test.ts | 4 +- test/install-express-prompt.test.ts | 154 +++++- test/onboard-selection-vllm.test.ts | 2 +- test/seed-hermes-dashboard-config.test.ts | 53 ++ 24 files changed, 1760 insertions(+), 104 deletions(-) create mode 100644 src/lib/inference/vllm-experimental-single-user.test.ts diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 59980dee75e..9aba60c265f 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -127,6 +127,8 @@ Use these details when your first-run path needs more control. DGX Spark, DGX Station, and Windows WSL offer an interactive express-install path that chooses a managed local inference option for the platform. On DGX Station, accepting the express prompt selects the pinned `nemotron-3-ultra-550b-a55b` managed-vLLM recipe and completes onboarding without more provider, model, policy, or sandbox-name choices. Pass `--station-deepseek` to use DeepSeek V4 Flash for a Station demo instead. + Pass `--station-experimental-single-user` to opt in to the named reclaimed-v1/offload90 Station profile within the same managed-vLLM lifecycle; this flag does not change the default Station recipe. + The experimental profile currently requires its exact qualified runtime image to be preloaded, so a fresh pull remains blocked until a repository-qualified manifest is published and proven. Refer to [Platform Support](../reference/platform-support) and [Choose an Inference Provider](../inference/learn-and-choose/choose-inference-provider) for the current platform behavior. If `nemoclaw` is not found after installation and you use nvm or fnm, open a new terminal or reload your shell profile. @@ -196,6 +198,8 @@ Use these details when your first-run path needs more control. DGX Spark uses managed vLLM with `qwen3.6-35b-a3b-nvfp4` by default. DGX Station express install explicitly selects `nemotron-3-ultra-550b-a55b` instead of the Station managed-vLLM profile default, `deepseek-v4-flash`, and discloses the approximately `352 GB` model download before confirmation. To select DeepSeek V4 Flash while retaining the one-confirmation Station express flow, run `curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-deepseek`. + To opt in to the experimental single-user Station profile, run `curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-experimental-single-user`. + That profile fails closed unless Docker already has the exact qualified local image ID `sha256:e44830d3f0197850e5a63ca895b13a047f358ff4aea15b1655abf61391bb883e`; it does not substitute another vLLM image. Unless `NEMOCLAW_POLICY_TIER` is set, express install applies policy in `suggested` mode with the `balanced` tier, including the base sandbox policy and supported package, model, web-search, and local-inference presets. Express install uses `my-assistant` as the sandbox name across all platforms unless `NEMOCLAW_SANDBOX_NAME` is set. Windows WSL selects the Windows-host Ollama setup path. @@ -204,6 +208,7 @@ Use these details when your first-run path needs more control. Express install automates the configuration but does not change DGX Station's Deferred support status. Full NemoClaw onboarding with this recipe has not completed end-to-end validation on physical DGX Station hardware. + The experimental single-user profile has prior functional qualification evidence, but this change has not completed a fresh, current-PR physical Station qualification. The installer auto-launches `nemoclaw onboard` when it can find the new binary. diff --git a/docs/inference/choose-inference-provider.mdx b/docs/inference/choose-inference-provider.mdx index a87cc050137..dd08ca002de 100644 --- a/docs/inference/choose-inference-provider.mdx +++ b/docs/inference/choose-inference-provider.mdx @@ -35,6 +35,13 @@ Use this status table to distinguish validated provider integrations from adapte | Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and DGX Station. DGX Station remains deferred until its managed-vLLM onboarding path is validated end-to-end on physical hardware. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pins runtime images to immutable digests. DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest published under `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use the matching `linux/arm64` or `linux/amd64` digest published under `nvcr.io/nvidia/vllm:26.03.post1-py3`. The DGX Station express installer selects `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` with a pinned Hugging Face revision and the multi-platform index digest published under `vllm/vllm-openai:v0.22.0`; that index resolves to the `linux/arm64` manifest on Station. Direct managed-vLLM profile defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls from `nvcr.io` require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing. | {/* provider-status:end */} + +DGX Station also has an explicitly opt-in managed-vLLM profile selected by the installer flag `--station-experimental-single-user`. +It serves `nemotron-ultra` through `vllm-local` with a `262144` context window and leaves the canonical Station recipe unchanged when the flag is absent. +This profile is experimental and currently requires the exact qualified local Docker image ID to be preloaded; a fresh pull is blocked until a repository-qualified manifest is published and proven. +Refer to [Set Up vLLM](../local-inference/set-up-vllm#opt-in-to-the-experimental-single-user-station-profile) for the runtime contract and current qualification boundary. + + ## Hosted Providers Hosted providers use provider-managed endpoints and curated model choices. diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 0e6cd29bde3..39ffde8981c 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -75,20 +75,24 @@ Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PR In an interactive run, the managed path lists the supported model registry for your host profile. Press **Enter** to use the profile default or select another validated model with its registered `vllm serve` arguments. -NemoClaw pulls the vLLM image, downloads model weights into `~/.cache/huggingface`, starts the `nemoclaw-vllm` container on `localhost:8000`, streams Hugging Face download progress, and polls `/v1/models` until the model is ready. +NemoClaw resolves the pinned vLLM image, downloads model weights into `~/.cache/huggingface`, starts the `nemoclaw-vllm` container on `localhost:8000`, streams Hugging Face download progress, and polls `/v1/models` until the model is ready. +Registry-backed recipes pull that image; a local-only profile requires its exact image to be preloaded. The container carries a NemoClaw ownership label so later onboarding runs can distinguish it from a bring-your-own server without trusting the container name alone. If an existing `nemoclaw-vllm` container does not carry that label, NemoClaw preserves it and asks you to remove or rename it before installing managed vLLM. If another vLLM server already occupies port `8000`, the managed path stops and asks you to select the existing server or stop it first. Managed profiles and model-specific recipes use immutable image digests: - DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest `sha256:9204569b17ee4c0eff75194b8e6e458479c8aee18953b5ab9cf359fcdac659e2` with a compressed layer size of `9.60 GB` under `nvcr.io/nvidia/vllm:26.05.post1-py3`. -- The DGX Station Nemotron 3 Ultra express recipe uses the multi-platform index digest `sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416` under `vllm/vllm-openai:v0.22.0`; on DGX Station, that index selects a `linux/arm64` manifest with a compressed layer size of `10.67 GB`. +- The canonical DGX Station Nemotron 3 Ultra express recipe uses the multi-platform index digest `sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416` under `vllm/vllm-openai:v0.22.0`; on DGX Station, that index selects a `linux/arm64` manifest with a compressed layer size of `10.67 GB`. It also pins Hugging Face revision `183968f87ae4cedce3039313cac1fd43d112c578` for the approximately `352.38 GB` model download. +- The opt-in DGX Station `experimental-single-user` profile requires the preloaded local image ID `sha256:e44830d3f0197850e5a63ca895b13a047f358ff4aea15b1655abf61391bb883e`. + This ID is not currently published as a pullable repository manifest, and no existing registry tag has been proven to resolve to it. + NemoClaw does not substitute the canonical Ultra image or another vLLM image when the qualified image is absent. - Generic Linux `arm64` hosts use `sha256:447995cbb57e6c7cf792cab95e9852e5f62b5fb6d2f39e030fa4eda9a54eadb4` with a compressed layer size of `9.28 GB` under `nvcr.io/nvidia/vllm:26.03.post1-py3`. - Generic Linux `amd64` hosts use `sha256:7be6c2f676c36059a494fe17254e69ae5c677535ba6191044e5fc8e42a91c773` with a compressed layer size of `8.93 GB` under `nvcr.io/nvidia/vllm:26.03.post1-py3`. Image pulls from `nvcr.io` require NGC registry login. -The Station Nemotron 3 Ultra recipe pulls its vLLM image from Docker Hub instead. +The canonical Station Nemotron 3 Ultra recipe pulls its vLLM image from Docker Hub instead. The first run duration depends on image and model size, storage performance, and network throughput. Later runs reuse the cached image and model weights. @@ -103,8 +107,9 @@ For Docker's classic image store, NemoClaw checks the absolute `DockerRootDir` r For Docker's containerd image store, it checks the top-level `root` from `/etc/containerd/config.toml`, or `/var/lib/containerd` when that configuration file is absent, and also checks `DockerRootDir` for pull staging. The check uses the location with the least available capacity. -NemoClaw recognizes an image cache hit only when Docker contains the exact platform digest. -A cache hit skips the capacity check, but the explicit watched `docker pull` still runs against that immutable digest. +For registry-backed recipes, NemoClaw recognizes an image cache hit only when Docker contains the exact platform digest. +A registry-backed cache hit skips the capacity check, but the explicit watched `docker pull` still runs against that immutable digest. +The local-only experimental profile performs no pull and verifies the qualified image ID directly. The model-download helper and the long-lived vLLM container use `--pull=never`, so an implicit pull cannot select another image. NemoClaw treats `/var/run/docker.sock` and `/run/docker.sock` as the local default Docker socket paths. @@ -149,6 +154,87 @@ The registered Ultra recipe tracks the [official DGX Station deployment guide](h NemoClaw intentionally keeps its existing bridge-networked managed-inference topology instead of importing the playbook's host-network setting. The container publishes port `8000` through Docker, so apply the firewall guidance at the top of this page. +### Opt In to the Experimental Single-User Station Profile + +The `experimental-single-user` profile extends the existing managed-vLLM and Station express lifecycle; it is not a separate launcher. +It is explicit opt-in and does not change the canonical Station express recipe above. +The canonical recipe remains the default while Nemotron-team quality validation of this experimental profile is pending. +Use it only on a detected DGX Station: + +```bash +curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-experimental-single-user +``` + +The installer maps this flag to `NEMOCLAW_VLLM_PROFILE=experimental-single-user` and rejects conflicting model, profile, extra-argument, DeepSeek, and non-Station selections. + + +The qualified runtime exists today only as local Docker image ID `sha256:e44830d3f0197850e5a63ca895b13a047f358ff4aea15b1655abf61391bb883e`. +Before running the installer, this command must return that exact ID: + +```bash +docker image inspect -f '{{.Id}}' sha256:e44830d3f0197850e5a63ca895b13a047f358ff4aea15b1655abf61391bb883e +``` + +A fresh pull or fresh machine install remains blocked until a repository-qualified manifest pin is published and proven to resolve to the qualified image. +The profile fails closed instead of silently using a different vLLM image. + + +The profile uses the reclaimed-v1/offload90 configuration with this model identity: + +| Setting | Value | +|---|---| +| Hugging Face model | `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` | +| Revision | `183968f87ae4cedce3039313cac1fd43d112c578` | +| Served model | `nemotron-ultra` | +| Maximum context | `262144` | + +Its container runs with host networking and host IPC, `16 GiB` shared memory, a `650 GiB` container memory limit, tensor parallel size `1`, and a `1200`-second startup timeout. +The installer fails closed unless exactly one GB300 is available, HBM is below `245000 MiB`, and aggregate and volatile uncorrected ECC are zero before launch; it samples the same exclusive HBM ceiling and ECC contract throughout startup. +Full workload-level HBM/ECC qualification through chat and tool calls remains part of the blocked current-PR physical Station acceptance run. + +After `vllm serve` and the pinned revision's local snapshot path, the profile supplies exactly these vLLM arguments: + +```text +--served-model-name nemotron-ultra +--host 0.0.0.0 +--port 8000 +--tensor-parallel-size 1 +--trust-remote-code +--cpu-offload-gb 90 +--cpu-offload-params experts +--kernel_config '{"enable_flashinfer_autotune": false}' +--max-num-seqs 1 +--gpu-memory-utilization 0.9 +--kv-cache-memory-bytes 4294967296 +--reasoning-parser nemotron_v3 +--enable-auto-tool-choice +--tool-call-parser qwen3_coder +``` + +The required environment is: + +```text +VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY=1 +VLLM_NVFP4_GEMM_BACKEND=flashinfer-trtllm +NEMOTRON_ULTRA_MODEL_OPT_NVFP4_MOE_RECLAIM=1 +HF_HUB_OFFLINE=1 +TRANSFORMERS_OFFLINE=1 +PYTHONHASHSEED=0 +``` + +Prefix caching remains disabled, matching the pinned Station guide. +That setting predates this experimental profile; it is not an optimization introduced by this work. +The profile does not add `--enable-prefix-caching`, does not use MTP speculative decoding, and disables FlashInfer autotuning explicitly. +It also does not use the Gen4B/autotuned result, which measured `44.88197518340701` tokens/s/user but took `2174.438460592006` seconds to start. +The selected reclaimed-v1/offload90 parent configuration measured `41.87059772268699` parser-free AIPerf tokens/s/user and `285.592754706` seconds to API readiness. +The Hermes parser overlay has functional qualification but no separate AIPerf score. + +NemoClaw routes the profile through provider `vllm-local`, model `nemotron-ultra`, and effective context length `262144` using the normal inference and Hermes lifecycle. +Generated gateway configuration and the dashboard mirror receive the same route and context; do not edit sandbox configuration files directly. +Nemotron Ultra requires the `qwen3_coder` tool-call parser, not the Hermes tool parser. + +This profile has prior benchmark and functional evidence, but it has not completed a fresh physical-DGX-Station qualification from the current PR. + DGX Station and its managed-vLLM paths remain Deferred until NemoClaw completes end-to-end validation on physical DGX Station hardware. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 3bdfc29328e..5fc39dabfc9 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -477,6 +477,23 @@ For scripted installer runs, pass explicit acceptance to the `bash` side of the curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 bash ``` +On a detected DGX Station, the installer also accepts these mutually exclusive express-install selections: + +| Installer flag | Effect | +|---|---| +| `--station-deepseek` | Select DeepSeek V4 Flash while retaining the Station express flow. | +| `--station-experimental-single-user` | Explicitly opt in to the reclaimed-v1/offload90 Nemotron Ultra profile. The canonical Station recipe remains the default when this flag is absent. | + +Pass installer arguments after `bash -s --`: + +```bash +curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-experimental-single-user +``` + +The experimental flag requires the exact qualified local Docker image ID `sha256:e44830d3f0197850e5a63ca895b13a047f358ff4aea15b1655abf61391bb883e` to be preloaded. +No pullable repository manifest is currently proven for that image, so fresh installs fail closed instead of substituting another vLLM runtime. +Refer to [Set Up vLLM](../inference/local-inference/set-up-vllm#opt-in-to-the-experimental-single-user-station-profile) for the exact runtime contract and image check. + If the installer cannot prompt for the notice in a terminal and no explicit acceptance is set, it exits before installing Node.js or the NemoClaw CLI. @@ -3249,6 +3266,7 @@ Set them before running `$$nemoclaw onboard`. | `NEMOCLAW_INSTALL_REF` | git ref | For internal installer commands: the git ref to install from. Overridden by the `--install-ref` flag. | | `NEMOCLAW_INSTALL_TAG` | release tag | For internal installer commands: the release tag to install. Defaults to the admin-promoted `lkg` tag when unset. Overridden by the `--install-tag` flag. | | `NEMOCLAW_VLLM_MODEL` | registry slug or Hugging Face model id | Selects the model the managed-vLLM install path serves. Recognised slugs: `qwen3.6-27b`, `qwen3.6-35b-a3b-nvfp4`, `nemotron-3-nano-4b`, `deepseek-v4-flash`, `nemotron-3-ultra-550b-a55b`, `deepseek-r1-distill-70b`. Unset uses the per-platform profile default. The DGX Station express installer sets `nemotron-3-ultra-550b-a55b` explicitly. Gated models (e.g. `deepseek-r1-distill-70b`) require `HF_TOKEN` or `HUGGING_FACE_HUB_TOKEN`. | +| `NEMOCLAW_VLLM_PROFILE` | profile name | Selects an explicit managed-vLLM runtime profile. `experimental-single-user` is restricted to DGX Station and selects the reclaimed-v1/offload90 recipe; leaving this variable unset preserves the existing per-model and per-platform behavior. Prefer the installer flag `--station-experimental-single-user`, which validates conflicting selections and sets the matching model and served name. | | `NEMOCLAW_VLLM_EXTRA_ARGS_JSON` | JSON array of non-blank strings | Appends advanced operator-owned tokens to the managed `vllm serve` command after NemoClaw's registry defaults. Example: `["--max-num-seqs","2"]`. Malformed JSON, non-string tokens, or blank tokens fail before Docker work starts. | | `NEMOCLAW_IGNORE_VLLM_DISK_SPACE` | exactly `1` | Allows managed vLLM to continue when verified Docker image-storage or registered model-cache capacity is insufficient; other values do not bypass either check. Inconclusive checks report diagnostics and continue automatically. | diff --git a/docs/reference/platform-support.mdx b/docs/reference/platform-support.mdx index 0131ee518b2..0a3cc899456 100644 --- a/docs/reference/platform-support.mdx +++ b/docs/reference/platform-support.mdx @@ -107,6 +107,10 @@ NemoClaw routes inference through the OpenShell gateway. Each row below is a pro | Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and DGX Station. DGX Station remains deferred until its managed-vLLM onboarding path is validated end-to-end on physical hardware. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pins runtime images to immutable digests. DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest published under `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use the matching `linux/arm64` or `linux/amd64` digest published under `nvcr.io/nvidia/vllm:26.03.post1-py3`. The DGX Station express installer selects `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` with a pinned Hugging Face revision and the multi-platform index digest published under `vllm/vllm-openai:v0.22.0`; that index resolves to the `linux/arm64` manifest on Station. Direct managed-vLLM profile defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls from `nvcr.io` require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing. | {/* provider-status-full:end */} +The DGX Station `experimental-single-user` managed-vLLM profile is available through the explicit installer flag `--station-experimental-single-user`; advanced direct onboarding may opt in with the matching `NEMOCLAW_VLLM_PROFILE` and `NEMOCLAW_VLLM_MODEL` selectors. +It does not change the default Station recipe or promote DGX Station beyond Deferred status. +The profile currently requires its exact qualified runtime image to be preloaded because no pullable repository manifest has been published and proven for that image; do not present it as a fresh-install or current-PR physical-hardware qualification. + ## Messaging Integrations NemoClaw configures messaging channels during onboarding. The OpenShell gateway runs each channel as a supervised process; NemoClaw supplies onboarding, credential delivery, and policy presets for the sandbox egress rules. diff --git a/install.sh b/install.sh index ab867b4c891..09bbd00fe1f 100755 --- a/install.sh +++ b/install.sh @@ -114,6 +114,8 @@ bootstrap_usage() { printf " Options:\n" printf " --non-interactive Skip prompts (uses env vars / defaults)\n" printf " --station-deepseek Use DeepSeek V4 Flash for DGX Station express install\n" + printf " --station-experimental-single-user\n" + printf " Use the experimental single-user DGX Station profile\n" printf " --yes-i-accept-third-party-software Accept the third-party software notice without prompting\n" printf " --fresh Discard any failed/interrupted onboarding session and start over\n" printf " --version, -v Print installer version and exit\n" diff --git a/scripts/install.sh b/scripts/install.sh index 1688cb737cb..46654d67fb1 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -636,6 +636,8 @@ usage() { printf " --yes-i-accept-third-party-software Accept the third-party software notice without prompting\n" printf " --fresh Discard any failed/interrupted onboarding session and start over\n" printf " --station-deepseek Use DeepSeek V4 Flash for DGX Station express install\n" + printf " --station-experimental-single-user\n" + printf " Use the experimental single-user DGX Station profile\n" printf " --version, -v Print installer version and exit\n" printf " --help, -h Show this help message and exit\n\n" printf " ${C_DIM}Environment:${C_RESET}\n" @@ -2665,6 +2667,8 @@ STATION_ULTRA_VLLM_MODEL="nemotron-3-ultra-550b-a55b" STATION_ULTRA_SERVED_MODEL="nvidia/nemotron-3-ultra-550b-a55b" STATION_DEEPSEEK_VLLM_MODEL="deepseek-v4-flash" STATION_DEEPSEEK_SERVED_MODEL="deepseek-ai/DeepSeek-V4-Flash" +STATION_EXPERIMENTAL_SINGLE_USER_PROFILE="experimental-single-user" +STATION_EXPERIMENTAL_SINGLE_USER_SERVED_MODEL="nemotron-ultra" normalize_station_vllm_model() { printf "%s" "${1:-}" | tr '[:upper:]' '[:lower:]' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' @@ -2698,10 +2702,67 @@ validate_station_deepseek_override() { esac } +validate_station_experimental_single_user_override() { + local platform="$1" + if [ "${STATION_EXPERIMENTAL_SINGLE_USER:-}" != "1" ]; then + return 0 + fi + if [ "${STATION_DEEPSEEK:-}" = "1" ]; then + error "--station-experimental-single-user cannot be combined with --station-deepseek." + fi + if [ "$platform" != "DGX Station" ]; then + error "--station-experimental-single-user requires a detected DGX Station (detected: ${platform:-unsupported platform})." + fi + if [ "${NEMOCLAW_NO_EXPRESS:-}" = "1" ]; then + error "--station-experimental-single-user cannot be combined with NEMOCLAW_NO_EXPRESS=1. Remove one override." + fi + if [ "${NON_INTERACTIVE:-}" = "1" ]; then + error "--station-experimental-single-user selects the DGX Station express prompt and cannot be combined with --non-interactive." + fi + if [ -n "${NEMOCLAW_PROVIDER:-}" ]; then + error "--station-experimental-single-user conflicts with NEMOCLAW_PROVIDER=${NEMOCLAW_PROVIDER}. Remove the provider override to use Station express install." + fi + + local requested_model + requested_model="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_MODEL:-}")" + case "$requested_model" in + "" | "$STATION_ULTRA_VLLM_MODEL" | "$STATION_ULTRA_SERVED_MODEL" | "nvidia/nvidia-nemotron-3-ultra-550b-a55b-nvfp4") ;; + *) + error "--station-experimental-single-user conflicts with NEMOCLAW_VLLM_MODEL='${NEMOCLAW_VLLM_MODEL}'. Remove one override or set NEMOCLAW_VLLM_MODEL=${STATION_ULTRA_VLLM_MODEL}." + ;; + esac + + local requested_served_model + requested_served_model="$(normalize_station_vllm_model "${NEMOCLAW_MODEL:-}")" + case "$requested_served_model" in + "" | "$STATION_EXPERIMENTAL_SINGLE_USER_SERVED_MODEL") ;; + *) + error "--station-experimental-single-user conflicts with NEMOCLAW_MODEL='${NEMOCLAW_MODEL}'. Remove one override or set NEMOCLAW_MODEL=${STATION_EXPERIMENTAL_SINGLE_USER_SERVED_MODEL}." + ;; + esac + + local requested_profile + requested_profile="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_PROFILE:-}")" + case "$requested_profile" in + "" | "$STATION_EXPERIMENTAL_SINGLE_USER_PROFILE") ;; + *) + error "--station-experimental-single-user conflicts with NEMOCLAW_VLLM_PROFILE='${NEMOCLAW_VLLM_PROFILE}'. Remove one override or set NEMOCLAW_VLLM_PROFILE=${STATION_EXPERIMENTAL_SINGLE_USER_PROFILE}." + ;; + esac + + if [ -n "$(printf "%s" "${NEMOCLAW_VLLM_EXTRA_ARGS_JSON:-}" | tr -d '[:space:]')" ]; then + error "--station-experimental-single-user cannot be combined with NEMOCLAW_VLLM_EXTRA_ARGS_JSON. The qualified profile supplies the exact vLLM arguments." + fi +} + configure_station_express_model() { local selected_model selected_model="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_MODEL:-}")" - if [ "${STATION_DEEPSEEK:-}" = "1" ]; then + if [ "${STATION_EXPERIMENTAL_SINGLE_USER:-}" = "1" ]; then + NEMOCLAW_VLLM_PROFILE="$STATION_EXPERIMENTAL_SINGLE_USER_PROFILE" + NEMOCLAW_VLLM_MODEL="$STATION_ULTRA_VLLM_MODEL" + NEMOCLAW_MODEL="$STATION_EXPERIMENTAL_SINGLE_USER_SERVED_MODEL" + elif [ "${STATION_DEEPSEEK:-}" = "1" ]; then NEMOCLAW_VLLM_MODEL="$STATION_DEEPSEEK_VLLM_MODEL" NEMOCLAW_MODEL="$STATION_DEEPSEEK_SERVED_MODEL" elif [ -z "$selected_model" ]; then @@ -2712,12 +2773,22 @@ configure_station_express_model() { "$STATION_ULTRA_VLLM_MODEL" | "nvidia/nvidia-nemotron-3-ultra-550b-a55b-nvfp4") NEMOCLAW_MODEL="$STATION_ULTRA_SERVED_MODEL" ;; + "$STATION_ULTRA_SERVED_MODEL") + # The served alias is useful in route output but is not a Hugging Face + # repository ID. Normalize it to the registered model slug before the + # existing managed-vLLM selector consumes it. + NEMOCLAW_VLLM_MODEL="$STATION_ULTRA_VLLM_MODEL" + NEMOCLAW_MODEL="$STATION_ULTRA_SERVED_MODEL" + ;; "$STATION_DEEPSEEK_VLLM_MODEL" | "deepseek-ai/deepseek-v4-flash") NEMOCLAW_MODEL="$STATION_DEEPSEEK_SERVED_MODEL" ;; esac fi export NEMOCLAW_VLLM_MODEL + if [ "${STATION_EXPERIMENTAL_SINGLE_USER:-}" = "1" ]; then + export NEMOCLAW_VLLM_PROFILE + fi if [ -n "${NEMOCLAW_MODEL:-}" ]; then export NEMOCLAW_MODEL fi @@ -2746,7 +2817,10 @@ describe_express_install() { sandbox_summary="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" ;; "DGX Station") - if [ "${STATION_DEEPSEEK:-}" = "1" ]; then + if [ "${STATION_EXPERIMENTAL_SINGLE_USER:-}" = "1" ]; then + inference_summary="managed local vLLM with the experimental single-user Nemotron Ultra profile" + inference_disclosure="This opt-in profile uses the qualified single-user Station runtime and model configuration." + elif [ "${STATION_DEEPSEEK:-}" = "1" ]; then inference_summary="managed local vLLM with DeepSeek V4 Flash" inference_disclosure="Managed vLLM pulls the configured Station image/model and runs a local inference container." elif [ -n "$(printf "%s" "${NEMOCLAW_VLLM_MODEL:-}" | tr -d '[:space:]')" ]; then @@ -2799,6 +2873,7 @@ maybe_offer_express_install() { local platform platform="$(detect_express_platform)" validate_station_deepseek_override "$platform" + validate_station_experimental_single_user_override "$platform" # Not on a platform we have an express recipe for — say nothing. if [ -z "$platform" ]; then return 0 @@ -2886,12 +2961,14 @@ main() { ACCEPT_THIRD_PARTY_SOFTWARE="" FRESH="" STATION_DEEPSEEK="" + STATION_EXPERIMENTAL_SINGLE_USER="" for arg in "$@"; do case "$arg" in --non-interactive) NON_INTERACTIVE=1 ;; --yes-i-accept-third-party-software) ACCEPT_THIRD_PARTY_SOFTWARE=1 ;; --fresh) FRESH=1 ;; --station-deepseek) STATION_DEEPSEEK=1 ;; + --station-experimental-single-user) STATION_EXPERIMENTAL_SINGLE_USER=1 ;; --version | -v) local version_suffix version_suffix="$(installer_version_for_display)" diff --git a/src/lib/actions/sandbox/connect-vllm-preflight.ts b/src/lib/actions/sandbox/connect-vllm-preflight.ts index 811f91fc483..77bbe92fcfa 100644 --- a/src/lib/actions/sandbox/connect-vllm-preflight.ts +++ b/src/lib/actions/sandbox/connect-vllm-preflight.ts @@ -6,17 +6,17 @@ import { preflightVllmModelEnv } from "../../inference/vllm-models"; // Managed-vLLM env vars only steer the express-vLLM install path, but users // often re-export them in the same shell they later run `connect` in. Run the -// installer's validators up-front so typos, malformed extra args, or a gated -// model with no `HF_TOKEN` fail fast on the host — before any sandbox readiness -// probe, inference-route reset, or SSH attach — instead of being silently -// ignored. +// installer's validators up-front so model/profile conflicts, malformed extra +// args, or a gated model with no `HF_TOKEN` fail fast on the host — before any +// sandbox readiness probe, inference-route reset, or SSH attach — instead of +// being silently ignored. export function preflightVllmModelEnvOrExit(): void { const result = preflightVllmModelEnv(); if (result.ok) return; console.error(""); console.error(` Error: ${result.message}`); console.error( - ` Hint: NEMOCLAW_VLLM_MODEL is consumed by the managed-vLLM install path, and NEMOCLAW_VLLM_EXTRA_ARGS_JSON is consumed there too; neither is used by \`${CLI_NAME} connect\`.`, + ` Hint: NEMOCLAW_VLLM_MODEL, NEMOCLAW_VLLM_PROFILE, and NEMOCLAW_VLLM_EXTRA_ARGS_JSON are consumed by the managed-vLLM install path; they are not used by \`${CLI_NAME} connect\`.`, ); console.error( " Unset the managed-vLLM env var before reconnecting, or fix the value and re-run the install path that serves the model.", diff --git a/src/lib/actions/sandbox/rebuild-env-isolation.test.ts b/src/lib/actions/sandbox/rebuild-env-isolation.test.ts index 746715bc63a..d432fba0a76 100644 --- a/src/lib/actions/sandbox/rebuild-env-isolation.test.ts +++ b/src/lib/actions/sandbox/rebuild-env-isolation.test.ts @@ -52,6 +52,7 @@ describe("AMBIENT_RECREATE_ENV_VARS contract PRA-4 (#5735)", () => { "NEMOCLAW_PREFERRED_API", "NEMOCLAW_REASONING", "NEMOCLAW_VLLM_MODEL", + "NEMOCLAW_VLLM_PROFILE", "NEMOCLAW_VLLM_EXTRA_ARGS_JSON", "NEMOCLAW_FROM_DOCKERFILE", "NEMOCLAW_WEB_SEARCH_PROVIDER", @@ -122,6 +123,7 @@ describe("isolateAmbientRecreateEnv", () => { NEMOCLAW_PREFERRED_API: "openai-responses", NEMOCLAW_REASONING: "false", NEMOCLAW_VLLM_MODEL: "ambient-vllm-model", + NEMOCLAW_VLLM_PROFILE: "experimental-single-user", NEMOCLAW_VLLM_EXTRA_ARGS_JSON: '{"ambient":true}', NEMOCLAW_FROM_DOCKERFILE: "/tmp/unrelated.Dockerfile", NEMOCLAW_WEB_SEARCH_PROVIDER: "tavily", @@ -152,6 +154,7 @@ describe("isolateAmbientRecreateEnv", () => { expect(env.NEMOCLAW_PREFERRED_API).toBe("openai-responses"); expect(env.NEMOCLAW_REASONING).toBe("false"); expect(env.NEMOCLAW_VLLM_MODEL).toBe("ambient-vllm-model"); + expect(env.NEMOCLAW_VLLM_PROFILE).toBe("experimental-single-user"); expect(env.NEMOCLAW_VLLM_EXTRA_ARGS_JSON).toBe('{"ambient":true}'); expect(env.NEMOCLAW_FROM_DOCKERFILE).toBe("/tmp/unrelated.Dockerfile"); expect(env.NEMOCLAW_WEB_SEARCH_PROVIDER).toBe("tavily"); diff --git a/src/lib/actions/sandbox/rebuild-env-isolation.ts b/src/lib/actions/sandbox/rebuild-env-isolation.ts index b0d456f7a25..a6d486f0e4c 100644 --- a/src/lib/actions/sandbox/rebuild-env-isolation.ts +++ b/src/lib/actions/sandbox/rebuild-env-isolation.ts @@ -22,7 +22,7 @@ // → src/lib/onboard/providers.ts (hosted model aliases) // - NEMOCLAW_PREFERRED_API → src/lib/onboard/setup-nim-selection.ts // - NEMOCLAW_REASONING → src/lib/onboard/reasoning-mode.ts -// - NEMOCLAW_VLLM_MODEL / NEMOCLAW_VLLM_EXTRA_ARGS_JSON +// - NEMOCLAW_VLLM_MODEL / NEMOCLAW_VLLM_PROFILE / NEMOCLAW_VLLM_EXTRA_ARGS_JSON // → src/lib/onboard/setup-nim-vllm.ts // - NEMOCLAW_FROM_DOCKERFILE → src/lib/onboard/entry-options.ts // - NEMOCLAW_POLICY_TIER / NEMOCLAW_POLICY_MODE / NEMOCLAW_POLICY_PRESETS @@ -48,6 +48,7 @@ export const AMBIENT_RECREATE_ENV_VARS = [ "NEMOCLAW_PREFERRED_API", "NEMOCLAW_REASONING", "NEMOCLAW_VLLM_MODEL", + "NEMOCLAW_VLLM_PROFILE", "NEMOCLAW_VLLM_EXTRA_ARGS_JSON", "NEMOCLAW_FROM_DOCKERFILE", "NEMOCLAW_WEB_SEARCH_PROVIDER", diff --git a/src/lib/inference/vllm-experimental-single-user.test.ts b/src/lib/inference/vllm-experimental-single-user.test.ts new file mode 100644 index 00000000000..7cbdb9b13e4 --- /dev/null +++ b/src/lib/inference/vllm-experimental-single-user.test.ts @@ -0,0 +1,469 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { EventEmitter } from "node:events"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + dockerCapture: vi.fn(), + dockerForceRm: vi.fn(), + dockerImageInspectFormat: vi.fn(), + dockerPullWithProgressWatchdog: vi.fn(), + dockerRunDetached: vi.fn(), + dockerSpawn: vi.fn(), + dockerStop: vi.fn(), + findUnwritableTreePath: vi.fn(), + getGpuIndicesByName: vi.fn<(_pattern: RegExp) => number[]>(() => []), + measureDirectorySizeBytes: vi.fn(), + probeDockerStorage: vi.fn(), + probeHostStorage: vi.fn(), + runCapture: vi.fn(), +})); + +vi.mock("../runner", () => ({ + runCapture: mocks.runCapture, +})); + +vi.mock("../adapters/docker", () => ({ + dockerCapture: mocks.dockerCapture, + dockerForceRm: mocks.dockerForceRm, + dockerImageInspectFormat: mocks.dockerImageInspectFormat, + dockerPullWithProgressWatchdog: mocks.dockerPullWithProgressWatchdog, + dockerRunDetached: mocks.dockerRunDetached, + dockerSpawn: mocks.dockerSpawn, + dockerStop: mocks.dockerStop, +})); + +vi.mock("./nim", () => ({ + getGpuIndicesByName: mocks.getGpuIndicesByName, +})); + +vi.mock("./vllm-storage", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + findUnwritableTreePath: mocks.findUnwritableTreePath, + measureDirectorySizeBytes: mocks.measureDirectorySizeBytes, + probeDockerStorage: mocks.probeDockerStorage, + probeHostStorage: mocks.probeHostStorage, + }; +}); + +import { + buildVllmRunArgs, + detectVllmProfile, + installVllm, + NEMOCLAW_VLLM_CONTAINER_NAME, + NEMOCLAW_VLLM_MANAGED_LABEL, + resolveVllmRuntimeProfile, +} from "./vllm"; +import { + buildVllmServeCommand, + EXPERIMENTAL_SINGLE_USER_PROFILE, + NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE, + resolveVllmModelProfile, + VLLM_MODELS, +} from "./vllm-models"; + +beforeEach(() => { + mocks.dockerImageInspectFormat.mockReturnValue(""); + mocks.findUnwritableTreePath.mockReturnValue(null); + mocks.measureDirectorySizeBytes.mockReturnValue(0n); + mocks.probeDockerStorage.mockReturnValue({ + ok: true, + capacity: { availableBytes: 1_000_000_000_000n, path: "/docker", source: "Docker" }, + }); + mocks.probeHostStorage.mockReturnValue({ + ok: true, + capacity: { + availableBytes: 1_000_000_000_000n, + path: path.join(os.homedir(), ".cache", "huggingface"), + source: "Hugging Face cache", + }, + }); +}); + +function mockDockerSpawnSuccess(): EventEmitter & { + stdout: EventEmitter; + stderr: EventEmitter; +} { + const proc = new EventEmitter() as EventEmitter & { + stdout: EventEmitter; + stderr: EventEmitter; + }; + proc.stdout = new EventEmitter(); + proc.stderr = new EventEmitter(); + process.nextTick(() => proc.emit("exit", 0)); + return proc; +} + +function mockSuccessfulVllmInstall( + containerName: string, + endpoints: { + healthStatus?: string; + modelsResponse?: string; + qualificationSamples?: readonly string[]; + } = {}, +): void { + const qualificationSamples = [...(endpoints.qualificationSamples ?? ["1000, 0, 0"])]; + let lastQualificationSample = qualificationSamples.at(-1) ?? ""; + mocks.runCapture.mockImplementation((cmd: readonly string[]) => { + if (cmd[0] === "sh") return "/usr/bin/tool\n"; + if (cmd[0] === "nvidia-smi") { + const sample = qualificationSamples.shift() ?? lastQualificationSample; + lastQualificationSample = sample; + return sample; + } + if (cmd[0] !== "curl") return ""; + const url = cmd.at(-1) ?? ""; + return url.endsWith("/health") + ? (endpoints.healthStatus ?? "") + : (endpoints.modelsResponse ?? '{"data":[]}'); + }); + mocks.dockerPullWithProgressWatchdog.mockResolvedValue({ + status: 0, + signal: null, + output: "", + timedOut: false, + timeoutKind: null, + }); + mocks.dockerSpawn.mockReturnValue(mockDockerSpawnSuccess()); + mocks.dockerRunDetached.mockReturnValue({ status: 0, stdout: "", stderr: "", error: null }); + const ownershipResponses: (() => string)[] = [() => "", () => ""]; + mocks.dockerCapture.mockImplementation((args: readonly string[]) => { + if (args[0] === "container") return (ownershipResponses.shift() ?? (() => ""))(); + if (args[0] === "ps") return `${containerName}\n`; + return ""; + }); +} + +describe("experimental single-user vLLM runtime profile", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("resolves through the existing Station runtime with exact generated Docker arguments", () => { + mocks.getGpuIndicesByName.mockReturnValue([0]); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + const ultra = VLLM_MODELS.find((model) => model.envValue === "nemotron-3-ultra-550b-a55b"); + + expect(profile).not.toBeNull(); + expect(ultra).toBeDefined(); + const effective = resolveVllmModelProfile(ultra!, "station", { + NEMOCLAW_VLLM_PROFILE: EXPERIMENTAL_SINGLE_USER_PROFILE, + }); + const runtime = resolveVllmRuntimeProfile(profile!, effective.model); + const flags = runtime.buildDockerRunFlags!(); + + expect(runtime).toEqual( + expect.objectContaining({ + image: NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, + loadTimeoutSec: 1200, + networkMode: "host", + imagePullPolicy: "local-only", + expectedImageId: NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, + qualification: { + gpuNameIncludes: "GB300", + gpuCount: 1, + hbmSafetyCeilingMiB: 245000, + requireZeroUncorrectedEcc: true, + }, + }), + ); + expect(flags).toEqual([ + "--gpus", + "device=0", + "--ipc=host", + "-v", + `${path.join(os.homedir(), ".cache", "huggingface")}:/root/.cache/huggingface`, + "-e", + "HF_HOME=/root/.cache/huggingface", + "--network", + "host", + "--shm-size", + "16g", + "--memory", + "650g", + "-e", + "VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY=1", + "-e", + "VLLM_NVFP4_GEMM_BACKEND=flashinfer-trtllm", + "-e", + "NEMOTRON_ULTRA_MODEL_OPT_NVFP4_MOE_RECLAIM=1", + "-e", + "HF_HUB_OFFLINE=1", + "-e", + "TRANSFORMERS_OFFLINE=1", + "-e", + "PYTHONHASHSEED=0", + ]); + + expect(buildVllmRunArgs(runtime, effective.model, flags, {} as NodeJS.ProcessEnv)).toEqual([ + "--pull=never", + "--restart", + "unless-stopped", + ...flags, + "--label", + `${NEMOCLAW_VLLM_MANAGED_LABEL}=true`, + "--name", + NEMOCLAW_VLLM_CONTAINER_NAME, + "--entrypoint", + "/bin/bash", + NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, + "-lc", + buildVllmServeCommand(effective.model, {} as NodeJS.ProcessEnv), + ]); + }); +}); + +describe("experimental single-user vLLM install", () => { + let errSpy: ReturnType; + let mkdirSpy: ReturnType; + let stdoutWrite: ReturnType; + const originalEnv = { ...process.env }; + + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(console, "log").mockImplementation(() => {}); + errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + mkdirSpy = vi.spyOn(fs, "mkdirSync").mockImplementation(() => undefined); + stdoutWrite = vi.spyOn(process.stdout, "write").mockImplementation(() => true); + process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + process.env.NEMOCLAW_VLLM_PROFILE = EXPERIMENTAL_SINGLE_USER_PROFILE; + delete process.env.NEMOCLAW_VLLM_EXTRA_ARGS_JSON; + delete process.env.NEMOCLAW_IGNORE_VLLM_DISK_SPACE; + delete process.env.HF_TOKEN; + delete process.env.HUGGING_FACE_HUB_TOKEN; + }); + + afterEach(() => { + vi.restoreAllMocks(); + mkdirSpy.mockRestore(); + stdoutWrite.mockRestore(); + process.env = { ...originalEnv }; + }); + + it("fails closed before downloads when the qualified image is absent", async () => { + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + const beforeInstall = vi.fn(); + mockSuccessfulVllmInstall(profile.containerName); + mocks.dockerImageInspectFormat.mockReturnValue(""); + + const result = await installVllm(profile, { + hasImage: false, + nonInteractive: true, + promptFn: vi.fn(), + beforeInstall, + }); + + expect(result).toEqual({ ok: false }); + expect(beforeInstall).toHaveBeenCalledWith("nemotron-ultra"); + expect(mocks.probeHostStorage).not.toHaveBeenCalled(); + expect(mocks.probeDockerStorage).not.toHaveBeenCalled(); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); + const errors = errSpy.mock.calls.map((call: unknown[]) => String(call[0])).join("\n"); + expect(errors).toContain("qualified experimental runtime image is not published"); + expect(errors).toContain(NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId); + expect(errors).toContain("No fallback image was substituted"); + }); + + it("runs the exact preloaded profile and validates both readiness endpoints", async () => { + mocks.getGpuIndicesByName.mockReturnValue([0]); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + const beforeInstall = vi.fn(); + mockSuccessfulVllmInstall(profile.containerName, { + healthStatus: "200", + modelsResponse: '{"data":[{"id":"nemotron-ultra","max_model_len":262144}]}', + }); + mocks.dockerImageInspectFormat.mockReturnValue( + NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, + ); + + const result = await installVllm(profile, { + hasImage: false, + nonInteractive: true, + promptFn: vi.fn(), + beforeInstall, + }); + + expect(result).toEqual({ ok: true }); + expect(beforeInstall).toHaveBeenCalledWith("nemotron-ultra"); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + const curlCalls = mocks.runCapture.mock.calls + .map((call) => call[0] as string[]) + .filter((args) => args[0] === "curl"); + expect(curlCalls.map((args) => args.at(-1))).toEqual([ + "http://127.0.0.1:8000/health", + "http://127.0.0.1:8000/v1/models", + ]); + expect(curlCalls[0]).toEqual(expect.arrayContaining(["-o", "/dev/null", "-w", "%{http_code}"])); + const telemetryCalls = mocks.runCapture.mock.calls + .map((call) => call[0] as string[]) + .filter((args) => args[0] === "nvidia-smi"); + expect(telemetryCalls).toHaveLength(2); + expect(telemetryCalls[0]).toEqual([ + "nvidia-smi", + "--id=0", + "--query-gpu=memory.used,ecc.errors.uncorrected.aggregate.total,ecc.errors.uncorrected.volatile.total", + "--format=csv,noheader,nounits", + ]); + + const [runArgs] = mocks.dockerRunDetached.mock.calls[0] as [string[]]; + expect(runArgs).not.toContain("-p"); + expect(runArgs).toEqual( + expect.arrayContaining([ + "--network", + "host", + "--ipc=host", + "--shm-size", + "16g", + "--memory", + "650g", + NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, + ]), + ); + const serveCommand = runArgs.at(-1) ?? ""; + expect(serveCommand).toContain( + "vllm serve /root/.cache/huggingface/hub/models--nvidia--NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4/snapshots/183968f87ae4cedce3039313cac1fd43d112c578", + ); + expect(serveCommand).toContain("--served-model-name nemotron-ultra"); + expect(serveCommand).not.toContain("--enable-prefix-caching"); + expect(serveCommand).not.toContain("--speculative-config"); + }); + + it.each([ + ["non-200 health", "503", '{"data":[{"id":"nemotron-ultra","max_model_len":262144}]}'], + ["wrong alias", "200", '{"data":[{"id":"other-model","max_model_len":262144}]}'], + [ + "extra model", + "200", + '{"data":[{"id":"nemotron-ultra","max_model_len":262144},{"id":"other","max_model_len":262144}]}', + ], + ["wrong context", "200", '{"data":[{"id":"nemotron-ultra","max_model_len":131072}]}'], + ["missing context", "200", '{"data":[{"id":"nemotron-ultra"}]}'], + ["malformed catalog", "200", "not-json"], + ])("does not accept readiness with %s", async (_name, healthStatus, modelsResponse) => { + mocks.getGpuIndicesByName.mockReturnValue([0]); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName, { healthStatus, modelsResponse }); + mocks.dockerImageInspectFormat.mockReturnValue( + NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, + ); + mocks.dockerCapture.mockImplementation(() => ""); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: false }); + expect(mocks.dockerRunDetached).toHaveBeenCalledTimes(1); + expect(mocks.dockerStop).toHaveBeenCalledWith( + profile.containerName, + expect.objectContaining({ ignoreError: true, suppressOutput: true }), + ); + }); + + it("rejects an image identity mismatch before storage or downloads", async () => { + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName); + mocks.dockerImageInspectFormat.mockReturnValue("sha256:not-qualified"); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: false }); + expect(mocks.probeHostStorage).not.toHaveBeenCalled(); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(errSpy).toHaveBeenCalledWith( + expect.stringContaining("qualified image identity mismatch"), + ); + }); + + it("fails before downloads when the qualified Station GPU is unavailable", async () => { + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName); + mocks.dockerImageInspectFormat.mockReturnValue( + NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, + ); + mocks.getGpuIndicesByName.mockReturnValue([]); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: false }); + expect(mocks.probeHostStorage).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(errSpy).toHaveBeenCalledWith( + expect.stringContaining("requires exactly 1 GPU matching 'GB300'; detected 0"), + ); + }); + + it.each([ + ["HBM ceiling", "245000, 0, 0", "qualified HBM safety ceiling exceeded"], + ["aggregate ECC", "1000, 1, 0", "uncorrected ECC is nonzero"], + ["volatile ECC", "1000, 0, 1", "uncorrected ECC is nonzero"], + ["malformed telemetry", "N/A, 0, 0", "telemetry is unavailable or malformed"], + ])("fails before downloads when qualified %s preflight fails", async (_name, sample, message) => { + mocks.getGpuIndicesByName.mockReturnValue([0]); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName, { qualificationSamples: [sample] }); + mocks.dockerImageInspectFormat.mockReturnValue( + NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, + ); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: false }); + expect(mocks.probeHostStorage).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); + expect(errSpy).toHaveBeenCalledWith(expect.stringContaining(message)); + }); + + it("stops the managed container when startup HBM reaches the exclusive ceiling", async () => { + mocks.getGpuIndicesByName.mockReturnValue([0]); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName, { + healthStatus: "200", + modelsResponse: '{"data":[{"id":"nemotron-ultra","max_model_len":262144}]}', + qualificationSamples: ["1000, 0, 0", "245000, 0, 0"], + }); + mocks.dockerImageInspectFormat.mockReturnValue( + NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, + ); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: false }); + expect(mocks.dockerRunDetached).toHaveBeenCalledTimes(1); + expect(mocks.dockerStop).toHaveBeenCalledWith( + profile.containerName, + expect.objectContaining({ ignoreError: true, suppressOutput: true }), + ); + expect(errSpy).toHaveBeenCalledWith( + expect.stringContaining("qualified HBM safety ceiling exceeded"), + ); + }); +}); diff --git a/src/lib/inference/vllm-models.test.ts b/src/lib/inference/vllm-models.test.ts index 0b3e81bbe38..1960f1667ef 100644 --- a/src/lib/inference/vllm-models.test.ts +++ b/src/lib/inference/vllm-models.test.ts @@ -7,12 +7,17 @@ import { assertGatedModelAccess, buildVllmServeCommand, DEFAULT_VLLM_MODEL, + EXPERIMENTAL_SINGLE_USER_PROFILE, modelsForPlatform, + NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE, parseVllmExtraServeArgs, preflightVllmModelEnv, + resolveVllmModelProfile, selectVllmModelFromEnv, + selectVllmModelProfileFromEnv, VLLM_EXTRA_ARGS_ENV, VLLM_MODELS, + VLLM_PROFILE_ENV, } from "./vllm-models"; describe("vllm model registry", () => { @@ -73,26 +78,200 @@ describe("vllm model registry", () => { }); const cmd = buildVllmServeCommand(ultra!); - expect(cmd).toContain("vllm serve nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4"); - expect(cmd).not.toContain("pip install"); - expect(cmd).toContain("--revision 183968f87ae4cedce3039313cac1fd43d112c578"); - expect(cmd).toContain("--served-model-name nvidia/nemotron-3-ultra-550b-a55b"); - expect(cmd).toContain("--cpu-offload-gb 150 --cpu-offload-params experts"); - expect(cmd).toContain(`--kernel_config '{"enable_flashinfer_autotune": false}'`); - expect(cmd).toContain( - `--speculative-config '{"method":"nemotron_h_mtp","num_speculative_tokens":3}'`, + expect(cmd).toBe( + [ + "export VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY=1", + "&& export VLLM_NVFP4_GEMM_BACKEND=flashinfer-trtllm", + "&& vllm serve nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", + "--tensor-parallel-size 1", + "--pipeline-parallel-size 1", + "--data-parallel-size 1", + "--port 8000", + "--trust-remote-code", + "--max-model-len 262144", + "--revision 183968f87ae4cedce3039313cac1fd43d112c578", + "--served-model-name nvidia/nemotron-3-ultra-550b-a55b", + "--host 0.0.0.0", + "--cpu-offload-gb 150", + "--cpu-offload-params experts", + `--kernel_config '{"enable_flashinfer_autotune": false}'`, + `--speculative-config '{"method":"nemotron_h_mtp","num_speculative_tokens":3}'`, + "--max-num-seqs 256", + "--gpu-memory-utilization 0.9", + "--reasoning-parser nemotron_v3", + "--enable-auto-tool-choice", + "--tool-call-parser qwen3_coder", + `--default-chat-template-kwargs '{"enable_thinking":true,"force_nonempty_content":true}'`, + ].join(" "), ); - expect(cmd).toContain("--max-num-seqs 256 --gpu-memory-utilization 0.9"); - expect(cmd).toContain("--reasoning-parser nemotron_v3"); - expect(cmd).toContain("--enable-auto-tool-choice --tool-call-parser qwen3_coder"); - expect(cmd).toContain( - `--default-chat-template-kwargs '{"enable_thinking":true,"force_nonempty_content":true}'`, + }); + + it("keeps the qualified single-user recipe explicit, exact, and separate from the canonical Ultra recipe", () => { + const ultra = VLLM_MODELS.find((m) => m.envValue === "nemotron-3-ultra-550b-a55b"); + expect(ultra).toBeDefined(); + + const canonical = resolveVllmModelProfile(ultra!, "station", {} as NodeJS.ProcessEnv); + expect(canonical).toEqual({ model: ultra, profile: null }); + expect(canonical.model).toBe(ultra); + + const resolved = resolveVllmModelProfile(ultra!, "station", { + [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, + } as NodeJS.ProcessEnv); + const profile = resolved.profile; + expect(profile).toBeDefined(); + expect(profile).toEqual({ + id: "experimental-single-user", + label: "Experimental single-user", + experimental: true, + optIn: true, + platforms: ["station"], + servedModelId: "nemotron-ultra", + revision: "183968f87ae4cedce3039313cac1fd43d112c578", + exactServeArgs: [ + "--served-model-name", + "nemotron-ultra", + "--host", + "0.0.0.0", + "--port", + "8000", + "--tensor-parallel-size", + "1", + "--trust-remote-code", + "--cpu-offload-gb", + "90", + "--cpu-offload-params", + "experts", + "--kernel_config", + `'{"enable_flashinfer_autotune": false}'`, + "--max-num-seqs", + "1", + "--gpu-memory-utilization", + "0.9", + "--kv-cache-memory-bytes", + "4294967296", + "--reasoning-parser", + "nemotron_v3", + "--enable-auto-tool-choice", + "--tool-call-parser", + "qwen3_coder", + ], + runtime: { + image: NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, + imageDownloadSizeBytes: 10_670_087_425, + modelDownloadSizeBytes: 352_381_245_521, + loadTimeoutSec: 1200, + dockerRunArgs: ["--shm-size", "16g", "--memory", "650g"], + networkMode: "host", + containerEnv: { + VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: "1", + VLLM_NVFP4_GEMM_BACKEND: "flashinfer-trtllm", + NEMOTRON_ULTRA_MODEL_OPT_NVFP4_MOE_RECLAIM: "1", + HF_HUB_OFFLINE: "1", + TRANSFORMERS_OFFLINE: "1", + PYTHONHASHSEED: "0", + }, + imagePullPolicy: "local-only", + expectedImageId: "sha256:e44830d3f0197850e5a63ca895b13a047f358ff4aea15b1655abf61391bb883e", + qualification: { + gpuNameIncludes: "GB300", + gpuCount: 1, + hbmSafetyCeilingMiB: 245000, + requireZeroUncorrectedEcc: true, + }, + }, + readiness: { + healthEndpoint: true, + servedModelId: "nemotron-ultra", + maxModelLen: 262144, + }, + }); + + expect(resolved.model.revision).toBe("183968f87ae4cedce3039313cac1fd43d112c578"); + expect(resolved.model.servedModelId).toBe("nemotron-ultra"); + expect(resolved.model.serveModelPath).toMatch( + /\/snapshots\/183968f87ae4cedce3039313cac1fd43d112c578$/, + ); + expect(resolved.model.readiness).toEqual({ + healthEndpoint: true, + servedModelId: "nemotron-ultra", + maxModelLen: 262144, + }); + expect(resolved.model.serveEnv).toBeUndefined(); + + const cmd = buildVllmServeCommand(resolved.model, {} as NodeJS.ProcessEnv); + expect(cmd).toBe( + [ + "vllm serve", + "/root/.cache/huggingface/hub/models--nvidia--NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4/snapshots/183968f87ae4cedce3039313cac1fd43d112c578", + "--served-model-name nemotron-ultra", + "--host 0.0.0.0", + "--port 8000", + "--tensor-parallel-size 1", + "--trust-remote-code", + "--cpu-offload-gb 90", + "--cpu-offload-params experts", + `--kernel_config '{"enable_flashinfer_autotune": false}'`, + "--max-num-seqs 1", + "--gpu-memory-utilization 0.9", + "--kv-cache-memory-bytes 4294967296", + "--reasoning-parser nemotron_v3", + "--enable-auto-tool-choice", + "--tool-call-parser qwen3_coder", + ].join(" "), ); - expect(cmd).toMatch( - /^export VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY=1 && export VLLM_NVFP4_GEMM_BACKEND=flashinfer-trtllm && /, + for (const forbidden of [ + "--enable-prefix-caching", + "--speculative-config", + "--default-chat-template-kwargs", + "--pipeline-parallel-size", + "--data-parallel-size", + "--cpu-offload-gb 150", + "--max-num-seqs 256", + "--tool-call-parser hermes", + `{"enable_flashinfer_autotune": true}`, + ]) { + expect(cmd).not.toContain(forbidden); + } + + const driftedCanonical = { ...ultra!, revision: "future-canonical-revision" }; + const independentlyPinned = resolveVllmModelProfile(driftedCanonical, "station", { + [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, + } as NodeJS.ProcessEnv).model; + expect(independentlyPinned.revision).toBe("183968f87ae4cedce3039313cac1fd43d112c578"); + expect(independentlyPinned.serveModelPath).toMatch( + /\/snapshots\/183968f87ae4cedce3039313cac1fd43d112c578$/, ); }); + it("rejects invalid or incompatible explicit model profiles before runtime work", () => { + const ultra = VLLM_MODELS.find((m) => m.envValue === "nemotron-3-ultra-550b-a55b"); + const qwen = VLLM_MODELS.find((m) => m.envValue === "qwen3.6-27b"); + expect(ultra).toBeDefined(); + expect(qwen).toBeDefined(); + + expect(() => + selectVllmModelProfileFromEnv({ + [VLLM_PROFILE_ENV]: "unknown-profile", + } as NodeJS.ProcessEnv), + ).toThrow(/Unknown NEMOCLAW_VLLM_PROFILE='unknown-profile'/); + expect(() => + resolveVllmModelProfile(qwen!, "station", { + [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, + } as NodeJS.ProcessEnv), + ).toThrow(/requires NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b'/); + expect(() => + resolveVllmModelProfile(ultra!, "linux", { + [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, + } as NodeJS.ProcessEnv), + ).toThrow(/is not supported on linux/); + expect(() => + resolveVllmModelProfile(ultra!, "station", { + [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, + [VLLM_EXTRA_ARGS_ENV]: '["--enable-prefix-caching"]', + } as NodeJS.ProcessEnv), + ).toThrow(/cannot be combined with NEMOCLAW_VLLM_EXTRA_ARGS_JSON/); + }); + it("rejects an unknown NEMOCLAW_VLLM_MODEL with a helpful message", () => { expect(() => selectVllmModelFromEnv({ NEMOCLAW_VLLM_MODEL: "made-up-model" } as NodeJS.ProcessEnv), @@ -378,6 +557,60 @@ describe("preflightVllmModelEnv", () => { ).toEqual({ ok: true }); }); + it("succeeds for the explicit qualified Station profile and matching Ultra model", () => { + expect( + preflightVllmModelEnv({ + NEMOCLAW_VLLM_MODEL: "nemotron-3-ultra-550b-a55b", + [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, + } as NodeJS.ProcessEnv), + ).toEqual({ ok: true }); + }); + + it("rejects invalid or incompatible profile input", () => { + expect( + preflightVllmModelEnv({ + [VLLM_PROFILE_ENV]: "unknown-profile", + } as NodeJS.ProcessEnv), + ).toMatchObject({ + ok: false, + message: expect.stringMatching(/Unknown NEMOCLAW_VLLM_PROFILE='unknown-profile'/), + }); + expect( + preflightVllmModelEnv({ + [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, + } as NodeJS.ProcessEnv), + ).toEqual({ + ok: false, + message: + "NEMOCLAW_VLLM_PROFILE='experimental-single-user' requires " + + "NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b'.", + }); + expect( + preflightVllmModelEnv({ + NEMOCLAW_VLLM_MODEL: "qwen3.6-27b", + [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, + } as NodeJS.ProcessEnv), + ).toEqual({ + ok: false, + message: + "NEMOCLAW_VLLM_PROFILE='experimental-single-user' requires " + + "NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b'.", + }); + expect( + preflightVllmModelEnv({ + NEMOCLAW_VLLM_MODEL: "nemotron-3-ultra-550b-a55b", + [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, + [VLLM_EXTRA_ARGS_ENV]: '["--enable-prefix-caching"]', + } as NodeJS.ProcessEnv), + ).toEqual({ + ok: false, + message: + "NEMOCLAW_VLLM_PROFILE='experimental-single-user' cannot be combined with " + + "NEMOCLAW_VLLM_EXTRA_ARGS_JSON; extra arguments would invalidate the qualified " + + "runtime contract.", + }); + }); + it("succeeds for a gated slug when HF_TOKEN is set", () => { expect( preflightVllmModelEnv({ diff --git a/src/lib/inference/vllm-models.ts b/src/lib/inference/vllm-models.ts index e0b927e61e9..a42c924bf2f 100644 --- a/src/lib/inference/vllm-models.ts +++ b/src/lib/inference/vllm-models.ts @@ -29,6 +29,30 @@ export type VllmPlatform = "spark" | "station" | "linux"; +export const VLLM_PROFILE_ENV = "NEMOCLAW_VLLM_PROFILE"; +export const EXPERIMENTAL_SINGLE_USER_PROFILE = "experimental-single-user" as const; +export type VllmModelProfileId = typeof EXPERIMENTAL_SINGLE_USER_PROFILE; + +export interface VllmRuntimeQualification { + /** GPU model fragment used to select the qualified accelerator. */ + gpuNameIncludes: string; + /** Exact number of matching accelerators required by this single-user recipe. */ + gpuCount: number; + /** Exclusive HBM ceiling used by the Station qualification procedure. */ + hbmSafetyCeilingMiB: number; + /** The qualification procedure must fail closed on non-zero uncorrected ECC. */ + requireZeroUncorrectedEcc: boolean; +} + +export interface VllmReadinessContract { + /** Require the health endpoint in addition to the OpenAI model catalog. */ + healthEndpoint: boolean; + /** Require one exact served alias rather than accepting any non-empty catalog. */ + servedModelId: string; + /** Exact context length that the model catalog must advertise. */ + maxModelLen: number; +} + export interface VllmRuntimeOverride { /** Model-specific runtime image, pinned by digest. */ image: string; @@ -40,6 +64,38 @@ export interface VllmRuntimeOverride { loadTimeoutSec?: number; /** Additional `docker run` arguments required by this recipe. */ dockerRunArgs?: readonly string[]; + /** Networking mode for the long-lived inference container. Defaults to bridge. */ + networkMode?: "bridge" | "host"; + /** Non-secret environment passed to the long-lived inference container. */ + containerEnv?: Readonly>; + /** Registry images are pulled; local-only images must already exist exactly. */ + imagePullPolicy?: "registry" | "local-only"; + /** Docker image/config ID that must back image, even when a manifest ref is used. */ + expectedImageId?: string; + /** Hardware qualification limits associated with this reviewed recipe. */ + qualification?: VllmRuntimeQualification; +} + +export interface VllmModelProfileOverride { + /** Stable opt-in selector consumed from NEMOCLAW_VLLM_PROFILE. */ + id: VllmModelProfileId; + /** Human label for summaries and documentation. */ + label: string; + /** Profiles in this registry are never selected implicitly. */ + experimental: true; + optIn: true; + /** Platforms on which this exact recipe was qualified. */ + platforms: readonly VllmPlatform[]; + /** Served alias used by the gateway and dashboard. */ + servedModelId: string; + /** Immutable Hugging Face revision owned by this profile. */ + revision: string; + /** Complete ordered vLLM arguments after the model ID. Replaces shared/default args. */ + exactServeArgs: readonly string[]; + /** Required runtime/container replacement for this profile. */ + runtime: VllmRuntimeOverride; + /** Strict endpoint contract checked before onboarding continues. */ + readiness: VllmReadinessContract; } export const NEMOTRON_ULTRA_STATION_IMAGE = { @@ -50,6 +106,14 @@ export const NEMOTRON_ULTRA_STATION_IMAGE = { }, } as const; +// Exact Docker image/config ID qualified by the reclaimed-v1 offload90 study. +// It is local-only until a repository-qualified manifest publishes these bytes. +// Once published, image can become that manifest reference while expectedImageId +// continues to prove that the pulled image resolves to this exact ID. +export const NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE = { + localImageId: "sha256:e44830d3f0197850e5a63ca895b13a047f358ff4aea15b1655abf61391bb883e", +} as const; + export interface VllmModelDef { /** Hugging Face model id (also passed to `vllm serve`). */ id: string; @@ -63,6 +127,8 @@ export interface VllmModelDef { revision?: string; /** Stable model name exposed by the local OpenAI-compatible endpoint. */ servedModelId?: string; + /** Exact snapshot path used by a resolved profile instead of the repository ID. */ + serveModelPath?: string; /** Model-specific flags appended after the shared serving flags. */ modelArgs: string[]; /** True when the upstream HF repo requires accepting a licence. */ @@ -85,6 +151,12 @@ export interface VllmModelDef { serveEnv?: Record; /** Runtime overrides for recipes that cannot use the platform image. */ runtime?: VllmRuntimeOverride; + /** Complete serve args for a resolved model profile; base entries use shared args. */ + exactServeArgs?: readonly string[]; + /** Strict readiness contract for a resolved model profile. */ + readiness?: VllmReadinessContract; + /** Explicit model-scoped profiles. These are never interactive/default choices. */ + profiles?: Partial>; /** Whether startup must install vLLM's fastsafetensors extra. Defaults to true. */ installFastSafetensors?: boolean; } @@ -247,6 +319,76 @@ export const VLLM_MODELS: readonly VllmModelDef[] = [ // importing the playbook's host-network setting. dockerRunArgs: ["--shm-size", "16g", "--ulimit", "memlock=-1", "--ulimit", "stack=67108864"], }, + profiles: { + [EXPERIMENTAL_SINGLE_USER_PROFILE]: { + id: EXPERIMENTAL_SINGLE_USER_PROFILE, + label: "Experimental single-user", + experimental: true, + optIn: true, + platforms: ["station"], + servedModelId: "nemotron-ultra", + revision: "183968f87ae4cedce3039313cac1fd43d112c578", + exactServeArgs: [ + "--served-model-name", + "nemotron-ultra", + "--host", + "0.0.0.0", + "--port", + "8000", + "--tensor-parallel-size", + "1", + "--trust-remote-code", + "--cpu-offload-gb", + "90", + "--cpu-offload-params", + "experts", + "--kernel_config", + "'{\"enable_flashinfer_autotune\": false}'", + "--max-num-seqs", + "1", + "--gpu-memory-utilization", + "0.9", + "--kv-cache-memory-bytes", + "4294967296", + "--reasoning-parser", + "nemotron_v3", + "--enable-auto-tool-choice", + "--tool-call-parser", + "qwen3_coder", + ], + runtime: { + image: NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, + // Unused while local-only. A future manifest publication must make + // its compressed pull accounting explicit before this policy changes. + imageDownloadSizeBytes: NEMOTRON_ULTRA_STATION_IMAGE.arm64.downloadSizeBytes, + modelDownloadSizeBytes: 352_381_245_521, + loadTimeoutSec: 1200, + dockerRunArgs: ["--shm-size", "16g", "--memory", "650g"], + networkMode: "host", + containerEnv: { + VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: "1", + VLLM_NVFP4_GEMM_BACKEND: "flashinfer-trtllm", + NEMOTRON_ULTRA_MODEL_OPT_NVFP4_MOE_RECLAIM: "1", + HF_HUB_OFFLINE: "1", + TRANSFORMERS_OFFLINE: "1", + PYTHONHASHSEED: "0", + }, + imagePullPolicy: "local-only", + expectedImageId: NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, + qualification: { + gpuNameIncludes: "GB300", + gpuCount: 1, + hbmSafetyCeilingMiB: 245000, + requireZeroUncorrectedEcc: true, + }, + }, + readiness: { + healthEndpoint: true, + servedModelId: "nemotron-ultra", + maxModelLen: 262144, + }, + }, + }, // The digest-pinned vLLM image already contains the serving package, and // this recipe does not use the fastsafetensors load format. Avoid mutating // the reviewed runtime from the package index when the container starts. @@ -322,6 +464,94 @@ export function modelsForPlatform(platform: VllmPlatform): readonly VllmModelDef const HF_TOKEN_ENV_KEYS = ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"] as const; export const VLLM_EXTRA_ARGS_ENV = "NEMOCLAW_VLLM_EXTRA_ARGS_JSON"; +/** Parse and validate the explicit model-profile selector. */ +export function selectVllmModelProfileFromEnv( + env: NodeJS.ProcessEnv = process.env, +): VllmModelProfileId | null { + const requested = String(env[VLLM_PROFILE_ENV] ?? "") + .trim() + .toLowerCase(); + if (!requested) return null; + if (requested === EXPERIMENTAL_SINGLE_USER_PROFILE) return requested; + throw new Error( + "Unknown " + + VLLM_PROFILE_ENV + + "='" + + String(env[VLLM_PROFILE_ENV]) + + "'. Recognised values: '" + + EXPERIMENTAL_SINGLE_USER_PROFILE + + "'.", + ); +} + +export interface ResolvedVllmModelProfile { + model: VllmModelDef; + profile: VllmModelProfileOverride | null; +} + +function huggingFaceSnapshotContainerPath(modelId: string, revision: string): string { + return ( + "/root/.cache/huggingface/hub/models--" + + modelId.replaceAll("/", "--") + + "/snapshots/" + + revision + ); +} + +/** + * Resolve an explicit model-scoped profile into the effective recipe consumed + * by the existing managed-vLLM lifecycle. With no selector this returns the + * original model object unchanged, which freezes the canonical Station output. + */ +export function resolveVllmModelProfile( + model: VllmModelDef, + platform: VllmPlatform, + env: NodeJS.ProcessEnv = process.env, +): ResolvedVllmModelProfile { + const profileId = selectVllmModelProfileFromEnv(env); + if (!profileId) return { model, profile: null }; + + const profile = model.profiles?.[profileId]; + if (!profile) { + throw new Error( + VLLM_PROFILE_ENV + + "='" + + profileId + + "' requires NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b'.", + ); + } + if (!profile.platforms.includes(platform)) { + throw new Error( + VLLM_PROFILE_ENV + "='" + profileId + "' is not supported on " + platform + ".", + ); + } + if (parseVllmExtraServeArgs(env).length > 0) { + throw new Error( + VLLM_PROFILE_ENV + + "='" + + profileId + + "' cannot be combined with " + + VLLM_EXTRA_ARGS_ENV + + "; extra arguments would invalidate the qualified runtime contract.", + ); + } + + return { + profile, + model: { + ...model, + revision: profile.revision, + servedModelId: profile.servedModelId, + serveModelPath: huggingFaceSnapshotContainerPath(model.id, profile.revision), + modelArgs: [], + serveEnv: undefined, + runtime: profile.runtime, + exactServeArgs: [...profile.exactServeArgs], + readiness: profile.readiness, + }, + }; +} + /** * Look up the requested express-vLLM model from `NEMOCLAW_VLLM_MODEL`. * Returns `null` when the env var is empty so the caller can fall back to @@ -396,9 +626,38 @@ export function preflightVllmModelEnv( env: NodeJS.ProcessEnv = process.env, ): PreflightVllmModelResult { try { - parseVllmExtraServeArgs(env); + const profileId = selectVllmModelProfileFromEnv(env); + const extraArgs = parseVllmExtraServeArgs(env); + if (profileId && extraArgs.length > 0) { + throw new Error( + VLLM_PROFILE_ENV + + "='" + + profileId + + "' cannot be combined with " + + VLLM_EXTRA_ARGS_ENV + + "; extra arguments would invalidate the qualified runtime contract.", + ); + } const model = selectVllmModelFromEnv(env); - if (!model) return { ok: true }; + if (!model) { + if (profileId) { + throw new Error( + VLLM_PROFILE_ENV + + "='" + + profileId + + "' requires NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b'.", + ); + } + return { ok: true }; + } + if (profileId && !model.profiles?.[profileId]) { + throw new Error( + VLLM_PROFILE_ENV + + "='" + + profileId + + "' requires NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b'.", + ); + } assertGatedModelAccess(model, env); return { ok: true }; } catch (err) { @@ -478,16 +737,21 @@ export function buildVllmServeCommand( .map(([key, value]) => `export ${key}=${value}`) .join(" && ")} && ` : ""; - const args = [ - ...SHARED_VLLM_ARGS, - "--max-model-len", - String(model.maxModelLen), - ...(model.revision ? ["--revision", model.revision] : []), - ...(model.servedModelId ? ["--served-model-name", model.servedModelId] : []), - ...model.modelArgs, - ]; + const args = model.exactServeArgs + ? [...model.exactServeArgs] + : [ + ...SHARED_VLLM_ARGS, + "--max-model-len", + String(model.maxModelLen), + ...(model.revision ? ["--revision", model.revision] : []), + ...(model.servedModelId ? ["--served-model-name", model.servedModelId] : []), + ...model.modelArgs, + ]; const extraArgs = parseVllmExtraServeArgs(env).map(shellQuote); const setup = model.installFastSafetensors === false ? "" : "pip install vllm[fastsafetensors] && "; - return `${envPrefix}${setup}vllm serve ${model.id} ${[...args, ...extraArgs].join(" ")}`; + return `${envPrefix}${setup}vllm serve ${model.serveModelPath ?? model.id} ${[ + ...args, + ...extraArgs, + ].join(" ")}`; } diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index 0fdfe606d6a..fa460f1cdbf 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -63,7 +63,7 @@ import { resolveVllmRuntimeProfile, resolveVllmServedModelId, } from "./vllm"; -import { VLLM_MODELS } from "./vllm-models"; +import { buildVllmServeCommand, VLLM_MODELS } from "./vllm-models"; beforeEach(() => { mocks.dockerImageInspectFormat.mockReturnValue(""); @@ -115,14 +115,19 @@ function vllmContainerRow( function mockSuccessfulVllmInstall( containerName: string, ownershipResponses: readonly (() => string)[] = [() => "", () => ""], + endpoints: { + healthStatus?: string; + modelsResponse?: string; + } = {}, ): void { - const runCaptureByCommand: Record = { - curl: '{"data":[]}', - sh: "/usr/bin/tool\n", - }; - mocks.runCapture.mockImplementation( - (cmd: readonly string[]) => runCaptureByCommand[cmd[0] ?? ""] ?? "", - ); + mocks.runCapture.mockImplementation((cmd: readonly string[]) => { + if (cmd[0] === "sh") return "/usr/bin/tool\n"; + if (cmd[0] !== "curl") return ""; + const url = cmd.at(-1) ?? ""; + return url.endsWith("/health") + ? (endpoints.healthStatus ?? "") + : (endpoints.modelsResponse ?? '{"data":[]}'); + }); mocks.dockerPullWithProgressWatchdog.mockResolvedValue({ status: 0, signal: null, @@ -201,10 +206,37 @@ describe("vLLM profile detection", () => { expect.arrayContaining(["--gpus", "device=0", "--shm-size", "16g"]), ); - const args = buildVllmRunArgs(runtime, ultra!, runtime.buildDockerRunFlags!()); - expect(args).not.toContain("--network"); - expect(args).toEqual(expect.arrayContaining(["-p", "8000:8000"])); - expect(args).toContain(runtime.image); + const flags = runtime.buildDockerRunFlags!(); + const args = buildVllmRunArgs(runtime, ultra!, flags, {} as NodeJS.ProcessEnv); + expect(args).toEqual([ + "--pull=never", + "--restart", + "unless-stopped", + "--gpus", + "device=0", + "--ipc=host", + "-v", + `${path.join(os.homedir(), ".cache", "huggingface")}:/root/.cache/huggingface`, + "-e", + "HF_HOME=/root/.cache/huggingface", + "--shm-size", + "16g", + "--ulimit", + "memlock=-1", + "--ulimit", + "stack=67108864", + "--label", + `${NEMOCLAW_VLLM_MANAGED_LABEL}=true`, + "-p", + "8000:8000", + "--name", + NEMOCLAW_VLLM_CONTAINER_NAME, + "--entrypoint", + "/bin/bash", + runtime.image, + "-lc", + buildVllmServeCommand(ultra!, {} as NodeJS.ProcessEnv), + ]); }); it("keeps DGX Spark on the Qwen3.6 35B NVFP4 default", () => { @@ -474,6 +506,7 @@ describe("installVllm model resolution", () => { mkdirSpy = vi.spyOn(fs, "mkdirSync").mockImplementation(() => undefined); stdoutWrite = vi.spyOn(process.stdout, "write").mockImplementation(() => true); delete process.env.NEMOCLAW_VLLM_MODEL; + delete process.env.NEMOCLAW_VLLM_PROFILE; delete process.env.NEMOCLAW_VLLM_EXTRA_ARGS_JSON; delete process.env.NEMOCLAW_IGNORE_VLLM_DISK_SPACE; delete process.env.HF_TOKEN; diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index fe98350f528..08b8613c3c4 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -17,7 +17,7 @@ import { dockerSpawn, dockerStop, } from "../adapters/docker"; -import { buildValidatedCurlCommandArgs } from "../adapters/http/curl-args"; +import { buildCurlProbeSpawnArgs, buildValidatedCurlCommandArgs } from "../adapters/http/curl-args"; import { VLLM_PORT } from "../core/ports"; import { shellQuote } from "../core/shell-quote"; import { isAffirmativeAnswer } from "../onboard/prompt-helpers"; @@ -29,10 +29,12 @@ import { buildVllmServeCommand, NEMOTRON_ULTRA_STATION_IMAGE, parseVllmExtraServeArgs, + resolveVllmModelProfile, VLLM_EXTRA_ARGS_ENV, VLLM_MODELS, type VllmModelDef, type VllmPlatform, + type VllmRuntimeQualification, } from "./vllm-models"; import { resolveVllmInstallModel } from "./vllm-prompt"; import { @@ -82,6 +84,14 @@ export interface VllmProfile { // Optional pinned model snapshot size. Model-specific runtime overrides use // this to guard the host Hugging Face cache before a cold download. modelDownloadSizeBytes?: number; + // Host mode is profile-scoped because it also suppresses Docker port + // publication; raw run args cannot represent that invariant safely. + networkMode?: "bridge" | "host"; + // Registry is the historical default. Local-only fails closed unless the + // exact qualified Docker image/config ID is already present. + imagePullPolicy?: "registry" | "local-only"; + expectedImageId?: string; + qualification?: VllmRuntimeQualification; } // Platform manifests and decimal compressed sizes published by NGC for the @@ -489,8 +499,7 @@ export function buildVllmRunArgs( ...safeRunFlags, "--label", `${NEMOCLAW_VLLM_MANAGED_LABEL}=true`, - "-p", - `${String(VLLM_PORT)}:8000`, + ...(profile.networkMode === "host" ? [] : ["-p", String(VLLM_PORT) + ":8000"]), "--name", containerName, "--entrypoint", @@ -504,13 +513,24 @@ export function buildVllmRunArgs( export function resolveVllmRuntimeProfile(profile: VllmProfile, model: VllmModelDef): VllmProfile { const runtime = model.runtime; if (!runtime) return profile; - const extraRunArgs = [...(runtime.dockerRunArgs ?? [])]; + const extraRunArgs = [ + ...(runtime.networkMode === "host" ? ["--network", "host"] : []), + ...(runtime.dockerRunArgs ?? []), + ...Object.entries(runtime.containerEnv ?? {}).flatMap(([key, value]) => [ + "-e", + key + "=" + value, + ]), + ]; return { ...profile, image: runtime.image, imageDownloadSizeBytes: runtime.imageDownloadSizeBytes, modelDownloadSizeBytes: runtime.modelDownloadSizeBytes ?? profile.modelDownloadSizeBytes, loadTimeoutSec: runtime.loadTimeoutSec ?? profile.loadTimeoutSec, + networkMode: runtime.networkMode ?? profile.networkMode, + imagePullPolicy: runtime.imagePullPolicy ?? profile.imagePullPolicy, + expectedImageId: runtime.expectedImageId ?? profile.expectedImageId, + qualification: runtime.qualification ?? profile.qualification, dockerRunFlags: [...profile.dockerRunFlags, ...extraRunArgs], buildDockerRunFlags: profile.buildDockerRunFlags ? () => [...profile.buildDockerRunFlags!(), ...extraRunArgs] @@ -625,30 +645,135 @@ function vllmModelsEndpoint(): string { return `http://127.0.0.1:${String(VLLM_PORT)}/v1/models`; } -function vllmEndpointReady(): boolean { +function vllmHealthEndpoint(): string { + return `http://127.0.0.1:${String(VLLM_PORT)}/health`; +} + +function queryVllmEndpoint(url: string): string { const response = runCapture( [ "curl", - ...buildValidatedCurlCommandArgs([ - "-sf", - "--connect-timeout", - "2", - "--max-time", - "5", - vllmModelsEndpoint(), - ]), + ...buildValidatedCurlCommandArgs(["-sf", "--connect-timeout", "2", "--max-time", "5", url]), ], { ignoreError: true }, ).trim(); + return response; +} + +function vllmHealthReady(): boolean { + const validated = buildValidatedCurlCommandArgs([ + "-sf", + "--connect-timeout", + "2", + "--max-time", + "5", + vllmHealthEndpoint(), + ]); + const url = validated.at(-1); + if (!url) return false; + const response = runCapture( + ["curl", ...buildCurlProbeSpawnArgs(validated.slice(0, -1), url, "/dev/null", "json")], + { ignoreError: true }, + ).trim(); + return response === "200"; +} + +function vllmEndpointReady(model: VllmModelDef): boolean { + const readiness = model.readiness; + // vLLM's /health endpoint commonly returns an empty 200 response, so check + // its status code instead of treating a non-empty body as the success signal. + if (readiness?.healthEndpoint && !vllmHealthReady()) return false; + const response = queryVllmEndpoint(vllmModelsEndpoint()); if (!response) return false; try { - const parsed = JSON.parse(response) as { data?: unknown }; - return Array.isArray(parsed.data); + const parsed = JSON.parse(response) as { + data?: Array<{ id?: unknown; max_model_len?: unknown }>; + }; + if (!Array.isArray(parsed.data)) return false; + if (!readiness) return true; + return ( + parsed.data.length === 1 && + parsed.data[0]?.id === readiness.servedModelId && + Number(parsed.data[0]?.max_model_len) === readiness.maxModelLen + ); } catch { return false; } } +interface VllmQualificationMonitor { + gpuIndex: number; + peakHbmMiB: number; +} + +type VllmQualificationResult = + | { ok: true; monitor: VllmQualificationMonitor } + | { ok: false; reason: string }; + +function literalCaseInsensitivePattern(value: string): RegExp { + return new RegExp(value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i"); +} + +function sampleVllmQualification( + profile: VllmProfile, + monitor: VllmQualificationMonitor, +): VllmQualificationResult { + const qualification = profile.qualification; + if (!qualification) return { ok: true, monitor }; + const raw = runCapture( + [ + "nvidia-smi", + `--id=${String(monitor.gpuIndex)}`, + "--query-gpu=memory.used,ecc.errors.uncorrected.aggregate.total,ecc.errors.uncorrected.volatile.total", + "--format=csv,noheader,nounits", + ], + { ignoreError: true }, + ).trim(); + const rows = raw.split(/\r?\n/).filter((row) => row.trim().length > 0); + const values = rows.length === 1 ? rows[0].split(",").map((value) => value.trim()) : []; + if (values.length !== 3 || values.some((value) => !/^\d+$/.test(value))) { + return { + ok: false, + reason: `qualified GPU telemetry is unavailable or malformed for GPU ${String(monitor.gpuIndex)}`, + }; + } + + const [hbmMiB, aggregateEcc, volatileEcc] = values.map(Number); + monitor.peakHbmMiB = Math.max(monitor.peakHbmMiB, hbmMiB); + if (hbmMiB >= qualification.hbmSafetyCeilingMiB) { + return { + ok: false, + reason: + `qualified HBM safety ceiling exceeded on GPU ${String(monitor.gpuIndex)}: ` + + `${String(hbmMiB)} MiB (required <${String(qualification.hbmSafetyCeilingMiB)} MiB)`, + }; + } + if (qualification.requireZeroUncorrectedEcc && (aggregateEcc !== 0 || volatileEcc !== 0)) { + return { + ok: false, + reason: + `uncorrected ECC is nonzero on GPU ${String(monitor.gpuIndex)}: ` + + `aggregate=${String(aggregateEcc)}, volatile=${String(volatileEcc)}`, + }; + } + return { ok: true, monitor }; +} + +function initializeVllmQualification(profile: VllmProfile): VllmQualificationResult | null { + const qualification = profile.qualification; + if (!qualification) return null; + const indices = getGpuIndicesByName(literalCaseInsensitivePattern(qualification.gpuNameIncludes)); + if (indices.length !== qualification.gpuCount) { + return { + ok: false, + reason: + `qualified runtime requires exactly ${String(qualification.gpuCount)} GPU matching ` + + `'${qualification.gpuNameIncludes}'; detected ${String(indices.length)}`, + }; + } + return sampleVllmQualification(profile, { gpuIndex: indices[0], peakHbmMiB: 0 }); +} + function readContainerLogTail(profile: VllmProfile, lineCount = 80): string[] { const output = dockerCapture(["logs", "--tail", String(lineCount), profile.containerName], { env: buildVllmDockerEnv(), @@ -669,7 +794,11 @@ function printContainerLogTail(profile: VllmProfile): void { // Poll the real OpenAI-compatible models endpoint instead of interpreting // vLLM startup logs. Logs stay quiet on the happy path and print only on // failure. -function waitForVllmReady(profile: VllmProfile): Promise<{ ok: boolean; reason?: string }> { +function waitForVllmReady( + profile: VllmProfile, + model: VllmModelDef, + qualificationMonitor: VllmQualificationMonitor | null, +): Promise<{ ok: boolean; reason?: string }> { return new Promise((resolve) => { let resolved = false; const start = Date.now(); @@ -689,7 +818,20 @@ function waitForVllmReady(profile: VllmProfile): Promise<{ ok: boolean; reason?: function poll(): void { if (resolved) return; - if (vllmEndpointReady()) { + if (qualificationMonitor) { + const qualification = sampleVllmQualification(profile, qualificationMonitor); + if (!qualification.ok) { + done({ ok: false, reason: qualification.reason }); + return; + } + } + if (vllmEndpointReady(model)) { + if (qualificationMonitor && profile.qualification) { + emit( + `Qualified startup telemetry: peak HBM ${String(qualificationMonitor.peakHbmMiB)} MiB ` + + `(<${String(profile.qualification.hbmSafetyCeilingMiB)} MiB), uncorrected ECC 0`, + ); + } emit(`vLLM is serving on :${String(VLLM_PORT)}`); done({ ok: true }); return; @@ -889,14 +1031,33 @@ interface InstallVllmOptions { beforeInstall?: (modelId: string) => void; } +function inspectImageId(profile: VllmProfile): string { + return dockerImageInspectFormat("{{.Id}}", profile.image, { + env: buildVllmDockerEnv(), + ignoreError: true, + timeout: 10_000, + }).trim(); +} + function imageIsCached(profile: VllmProfile): boolean { - return Boolean( - dockerImageInspectFormat("{{.Id}}", profile.image, { - env: buildVllmDockerEnv(), - ignoreError: true, - timeout: 10_000, - }).trim(), - ); + return Boolean(inspectImageId(profile)); +} + +function verifyExpectedImageId( + profile: VllmProfile, + observedImageId: string, +): { ok: true } | { ok: false; reason: string } { + if (!profile.expectedImageId || observedImageId === profile.expectedImageId) { + return { ok: true }; + } + return { + ok: false, + reason: + "qualified image identity mismatch: expected " + + profile.expectedImageId + + ", resolved " + + (observedImageId || "no local image"), + }; } export function resolveVllmServedModelId(modelId: string, extraServeArgs: string[]): string { @@ -935,7 +1096,17 @@ export async function installVllm( promptFn: opts.promptFn, }); if (!resolved) return { ok: false }; - const { model, source: modelSource } = resolved; + const { model: selectedModel, source: modelSource } = resolved; + let model: VllmModelDef; + let activeModelProfile: ReturnType["profile"]; + try { + const effective = resolveVllmModelProfile(selectedModel, profile.platform); + model = effective.model; + activeModelProfile = effective.profile; + } catch (err) { + console.error(` vLLM install failed: ${(err as Error).message}`); + return { ok: false }; + } if (model.runtime && !model.platforms.includes(profile.platform)) { console.error(` vLLM install failed: ${model.label} is not supported on ${profile.name}`); return { ok: false }; @@ -959,12 +1130,19 @@ export async function installVllm( console.log( ` Model: ${model.id}${modelSource === "env" ? " (NEMOCLAW_VLLM_MODEL override)" : ""}`, ); + if (activeModelProfile) { + console.log(` Profile: ${activeModelProfile.id} (experimental, opt-in)`); + } if (extraServeArgs.length > 0) { console.log( ` Extra serve args: ${String(extraServeArgs.length)} token(s) from ${VLLM_EXTRA_ARGS_ENV}`, ); } - if (!opts.hasImage) console.log(" Image download on first run, cached after"); + if (runtimeProfile.imagePullPolicy === "local-only") { + console.log(" Qualified local image required; no pull or fallback is available"); + } else if (!opts.hasImage) { + console.log(" Image download on first run, cached after"); + } console.log(" Model download on first run, cached after"); console.log(""); @@ -990,6 +1168,35 @@ export async function installVllm( return { ok: false }; } + let localOnlyImageId: string | null = null; + if (runtimeProfile.imagePullPolicy === "local-only") { + localOnlyImageId = inspectImageId(runtimeProfile); + if (!localOnlyImageId) { + console.error( + " vLLM install blocked: the qualified experimental runtime image is not published by the configured image source.", + ); + console.error( + ` Required local image ID: ${runtimeProfile.expectedImageId ?? runtimeProfile.image}`, + ); + console.error( + " Publish a repository-qualified manifest for these exact bytes and prove its pulled .Id before enabling a fresh install. No fallback image was substituted.", + ); + return { ok: false }; + } + const identity = verifyExpectedImageId(runtimeProfile, localOnlyImageId); + if (!identity.ok) { + console.error(` vLLM install failed: ${identity.reason}`); + return { ok: false }; + } + } + + const qualification = initializeVllmQualification(runtimeProfile); + if (qualification && !qualification.ok) { + console.error(` vLLM install failed: ${qualification.reason}`); + return { ok: false }; + } + const qualificationMonitor = qualification?.monitor ?? null; + // Guard the host filesystem before an image pull or model-download // container can start. The cache path itself is created only after both // storage decisions pass, so Docker never creates it as root. @@ -997,7 +1204,7 @@ export async function installVllm( return { ok: false }; } - const hasImage = imageIsCached(runtimeProfile); + const hasImage = localOnlyImageId !== null || imageIsCached(runtimeProfile); if (!hasImage && !(await imageStorageAccepted(runtimeProfile, opts))) { return { ok: false }; } @@ -1008,10 +1215,19 @@ export async function installVllm( return { ok: false }; } - const pull = await pullImage(runtimeProfile); - if (!pull.ok) { - console.error(` vLLM install failed: ${String(pull.reason)}`); - return { ok: false }; + if (runtimeProfile.imagePullPolicy !== "local-only") { + const pull = await pullImage(runtimeProfile); + if (!pull.ok) { + console.error(` vLLM install failed: ${String(pull.reason)}`); + return { ok: false }; + } + if (runtimeProfile.expectedImageId) { + const identity = verifyExpectedImageId(runtimeProfile, inspectImageId(runtimeProfile)); + if (!identity.ok) { + console.error(` vLLM install failed: ${identity.reason}`); + return { ok: false }; + } + } } // A cold image pull can consume the same host filesystem that backs the @@ -1038,7 +1254,7 @@ export async function installVllm( `Launch can take 5 minutes to ${String(Math.ceil(runtimeProfile.loadTimeoutSec / 60))} minutes`, ); - const ready = await waitForVllmReady(runtimeProfile); + const ready = await waitForVllmReady(runtimeProfile, model, qualificationMonitor); if (!ready.ok) { printContainerLogTail(runtimeProfile); dockerStop(runtimeProfile.containerName, { diff --git a/src/lib/onboard/setup-nim-vllm.test.ts b/src/lib/onboard/setup-nim-vllm.test.ts index 9f45d705e73..84b84cf856e 100644 --- a/src/lib/onboard/setup-nim-vllm.test.ts +++ b/src/lib/onboard/setup-nim-vllm.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { requireValue } from "../core/require-value"; +import { applyVllmRuntimeContextWindow } from "../inference/vllm-runtime-context"; import type { SetupNimSelectionState } from "./setup-nim-flow"; import { buildDgxSparkExistingVllmHeadroomWarning, @@ -98,12 +99,39 @@ describe("setupNim vLLM route containment", () => { " To keep 'served/model' instead, start detailed setup:", ); expect(console.error).toHaveBeenCalledWith( - " unset NEMOCLAW_PROVIDER NEMOCLAW_MODEL NEMOCLAW_VLLM_MODEL", + " unset NEMOCLAW_PROVIDER NEMOCLAW_MODEL NEMOCLAW_VLLM_MODEL NEMOCLAW_VLLM_PROFILE", ); expect(console.error).toHaveBeenCalledWith(" nemoclaw onboard --fresh"); expect(console.error).toHaveBeenCalledWith(" Then select Local vLLM when prompted."); }); + it("carries the experimental Ultra route and 262144 context through the supported lifecycle", async () => { + const selection = state("nemotron-ultra"); + const runtimeEnv: NodeJS.ProcessEnv = {}; + const models = { + data: [{ id: "nemotron-ultra", max_model_len: 262144 }], + }; + const handler = createSetupNimVllmHandler( + deps({ + runCapture: () => JSON.stringify(models), + applyVllmRuntimeContextWindow: (response, model) => + applyVllmRuntimeContextWindow(response, model, { + env: runtimeEnv, + logger: { log: vi.fn(), warn: vi.fn() }, + }), + }), + ); + + await expect(handler(selection, { managedInstall: true })).resolves.toBe("selected"); + expect(selection).toMatchObject({ + provider: "vllm-local", + model: "nemotron-ultra", + endpointUrl: "http://host.openshell.internal:8000/v1", + preferredInferenceApi: "openai-completions", + }); + expect(runtimeEnv.NEMOCLAW_CONTEXT_WINDOW).toBe("262144"); + }); + it("warns on DGX Spark identified via GPU name (firmware-unknown GB10 host)", async () => { const selection = state(null); const handler = createSetupNimVllmHandler( diff --git a/src/lib/onboard/setup-nim-vllm.ts b/src/lib/onboard/setup-nim-vllm.ts index ead86ae7133..00aebeb618b 100644 --- a/src/lib/onboard/setup-nim-vllm.ts +++ b/src/lib/onboard/setup-nim-vllm.ts @@ -199,7 +199,9 @@ export function createSetupNimVllmHandler( ` To install '${requiredModel}', stop the existing vLLM server on localhost:${deps.VLLM_PORT}, then rerun the original install/onboard command.`, ); console.error(` To keep '${detectedModel}' instead, start detailed setup:`); - console.error(" unset NEMOCLAW_PROVIDER NEMOCLAW_MODEL NEMOCLAW_VLLM_MODEL"); + console.error( + " unset NEMOCLAW_PROVIDER NEMOCLAW_MODEL NEMOCLAW_VLLM_MODEL NEMOCLAW_VLLM_PROFILE", + ); console.error(` ${cliName()} onboard --fresh`); console.error(" Then select Local vLLM when prompted."); deps.exitProcess(1); diff --git a/src/lib/onboard/vllm-model-preflight.ts b/src/lib/onboard/vllm-model-preflight.ts index 6abebaade96..a9c87a8c78c 100644 --- a/src/lib/onboard/vllm-model-preflight.ts +++ b/src/lib/onboard/vllm-model-preflight.ts @@ -4,8 +4,8 @@ import { preflightVllmModelEnv } from "../inference/vllm-models"; /** - * Validate `NEMOCLAW_VLLM_MODEL` up front, before onboarding runs preflight or - * touches Docker, mirroring the `connect` preflight added in #4567. + * Validate managed-vLLM model/profile input up front, before onboarding runs + * preflight or touches Docker, mirroring the `connect` preflight added in #4567. * * The variable steers the express-vLLM installer, but it is only consumed deep * in the `[3/8]` provider step. That made it validated late and path-dependent: diff --git a/test/cli/dispatch-basics.test.ts b/test/cli/dispatch-basics.test.ts index 6f504b43d97..cc282bed9e4 100644 --- a/test/cli/dispatch-basics.test.ts +++ b/test/cli/dispatch-basics.test.ts @@ -422,7 +422,9 @@ describe("CLI dispatch", () => { expect(out).toMatch(/gated on Hugging Face/); expect(out).toMatch(/HF_TOKEN/); expect(out).toMatch(/HUGGING_FACE_HUB_TOKEN/); - expect(out).toContain("NEMOCLAW_VLLM_MODEL is consumed by the managed-vLLM install path"); + expect(out).toContain( + "NEMOCLAW_VLLM_MODEL, NEMOCLAW_VLLM_PROFILE, and NEMOCLAW_VLLM_EXTRA_ARGS_JSON are consumed by the managed-vLLM install path", + ); const calls = fs.existsSync(openshellLog) ? fs.readFileSync(openshellLog, "utf8") : ""; expect(calls).not.toMatch(/\bsandbox\s+(get|connect|list)\b/); } finally { diff --git a/test/generate-hermes-config.test.ts b/test/generate-hermes-config.test.ts index 0a629e2b4f8..7c426c01523 100644 --- a/test/generate-hermes-config.test.ts +++ b/test/generate-hermes-config.test.ts @@ -524,6 +524,31 @@ describe("agents/hermes/generate-config.ts", () => { expect(config.model.context_window).toBeUndefined(); }); + it("routes the experimental Station Ultra profile through vllm-local at 262144 context", () => { + const { config } = runConfigScript({ + NEMOCLAW_PROVIDER_KEY: "vllm-local", + NEMOCLAW_MODEL: "nemotron-ultra", + NEMOCLAW_CONTEXT_WINDOW: "262144", + }); + + expect(config._nemoclaw_upstream).toEqual({ + provider: "vllm-local", + model: "nemotron-ultra", + }); + expect(config.model).toMatchObject({ + default: "nemotron-ultra", + provider: "custom", + base_url: "https://inference.local/v1", + context_length: 262144, + }); + expect(config.model.context_window).toBeUndefined(); + expect(config.providers["vllm-local"]).toMatchObject({ + name: "vllm-local", + default_model: "nemotron-ultra", + discover_models: true, + }); + }); + it("chains the endpoint probe through to model.context_length in the generated config (#6177)", async () => { // Source-level regression across the boundary: the same probe onboarding // calls resolves a compatible endpoint's max_model_len into diff --git a/test/inference-options-docs.test.ts b/test/inference-options-docs.test.ts index ef6bdd29248..5be1db15f0e 100644 --- a/test/inference-options-docs.test.ts +++ b/test/inference-options-docs.test.ts @@ -285,7 +285,7 @@ describe("inference setup navigation", () => { (match) => match[1], ); - expect(hostValues).toEqual(["0.0.0.0"]); + expect(hostValues).toEqual(["0.0.0.0", "0.0.0.0"]); expect(markdown).toContain("default-deny inbound rules"); expect(markdown).toContain("only from the OpenShell Docker subnet to its gateway address"); }); @@ -299,7 +299,7 @@ describe("inference setup navigation", () => { tag: VLLM_IMAGES.ngc2605Post1.tag, }, { - prefix: "- The DGX Station Nemotron 3 Ultra express recipe", + prefix: "- The canonical DGX Station Nemotron 3 Ultra express recipe", image: VLLM_IMAGES.vllm022.arm64, tag: VLLM_IMAGES.vllm022.tag, }, diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index 77b24278f96..651e5f999b7 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -39,9 +39,9 @@ NON_INTERACTIVE="\${NON_INTERACTIVE:-}" NEMOCLAW_PROVIDER="\${NEMOCLAW_PROVIDER:-}" NEMOCLAW_NO_EXPRESS="\${NEMOCLAW_NO_EXPRESS:-}" maybe_offer_express_install -printf "RESULT NON_INTERACTIVE=%s SUDO_MODE=%s PROVIDER=%s MODEL=%s VLLM_MODEL=%s POLICY=%s YES=%s SANDBOX=%s\\n" \\ +printf "RESULT NON_INTERACTIVE=%s SUDO_MODE=%s PROVIDER=%s MODEL=%s VLLM_MODEL=%s VLLM_PROFILE=%s POLICY=%s YES=%s SANDBOX=%s\\n" \\ "\${NON_INTERACTIVE:-}" "\${NEMOCLAW_NON_INTERACTIVE_SUDO_MODE:-}" "\${NEMOCLAW_PROVIDER:-}" "\${NEMOCLAW_MODEL:-}" \\ - "\${NEMOCLAW_VLLM_MODEL:-}" "\${NEMOCLAW_POLICY_MODE:-}" "\${NEMOCLAW_YES:-}" "\${NEMOCLAW_SANDBOX_NAME:-}" + "\${NEMOCLAW_VLLM_MODEL:-}" "\${NEMOCLAW_VLLM_PROFILE:-}" "\${NEMOCLAW_POLICY_MODE:-}" "\${NEMOCLAW_YES:-}" "\${NEMOCLAW_SANDBOX_NAME:-}" ''' env = dict(os.environ) env["INSTALLER_UNDER_TEST"] = installer @@ -158,6 +158,23 @@ detect_express_platform ); }); + it("parses and documents the experimental single-user Station profile", () => { + const result = spawnSync( + "bash", + [INSTALLER_PAYLOAD, "--station-experimental-single-user", "--help"], + { + cwd: path.join(import.meta.dirname, ".."), + encoding: "utf-8", + }, + ); + const output = `${result.stdout}${result.stderr}`; + + expect(result.status, output).toBe(0); + expect(output).toMatch( + /--station-experimental-single-user\s+Use the experimental single-user DGX Station profile/, + ); + }); + it("offers express install when curl-piped stdin still has a controlling TTY", () => { const result = runExpressPromptWithTty("y\n", "pipe"); const output = `${result.stdout}${result.stderr}`; @@ -174,7 +191,7 @@ detect_express_platform expect(output).toMatch(/Run express install/); expect(output).toMatch(/Using express install for DGX Spark/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL= POLICY=suggested YES=1 SANDBOX=my-assistant/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL= VLLM_PROFILE= POLICY=suggested YES=1 SANDBOX=my-assistant/, ); }); @@ -192,7 +209,7 @@ detect_express_platform /Managed vLLM pulls the configured vLLM image\/model and runs a local vLLM inference container/, ); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL=custom-qwen3\.6 POLICY=suggested YES=1 SANDBOX=my-assistant/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL=custom-qwen3\.6 VLLM_PROFILE= POLICY=suggested YES=1 SANDBOX=my-assistant/, ); }); @@ -205,7 +222,7 @@ detect_express_platform expect(output).toMatch(/Detected DGX Spark/); expect(output).toMatch(/Sandbox name: custom-spark/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL= POLICY=suggested YES=1 SANDBOX=custom-spark/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL= VLLM_PROFILE= POLICY=suggested YES=1 SANDBOX=custom-spark/, ); }); @@ -220,10 +237,121 @@ detect_express_platform expect(output).toMatch(/approximately 352 GB model/); expect(output).toMatch(/Using express install for DGX Station/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b POLICY=suggested YES=1 SANDBOX=my-assistant/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b VLLM_PROFILE= POLICY=suggested YES=1 SANDBOX=my-assistant/, + ); + }); + + it("normalizes the canonical Ultra served alias to the registered model slug", () => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { + NEMOCLAW_VLLM_MODEL: "nvidia/nemotron-3-ultra-550b-a55b", + }); + const output = `${result.stdout}${result.stderr}`; + + expect(result.status, output).toBe(0); + expect(output).toMatch( + /MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b VLLM_PROFILE=/, + ); + }); + + it("uses the opt-in experimental single-user Station profile with one confirmation", () => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { + STATION_EXPERIMENTAL_SINGLE_USER: "1", + }); + const output = `${result.stdout}${result.stderr}`; + expect(result.status, output).toBe(0); + expect(output).toMatch( + /managed local vLLM with the experimental single-user Nemotron Ultra profile/, + ); + expect(output).toMatch(/This opt-in profile uses the qualified single-user Station runtime/); + expect(output.match(/Run express install with these settings\?/g)).toHaveLength(1); + expect(output).toMatch(/Using express install for DGX Station/); + expect(output).toMatch( + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=nemotron-ultra VLLM_MODEL=nemotron-3-ultra-550b-a55b VLLM_PROFILE=experimental-single-user POLICY=suggested YES=1 SANDBOX=my-assistant/, + ); + }); + + it("accepts matching experimental Station model and profile selectors", () => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { + STATION_EXPERIMENTAL_SINGLE_USER: "1", + NEMOCLAW_VLLM_MODEL: "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", + NEMOCLAW_MODEL: "nemotron-ultra", + NEMOCLAW_VLLM_PROFILE: "experimental-single-user", + }); + const output = `${result.stdout}${result.stderr}`; + expect(result.status, output).toBe(0); + expect(output).toMatch( + /MODEL=nemotron-ultra VLLM_MODEL=nemotron-3-ultra-550b-a55b VLLM_PROFILE=experimental-single-user/, ); }); + it("rejects combining the experimental Station profile with the DeepSeek override", () => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { + STATION_EXPERIMENTAL_SINGLE_USER: "1", + STATION_DEEPSEEK: "1", + }); + const output = `${result.stdout}${result.stderr}`; + expect(result.status, output).not.toBe(0); + expect(output).toMatch( + /--station-experimental-single-user cannot be combined with --station-deepseek/, + ); + expect(output).not.toMatch(/Run express install/); + }); + + it("rejects the experimental single-user profile on non-Station platforms", () => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Spark", { + STATION_EXPERIMENTAL_SINGLE_USER: "1", + }); + const output = `${result.stdout}${result.stderr}`; + expect(result.status, output).not.toBe(0); + expect(output).toMatch( + /--station-experimental-single-user requires a detected DGX Station \(detected: DGX Spark\)/, + ); + expect(output).not.toMatch(/Run express install/); + }); + + it.each([ + ["NEMOCLAW_NO_EXPRESS", "1", /cannot be combined with NEMOCLAW_NO_EXPRESS=1/], + ["NON_INTERACTIVE", "1", /cannot be combined with --non-interactive/], + ["NEMOCLAW_PROVIDER", "install-vllm", /conflicts with NEMOCLAW_PROVIDER=install-vllm/], + ])("rejects %s when the experimental Station profile would otherwise be ignored", (name, value, message) => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { + STATION_EXPERIMENTAL_SINGLE_USER: "1", + [name]: value, + }); + const output = `${result.stdout}${result.stderr}`; + expect(result.status, output).not.toBe(0); + expect(output).toMatch(message); + expect(output).not.toMatch(/Run express install/); + }); + + it.each([ + [ + "NEMOCLAW_VLLM_MODEL", + "deepseek-v4-flash", + /conflicts with NEMOCLAW_VLLM_MODEL='deepseek-v4-flash'/, + ], + ["NEMOCLAW_MODEL", "other-served-model", /conflicts with NEMOCLAW_MODEL='other-served-model'/], + [ + "NEMOCLAW_VLLM_PROFILE", + "another-profile", + /conflicts with NEMOCLAW_VLLM_PROFILE='another-profile'/, + ], + [ + "NEMOCLAW_VLLM_EXTRA_ARGS_JSON", + '["--enable-prefix-caching"]', + /cannot be combined with NEMOCLAW_VLLM_EXTRA_ARGS_JSON/, + ], + ])("rejects conflicting %s for the qualified experimental Station profile", (name, value, message) => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { + STATION_EXPERIMENTAL_SINGLE_USER: "1", + [name]: value, + }); + const output = `${result.stdout}${result.stderr}`; + expect(result.status, output).not.toBe(0); + expect(output).toMatch(message); + expect(output).not.toMatch(/Run express install/); + }); + it("uses DeepSeek V4 Flash for the Station demo override with one confirmation", () => { const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { STATION_DEEPSEEK: "1", @@ -236,7 +364,7 @@ detect_express_platform expect(output.match(/Run express install with these settings\?/g)).toHaveLength(1); expect(output).toMatch(/Using express install for DGX Station/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=deepseek-ai\/DeepSeek-V4-Flash VLLM_MODEL=deepseek-v4-flash POLICY=suggested YES=1 SANDBOX=my-assistant/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=deepseek-ai\/DeepSeek-V4-Flash VLLM_MODEL=deepseek-v4-flash VLLM_PROFILE= POLICY=suggested YES=1 SANDBOX=my-assistant/, ); }); @@ -301,7 +429,7 @@ detect_express_platform expect(output).toMatch(/pulls the configured vLLM image\/model/); expect(output).not.toMatch(/approximately 352 GB model/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL=custom-station-model POLICY=suggested YES=1 SANDBOX=my-assistant/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL=custom-station-model VLLM_PROFILE= POLICY=suggested YES=1 SANDBOX=my-assistant/, ); }); @@ -314,7 +442,7 @@ detect_express_platform expect(output).toMatch(/managed local vLLM with NVIDIA Nemotron 3 Ultra 550B/); expect(output).toMatch(/approximately 352 GB model/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b POLICY=suggested YES=1 SANDBOX=my-assistant/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b VLLM_PROFILE= POLICY=suggested YES=1 SANDBOX=my-assistant/, ); }); @@ -372,7 +500,7 @@ detect_express_platform expect(output).toMatch(/Run express install/); expect(output).toMatch(/Using express install for Windows WSL/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-windows-ollama MODEL= VLLM_MODEL= POLICY=suggested YES=1 SANDBOX=/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-windows-ollama MODEL= VLLM_MODEL= VLLM_PROFILE= POLICY=suggested YES=1 SANDBOX=/, ); }); @@ -392,9 +520,9 @@ NON_INTERACTIVE="" NEMOCLAW_PROVIDER="" NEMOCLAW_NO_EXPRESS="" maybe_offer_express_install -printf "RESULT NON_INTERACTIVE=%s SUDO_MODE=%s PROVIDER=%s MODEL=%s VLLM_MODEL=%s POLICY=%s YES=%s SANDBOX=%s\\n" \\ +printf "RESULT NON_INTERACTIVE=%s SUDO_MODE=%s PROVIDER=%s MODEL=%s VLLM_MODEL=%s VLLM_PROFILE=%s POLICY=%s YES=%s SANDBOX=%s\\n" \\ "\${NON_INTERACTIVE:-}" "\${NEMOCLAW_NON_INTERACTIVE_SUDO_MODE:-}" "\${NEMOCLAW_PROVIDER:-}" "\${NEMOCLAW_MODEL:-}" \\ - "\${NEMOCLAW_VLLM_MODEL:-}" "\${NEMOCLAW_POLICY_MODE:-}" "\${NEMOCLAW_YES:-}" "\${NEMOCLAW_SANDBOX_NAME:-}" + "\${NEMOCLAW_VLLM_MODEL:-}" "\${NEMOCLAW_VLLM_PROFILE:-}" "\${NEMOCLAW_POLICY_MODE:-}" "\${NEMOCLAW_YES:-}" "\${NEMOCLAW_SANDBOX_NAME:-}" `, ], { @@ -414,7 +542,7 @@ printf "RESULT NON_INTERACTIVE=%s SUDO_MODE=%s PROVIDER=%s MODEL=%s VLLM_MODEL=% expect(output).toMatch(/Skipping express prompt \(no TTY\)/); expect(output).not.toMatch(/Run express install/); expect(output).toMatch( - /RESULT NON_INTERACTIVE= SUDO_MODE= PROVIDER= MODEL= VLLM_MODEL= POLICY= YES= SANDBOX=/, + /RESULT NON_INTERACTIVE= SUDO_MODE= PROVIDER= MODEL= VLLM_MODEL= VLLM_PROFILE= POLICY= YES= SANDBOX=/, ); }, ); diff --git a/test/onboard-selection-vllm.test.ts b/test/onboard-selection-vllm.test.ts index f6beebf0e6f..e926aa53462 100644 --- a/test/onboard-selection-vllm.test.ts +++ b/test/onboard-selection-vllm.test.ts @@ -703,7 +703,7 @@ const { setupNim } = require(${onboardPath}); ); assert.match( result.stderr, - /unset NEMOCLAW_PROVIDER NEMOCLAW_MODEL NEMOCLAW_VLLM_MODEL\s+nemoclaw onboard --fresh\s+Then select Local vLLM when prompted\./, + /unset NEMOCLAW_PROVIDER NEMOCLAW_MODEL NEMOCLAW_VLLM_MODEL NEMOCLAW_VLLM_PROFILE\s+nemoclaw onboard --fresh\s+Then select Local vLLM when prompted\./, ); }); }); diff --git a/test/seed-hermes-dashboard-config.test.ts b/test/seed-hermes-dashboard-config.test.ts index 52d56e6167e..f2506fc3130 100644 --- a/test/seed-hermes-dashboard-config.test.ts +++ b/test/seed-hermes-dashboard-config.test.ts @@ -122,6 +122,59 @@ describe.skipIf(!PY_YAML_AVAILABLE)("seed-dashboard-config.py", () => { expect(dash._nemoclaw_upstream).toEqual(GATEWAY_CONFIG._nemoclaw_upstream); }); + it("mirrors the experimental Station Ultra route and effective context", () => { + const stationUltraGateway = { + _config_version: 32, + _nemoclaw_upstream: { provider: "vllm-local", model: "nemotron-ultra" }, + model: { + default: "nemotron-ultra", + provider: "custom", + base_url: "https://inference.local/v1", + api_key: "sk-OPENSHELL-PROXY-REWRITE", + context_length: 262144, + }, + providers: { + "vllm-local": { + name: "vllm-local", + api: "https://inference.local/v1", + api_key: "sk-OPENSHELL-PROXY-REWRITE", + default_model: "nemotron-ultra", + discover_models: true, + }, + }, + custom_providers: [ + { + name: "vllm-local", + base_url: "https://inference.local/v1", + api_key: "sk-OPENSHELL-PROXY-REWRITE", + discover_models: true, + }, + ], + }; + const src = writeYaml("station-ultra-gateway.yaml", stationUltraGateway); + const dst = path.join(tmpDir, "station-ultra-dashboard.yaml"); + + const res = runSeed(src, dst); + expect(res.status).toBe(0); + + const dash = readYaml(dst) as Record; + expect(dash._nemoclaw_upstream).toEqual({ + provider: "vllm-local", + model: "nemotron-ultra", + }); + expect(dash.model).toMatchObject({ + default: "nemotron-ultra", + provider: "vllm-local", + base_url: "https://inference.local/v1", + context_length: 262144, + }); + expect(dash.providers["vllm-local"]).toMatchObject({ + name: "vllm-local", + default_model: "nemotron-ultra", + discover_models: true, + }); + }); + it("mirrors only the exact native Tavily backend into dashboard config", () => { const src = writeYaml("gw.yaml", { ...GATEWAY_CONFIG, From f1e41f8d88aff908c934901807c110e7ace4c348 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 09:45:17 -0700 Subject: [PATCH 10/21] test(inference): keep runtime mocks linear Signed-off-by: Aaron Erickson --- .../vllm-experimental-single-user.test.ts | 37 ++++++++++++------- src/lib/inference/vllm.test.ts | 18 ++++++--- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/lib/inference/vllm-experimental-single-user.test.ts b/src/lib/inference/vllm-experimental-single-user.test.ts index 7cbdb9b13e4..d3f6436130c 100644 --- a/src/lib/inference/vllm-experimental-single-user.test.ts +++ b/src/lib/inference/vllm-experimental-single-user.test.ts @@ -111,17 +111,23 @@ function mockSuccessfulVllmInstall( const qualificationSamples = [...(endpoints.qualificationSamples ?? ["1000, 0, 0"])]; let lastQualificationSample = qualificationSamples.at(-1) ?? ""; mocks.runCapture.mockImplementation((cmd: readonly string[]) => { - if (cmd[0] === "sh") return "/usr/bin/tool\n"; - if (cmd[0] === "nvidia-smi") { - const sample = qualificationSamples.shift() ?? lastQualificationSample; - lastQualificationSample = sample; - return sample; + switch (cmd[0]) { + case "sh": + return "/usr/bin/tool\n"; + case "nvidia-smi": { + const sample = qualificationSamples.shift() ?? lastQualificationSample; + lastQualificationSample = sample; + return sample; + } + case "curl": { + const url = cmd.at(-1) ?? ""; + return url.endsWith("/health") + ? (endpoints.healthStatus ?? "") + : (endpoints.modelsResponse ?? '{"data":[]}'); + } + default: + return ""; } - if (cmd[0] !== "curl") return ""; - const url = cmd.at(-1) ?? ""; - return url.endsWith("/health") - ? (endpoints.healthStatus ?? "") - : (endpoints.modelsResponse ?? '{"data":[]}'); }); mocks.dockerPullWithProgressWatchdog.mockResolvedValue({ status: 0, @@ -134,9 +140,14 @@ function mockSuccessfulVllmInstall( mocks.dockerRunDetached.mockReturnValue({ status: 0, stdout: "", stderr: "", error: null }); const ownershipResponses: (() => string)[] = [() => "", () => ""]; mocks.dockerCapture.mockImplementation((args: readonly string[]) => { - if (args[0] === "container") return (ownershipResponses.shift() ?? (() => ""))(); - if (args[0] === "ps") return `${containerName}\n`; - return ""; + switch (args[0]) { + case "container": + return (ownershipResponses.shift() ?? (() => ""))(); + case "ps": + return `${containerName}\n`; + default: + return ""; + } }); } diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index fa460f1cdbf..fdb96706109 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -121,12 +121,18 @@ function mockSuccessfulVllmInstall( } = {}, ): void { mocks.runCapture.mockImplementation((cmd: readonly string[]) => { - if (cmd[0] === "sh") return "/usr/bin/tool\n"; - if (cmd[0] !== "curl") return ""; - const url = cmd.at(-1) ?? ""; - return url.endsWith("/health") - ? (endpoints.healthStatus ?? "") - : (endpoints.modelsResponse ?? '{"data":[]}'); + switch (cmd[0]) { + case "sh": + return "/usr/bin/tool\n"; + case "curl": { + const url = cmd.at(-1) ?? ""; + return url.endsWith("/health") + ? (endpoints.healthStatus ?? "") + : (endpoints.modelsResponse ?? '{"data":[]}'); + } + default: + return ""; + } }); mocks.dockerPullWithProgressWatchdog.mockResolvedValue({ status: 0, From afdb8ef3b9d0313df8a99d8f9ee3f0868e49e7a5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 09:53:26 -0700 Subject: [PATCH 11/21] docs(installer): keep Station flags out of CLI parity Signed-off-by: Aaron Erickson --- docs/reference/commands.mdx | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 5fc39dabfc9..3cf49f45c76 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -477,23 +477,6 @@ For scripted installer runs, pass explicit acceptance to the `bash` side of the curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 bash ``` -On a detected DGX Station, the installer also accepts these mutually exclusive express-install selections: - -| Installer flag | Effect | -|---|---| -| `--station-deepseek` | Select DeepSeek V4 Flash while retaining the Station express flow. | -| `--station-experimental-single-user` | Explicitly opt in to the reclaimed-v1/offload90 Nemotron Ultra profile. The canonical Station recipe remains the default when this flag is absent. | - -Pass installer arguments after `bash -s --`: - -```bash -curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-experimental-single-user -``` - -The experimental flag requires the exact qualified local Docker image ID `sha256:e44830d3f0197850e5a63ca895b13a047f358ff4aea15b1655abf61391bb883e` to be preloaded. -No pullable repository manifest is currently proven for that image, so fresh installs fail closed instead of substituting another vLLM runtime. -Refer to [Set Up vLLM](../inference/local-inference/set-up-vllm#opt-in-to-the-experimental-single-user-station-profile) for the exact runtime contract and image check. - If the installer cannot prompt for the notice in a terminal and no explicit acceptance is set, it exits before installing Node.js or the NemoClaw CLI. From a6041c3163c3c64c440eeb79baf52e3902086245 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 09:55:36 -0700 Subject: [PATCH 12/21] fix(installer): reject conflicting Station profiles Signed-off-by: Aaron Erickson --- scripts/install.sh | 3 +++ test/install-express-prompt.test.ts | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/scripts/install.sh b/scripts/install.sh index 46654d67fb1..e381919329f 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2691,6 +2691,9 @@ validate_station_deepseek_override() { if [ -n "${NEMOCLAW_PROVIDER:-}" ]; then error "--station-deepseek conflicts with NEMOCLAW_PROVIDER=${NEMOCLAW_PROVIDER}. Remove the provider override to use Station express install." fi + if [ -n "$(printf "%s" "${NEMOCLAW_VLLM_PROFILE:-}" | tr -d '[:space:]')" ]; then + error "--station-deepseek conflicts with NEMOCLAW_VLLM_PROFILE='${NEMOCLAW_VLLM_PROFILE}'. Remove the profile override to use the DeepSeek Station recipe." + fi local requested_model requested_model="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_MODEL:-}")" diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index 651e5f999b7..949cad03a49 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -392,6 +392,19 @@ detect_express_platform expect(output).not.toMatch(/Run express install/); }); + it("rejects an inherited vLLM profile with the Station demo override", () => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { + STATION_DEEPSEEK: "1", + NEMOCLAW_VLLM_PROFILE: "experimental-single-user", + }); + const output = `${result.stdout}${result.stderr}`; + expect(result.status, output).not.toBe(0); + expect(output).toMatch( + /--station-deepseek conflicts with NEMOCLAW_VLLM_PROFILE='experimental-single-user'/, + ); + expect(output).not.toMatch(/Run express install/); + }); + it("rejects the Station demo override on non-Station platforms", () => { const result = runExpressPromptWithTty("\n", "pipe", "DGX Spark", { STATION_DEEPSEEK: "1", From 352ce0a41c42733104aa712c157814909b59724b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 09:57:22 -0700 Subject: [PATCH 13/21] docs(inference): warn on experimental host exposure Signed-off-by: Aaron Erickson --- docs/inference/set-up-vllm.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 39ffde8981c..530afa3575d 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -189,6 +189,7 @@ The profile uses the reclaimed-v1/offload90 configuration with this model identi | Maximum context | `262144` | Its container runs with host networking and host IPC, `16 GiB` shared memory, a `650 GiB` container memory limit, tensor parallel size `1`, and a `1200`-second startup timeout. +Because the qualified contract binds the unauthenticated vLLM API to `0.0.0.0:8000` on the host network, apply the default-deny firewall rules at the top of this page before opting in; this profile does not install or modify host firewall rules. The installer fails closed unless exactly one GB300 is available, HBM is below `245000 MiB`, and aggregate and volatile uncorrected ECC are zero before launch; it samples the same exclusive HBM ceiling and ECC contract throughout startup. Full workload-level HBM/ECC qualification through chat and tool calls remains part of the blocked current-PR physical Station acceptance run. From 2fc1278b7af600a8838f07bdb2b205116a5a2276 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 10:00:01 -0700 Subject: [PATCH 14/21] test(inference): tag experimental Station suites Signed-off-by: Aaron Erickson --- src/lib/inference/vllm-experimental-single-user.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/inference/vllm-experimental-single-user.test.ts b/src/lib/inference/vllm-experimental-single-user.test.ts index d3f6436130c..91f1152f6bd 100644 --- a/src/lib/inference/vllm-experimental-single-user.test.ts +++ b/src/lib/inference/vllm-experimental-single-user.test.ts @@ -151,7 +151,7 @@ function mockSuccessfulVllmInstall( }); } -describe("experimental single-user vLLM runtime profile", () => { +describe("experimental single-user vLLM runtime profile (#6883)", () => { beforeEach(() => { vi.clearAllMocks(); }); @@ -230,7 +230,7 @@ describe("experimental single-user vLLM runtime profile", () => { }); }); -describe("experimental single-user vLLM install", () => { +describe("experimental single-user vLLM install (#6883)", () => { let errSpy: ReturnType; let mkdirSpy: ReturnType; let stdoutWrite: ReturnType; From b3b895b613f03587b0ecb276a98213756ae869c6 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 10:05:55 -0700 Subject: [PATCH 15/21] fix(inference): enforce experimental runtime invariants Signed-off-by: Aaron Erickson --- docs/inference/set-up-vllm.mdx | 1 + scripts/install.sh | 20 ++++++++++ .../vllm-experimental-single-user.test.ts | 25 ++++++++++++ src/lib/inference/vllm-models.test.ts | 19 +++++++++ src/lib/inference/vllm-models.ts | 39 ++++++++++++++++++- test/install-express-prompt.test.ts | 4 ++ 6 files changed, 107 insertions(+), 1 deletion(-) diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 530afa3575d..5a5635dc878 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -166,6 +166,7 @@ curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-experimenta ``` The installer maps this flag to `NEMOCLAW_VLLM_PROFILE=experimental-single-user` and rejects conflicting model, profile, extra-argument, DeepSeek, and non-Station selections. +It also rejects ambient `NEMOCLAW_VLLM_PORT` values other than `8000` and `NEMOCLAW_CONTEXT_WINDOW` values other than `262144`, so host readiness, gateway routing, and the dashboard cannot drift from the qualified contract. The qualified runtime exists today only as local Docker image ID `sha256:e44830d3f0197850e5a63ca895b13a047f358ff4aea15b1655abf61391bb883e`. diff --git a/scripts/install.sh b/scripts/install.sh index e381919329f..c7a64b1e726 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2669,6 +2669,8 @@ STATION_DEEPSEEK_VLLM_MODEL="deepseek-v4-flash" STATION_DEEPSEEK_SERVED_MODEL="deepseek-ai/DeepSeek-V4-Flash" STATION_EXPERIMENTAL_SINGLE_USER_PROFILE="experimental-single-user" STATION_EXPERIMENTAL_SINGLE_USER_SERVED_MODEL="nemotron-ultra" +STATION_EXPERIMENTAL_SINGLE_USER_PORT="8000" +STATION_EXPERIMENTAL_SINGLE_USER_CONTEXT_WINDOW="262144" normalize_station_vllm_model() { printf "%s" "${1:-}" | tr '[:upper:]' '[:lower:]' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' @@ -2753,6 +2755,24 @@ validate_station_experimental_single_user_override() { ;; esac + local requested_port + requested_port="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_PORT:-}")" + case "$requested_port" in + "" | "$STATION_EXPERIMENTAL_SINGLE_USER_PORT") ;; + *) + error "--station-experimental-single-user conflicts with NEMOCLAW_VLLM_PORT='${NEMOCLAW_VLLM_PORT}'. Remove the override or set NEMOCLAW_VLLM_PORT=${STATION_EXPERIMENTAL_SINGLE_USER_PORT}." + ;; + esac + + local requested_context_window + requested_context_window="$(normalize_station_vllm_model "${NEMOCLAW_CONTEXT_WINDOW:-}")" + case "$requested_context_window" in + "" | "$STATION_EXPERIMENTAL_SINGLE_USER_CONTEXT_WINDOW") ;; + *) + error "--station-experimental-single-user conflicts with NEMOCLAW_CONTEXT_WINDOW='${NEMOCLAW_CONTEXT_WINDOW}'. Remove the override or set NEMOCLAW_CONTEXT_WINDOW=${STATION_EXPERIMENTAL_SINGLE_USER_CONTEXT_WINDOW}." + ;; + esac + if [ -n "$(printf "%s" "${NEMOCLAW_VLLM_EXTRA_ARGS_JSON:-}" | tr -d '[:space:]')" ]; then error "--station-experimental-single-user cannot be combined with NEMOCLAW_VLLM_EXTRA_ARGS_JSON. The qualified profile supplies the exact vLLM arguments." fi diff --git a/src/lib/inference/vllm-experimental-single-user.test.ts b/src/lib/inference/vllm-experimental-single-user.test.ts index 91f1152f6bd..d011e46b018 100644 --- a/src/lib/inference/vllm-experimental-single-user.test.ts +++ b/src/lib/inference/vllm-experimental-single-user.test.ts @@ -245,6 +245,8 @@ describe("experimental single-user vLLM install (#6883)", () => { process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; process.env.NEMOCLAW_VLLM_PROFILE = EXPERIMENTAL_SINGLE_USER_PROFILE; delete process.env.NEMOCLAW_VLLM_EXTRA_ARGS_JSON; + delete process.env.NEMOCLAW_VLLM_PORT; + delete process.env.NEMOCLAW_CONTEXT_WINDOW; delete process.env.NEMOCLAW_IGNORE_VLLM_DISK_SPACE; delete process.env.HF_TOKEN; delete process.env.HUGGING_FACE_HUB_TOKEN; @@ -257,6 +259,29 @@ describe("experimental single-user vLLM install (#6883)", () => { process.env = { ...originalEnv }; }); + it.each([ + ["NEMOCLAW_VLLM_PORT", "9000", "requires NEMOCLAW_VLLM_PORT=8000"], + ["NEMOCLAW_CONTEXT_WINDOW", "8192", "requires NEMOCLAW_CONTEXT_WINDOW=262144"], + ])("rejects conflicting %s before managed-vLLM side effects", async (name, value, message) => { + process.env[name] = value; + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + const beforeInstall = vi.fn(); + mockSuccessfulVllmInstall(profile.containerName); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + beforeInstall, + }); + + expect(result).toEqual({ ok: false }); + expect(beforeInstall).not.toHaveBeenCalled(); + expect(mocks.probeHostStorage).not.toHaveBeenCalled(); + expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); + expect(errSpy.mock.calls.flat().join("\n")).toContain(message); + }); + it("fails closed before downloads when the qualified image is absent", async () => { const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; const beforeInstall = vi.fn(); diff --git a/src/lib/inference/vllm-models.test.ts b/src/lib/inference/vllm-models.test.ts index 1960f1667ef..9021699476c 100644 --- a/src/lib/inference/vllm-models.test.ts +++ b/src/lib/inference/vllm-models.test.ts @@ -127,6 +127,7 @@ describe("vllm model registry", () => { platforms: ["station"], servedModelId: "nemotron-ultra", revision: "183968f87ae4cedce3039313cac1fd43d112c578", + requiredVllmPort: 8000, exactServeArgs: [ "--served-model-name", "nemotron-ultra", @@ -562,6 +563,8 @@ describe("preflightVllmModelEnv", () => { preflightVllmModelEnv({ NEMOCLAW_VLLM_MODEL: "nemotron-3-ultra-550b-a55b", [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, + NEMOCLAW_VLLM_PORT: "8000", + NEMOCLAW_CONTEXT_WINDOW: "262144", } as NodeJS.ProcessEnv), ).toEqual({ ok: true }); }); @@ -611,6 +614,22 @@ describe("preflightVllmModelEnv", () => { }); }); + it.each([ + ["NEMOCLAW_VLLM_PORT", "9000", /requires NEMOCLAW_VLLM_PORT=8000/], + ["NEMOCLAW_CONTEXT_WINDOW", "8192", /requires NEMOCLAW_CONTEXT_WINDOW=262144/], + ])("rejects conflicting qualified-profile runtime input %s", (name, value, message) => { + expect( + preflightVllmModelEnv({ + NEMOCLAW_VLLM_MODEL: "nemotron-3-ultra-550b-a55b", + [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, + [name]: value, + } as NodeJS.ProcessEnv), + ).toEqual({ + ok: false, + message: expect.stringMatching(message), + }); + }); + it("succeeds for a gated slug when HF_TOKEN is set", () => { expect( preflightVllmModelEnv({ diff --git a/src/lib/inference/vllm-models.ts b/src/lib/inference/vllm-models.ts index a42c924bf2f..5eb9433f04b 100644 --- a/src/lib/inference/vllm-models.ts +++ b/src/lib/inference/vllm-models.ts @@ -90,6 +90,8 @@ export interface VllmModelProfileOverride { servedModelId: string; /** Immutable Hugging Face revision owned by this profile. */ revision: string; + /** Host port that must stay aligned with this profile's exact serve arguments. */ + requiredVllmPort: number; /** Complete ordered vLLM arguments after the model ID. Replaces shared/default args. */ exactServeArgs: readonly string[]; /** Required runtime/container replacement for this profile. */ @@ -328,6 +330,7 @@ export const VLLM_MODELS: readonly VllmModelDef[] = [ platforms: ["station"], servedModelId: "nemotron-ultra", revision: "183968f87ae4cedce3039313cac1fd43d112c578", + requiredVllmPort: 8000, exactServeArgs: [ "--served-model-name", "nemotron-ultra", @@ -498,6 +501,37 @@ function huggingFaceSnapshotContainerPath(modelId: string, revision: string): st ); } +function assertVllmModelProfileRuntimeInputs( + profile: VllmModelProfileOverride, + env: NodeJS.ProcessEnv, +): void { + const requestedPort = String(env.NEMOCLAW_VLLM_PORT ?? "").trim(); + if ( + requestedPort && + (!/^[0-9]+$/.test(requestedPort) || Number(requestedPort) !== profile.requiredVllmPort) + ) { + throw new Error( + `${VLLM_PROFILE_ENV}='${profile.id}' requires ` + + `NEMOCLAW_VLLM_PORT=${String(profile.requiredVllmPort)}; remove the override or set it to ` + + `${String(profile.requiredVllmPort)}.`, + ); + } + + const requiredContextWindow = profile.readiness.maxModelLen; + const requestedContextWindow = String(env.NEMOCLAW_CONTEXT_WINDOW ?? "").trim(); + if ( + requestedContextWindow && + (!/^[0-9]+$/.test(requestedContextWindow) || + Number(requestedContextWindow) !== requiredContextWindow) + ) { + throw new Error( + `${VLLM_PROFILE_ENV}='${profile.id}' requires ` + + `NEMOCLAW_CONTEXT_WINDOW=${String(requiredContextWindow)}; remove the override or set it to ` + + `${String(requiredContextWindow)}.`, + ); + } +} + /** * Resolve an explicit model-scoped profile into the effective recipe consumed * by the existing managed-vLLM lifecycle. With no selector this returns the @@ -535,6 +569,7 @@ export function resolveVllmModelProfile( "; extra arguments would invalidate the qualified runtime contract.", ); } + assertVllmModelProfileRuntimeInputs(profile, env); return { profile, @@ -650,7 +685,8 @@ export function preflightVllmModelEnv( } return { ok: true }; } - if (profileId && !model.profiles?.[profileId]) { + const modelProfile = profileId ? model.profiles?.[profileId] : undefined; + if (profileId && !modelProfile) { throw new Error( VLLM_PROFILE_ENV + "='" + @@ -658,6 +694,7 @@ export function preflightVllmModelEnv( "' requires NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b'.", ); } + if (modelProfile) assertVllmModelProfileRuntimeInputs(modelProfile, env); assertGatedModelAccess(model, env); return { ok: true }; } catch (err) { diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index 949cad03a49..23b8355686b 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -276,6 +276,8 @@ detect_express_platform NEMOCLAW_VLLM_MODEL: "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", NEMOCLAW_MODEL: "nemotron-ultra", NEMOCLAW_VLLM_PROFILE: "experimental-single-user", + NEMOCLAW_VLLM_PORT: "8000", + NEMOCLAW_CONTEXT_WINDOW: "262144", }); const output = `${result.stdout}${result.stderr}`; expect(result.status, output).toBe(0); @@ -341,6 +343,8 @@ detect_express_platform '["--enable-prefix-caching"]', /cannot be combined with NEMOCLAW_VLLM_EXTRA_ARGS_JSON/, ], + ["NEMOCLAW_VLLM_PORT", "9000", /conflicts with NEMOCLAW_VLLM_PORT='9000'/], + ["NEMOCLAW_CONTEXT_WINDOW", "8192", /conflicts with NEMOCLAW_CONTEXT_WINDOW='8192'/], ])("rejects conflicting %s for the qualified experimental Station profile", (name, value, message) => { const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { STATION_EXPERIMENTAL_SINGLE_USER: "1", From 31faae307dca220f088c9741910dfdb3c9454244 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 10:20:49 -0700 Subject: [PATCH 16/21] fix(installer): fail closed on experimental profile drift Signed-off-by: Aaron Erickson --- docs/inference/set-up-vllm.mdx | 3 +- scripts/install.sh | 10 ++++ src/lib/inference/local-host-url.ts | 8 ++++ src/lib/inference/local.ts | 4 +- .../vllm-experimental-single-user.test.ts | 8 ++++ src/lib/inference/vllm-models.test.ts | 9 ++++ src/lib/inference/vllm-models.ts | 30 ++++++++---- src/lib/onboard/setup-nim-flow.test.ts | 47 +++++++++++++++++++ src/lib/onboard/setup-nim-flow.ts | 9 ++-- src/lib/onboard/vllm-menu.test.ts | 27 +++++++++++ src/lib/onboard/vllm-menu.ts | 26 +++++++++- test/install-express-prompt.test.ts | 8 ++++ 12 files changed, 173 insertions(+), 16 deletions(-) create mode 100644 src/lib/inference/local-host-url.ts diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 5a5635dc878..53c0b40096c 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -166,7 +166,8 @@ curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-experimenta ``` The installer maps this flag to `NEMOCLAW_VLLM_PROFILE=experimental-single-user` and rejects conflicting model, profile, extra-argument, DeepSeek, and non-Station selections. -It also rejects ambient `NEMOCLAW_VLLM_PORT` values other than `8000` and `NEMOCLAW_CONTEXT_WINDOW` values other than `262144`, so host readiness, gateway routing, and the dashboard cannot drift from the qualified contract. +It also rejects ambient `NEMOCLAW_VLLM_PORT` values other than `8000`, `NEMOCLAW_CONTEXT_WINDOW` values other than `262144`, and `NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL` routes other than `http://host.openshell.internal`, so host readiness, gateway routing, and the dashboard cannot drift from the qualified contract. +An existing server on port `8000` is not reused for this profile; stop it before opting in so the existing managed-vLLM lifecycle can prove the qualified image, arguments, environment, and hardware limits. The qualified runtime exists today only as local Docker image ID `sha256:e44830d3f0197850e5a63ca895b13a047f358ff4aea15b1655abf61391bb883e`. diff --git a/scripts/install.sh b/scripts/install.sh index c7a64b1e726..cfbdf925a49 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2671,6 +2671,7 @@ STATION_EXPERIMENTAL_SINGLE_USER_PROFILE="experimental-single-user" STATION_EXPERIMENTAL_SINGLE_USER_SERVED_MODEL="nemotron-ultra" STATION_EXPERIMENTAL_SINGLE_USER_PORT="8000" STATION_EXPERIMENTAL_SINGLE_USER_CONTEXT_WINDOW="262144" +STATION_EXPERIMENTAL_SINGLE_USER_SANDBOX_HOST_URL="http://host.openshell.internal" normalize_station_vllm_model() { printf "%s" "${1:-}" | tr '[:upper:]' '[:lower:]' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' @@ -2773,6 +2774,15 @@ validate_station_experimental_single_user_override() { ;; esac + local requested_sandbox_host_url + requested_sandbox_host_url="$(printf "%s" "${NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL:-}" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//; s:/*$::')" + case "$requested_sandbox_host_url" in + "" | "$STATION_EXPERIMENTAL_SINGLE_USER_SANDBOX_HOST_URL" | "host.openshell.internal") ;; + *) + error "--station-experimental-single-user conflicts with NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL='${NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL}'. Remove the override or set NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL=${STATION_EXPERIMENTAL_SINGLE_USER_SANDBOX_HOST_URL}." + ;; + esac + if [ -n "$(printf "%s" "${NEMOCLAW_VLLM_EXTRA_ARGS_JSON:-}" | tr -d '[:space:]')" ]; then error "--station-experimental-single-user cannot be combined with NEMOCLAW_VLLM_EXTRA_ARGS_JSON. The qualified profile supplies the exact vLLM arguments." fi diff --git a/src/lib/inference/local-host-url.ts b/src/lib/inference/local-host-url.ts new file mode 100644 index 00000000000..9ad83d60bc1 --- /dev/null +++ b/src/lib/inference/local-host-url.ts @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Sandbox-to-host route used by NemoClaw's managed local inference providers. */ +export const HOST_GATEWAY_URL = "http://host.openshell.internal"; + +/** Advanced override for the sandbox-facing local inference host route. */ +export const LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV = "NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL"; diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 3f839927005..b91a585da60 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -19,6 +19,7 @@ import { sleepSeconds } from "../core/wait"; import { containerCanReachHostLoopback, isWsl } from "../platform"; import { type CaptureResult, runCapture, runCaptureEx, shellQuote } from "../runner"; import { buildSubprocessEnv } from "../subprocess-env"; +import { HOST_GATEWAY_URL, LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV } from "./local-host-url"; import { detectNvidiaPlatform } from "./nim"; import { anyRegistryModelFits, @@ -59,8 +60,7 @@ export function resetOllamaContainerPortCache(): void { _ollamaContainerPort = null; } -export const HOST_GATEWAY_URL = "http://host.openshell.internal"; -export const LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV = "NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL"; +export { HOST_GATEWAY_URL, LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV } from "./local-host-url"; export const CONTAINER_REACHABILITY_IMAGE = "curlimages/curl:8.10.1"; // These tags are convenience aliases for callers that want to refer to a // specific bootstrap model by role rather than by string. The canonical diff --git a/src/lib/inference/vllm-experimental-single-user.test.ts b/src/lib/inference/vllm-experimental-single-user.test.ts index d011e46b018..83aad759d02 100644 --- a/src/lib/inference/vllm-experimental-single-user.test.ts +++ b/src/lib/inference/vllm-experimental-single-user.test.ts @@ -247,6 +247,7 @@ describe("experimental single-user vLLM install (#6883)", () => { delete process.env.NEMOCLAW_VLLM_EXTRA_ARGS_JSON; delete process.env.NEMOCLAW_VLLM_PORT; delete process.env.NEMOCLAW_CONTEXT_WINDOW; + delete process.env.NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL; delete process.env.NEMOCLAW_IGNORE_VLLM_DISK_SPACE; delete process.env.HF_TOKEN; delete process.env.HUGGING_FACE_HUB_TOKEN; @@ -261,7 +262,14 @@ describe("experimental single-user vLLM install (#6883)", () => { it.each([ ["NEMOCLAW_VLLM_PORT", "9000", "requires NEMOCLAW_VLLM_PORT=8000"], + ["NEMOCLAW_VLLM_PORT", "08000", "requires NEMOCLAW_VLLM_PORT=8000"], ["NEMOCLAW_CONTEXT_WINDOW", "8192", "requires NEMOCLAW_CONTEXT_WINDOW=262144"], + ["NEMOCLAW_CONTEXT_WINDOW", "0262144", "requires NEMOCLAW_CONTEXT_WINDOW=262144"], + [ + "NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL", + "http://127.0.0.1", + "requires NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL=http://host.openshell.internal", + ], ])("rejects conflicting %s before managed-vLLM side effects", async (name, value, message) => { process.env[name] = value; const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; diff --git a/src/lib/inference/vllm-models.test.ts b/src/lib/inference/vllm-models.test.ts index 9021699476c..57b9bb2d564 100644 --- a/src/lib/inference/vllm-models.test.ts +++ b/src/lib/inference/vllm-models.test.ts @@ -128,6 +128,7 @@ describe("vllm model registry", () => { servedModelId: "nemotron-ultra", revision: "183968f87ae4cedce3039313cac1fd43d112c578", requiredVllmPort: 8000, + requiredSandboxHostUrl: "http://host.openshell.internal", exactServeArgs: [ "--served-model-name", "nemotron-ultra", @@ -565,6 +566,7 @@ describe("preflightVllmModelEnv", () => { [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, NEMOCLAW_VLLM_PORT: "8000", NEMOCLAW_CONTEXT_WINDOW: "262144", + NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL: "host.openshell.internal", } as NodeJS.ProcessEnv), ).toEqual({ ok: true }); }); @@ -616,7 +618,14 @@ describe("preflightVllmModelEnv", () => { it.each([ ["NEMOCLAW_VLLM_PORT", "9000", /requires NEMOCLAW_VLLM_PORT=8000/], + ["NEMOCLAW_VLLM_PORT", "08000", /requires NEMOCLAW_VLLM_PORT=8000/], ["NEMOCLAW_CONTEXT_WINDOW", "8192", /requires NEMOCLAW_CONTEXT_WINDOW=262144/], + ["NEMOCLAW_CONTEXT_WINDOW", "0262144", /requires NEMOCLAW_CONTEXT_WINDOW=262144/], + [ + "NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL", + "http://127.0.0.1", + /requires NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL=http:\/\/host\.openshell\.internal/, + ], ])("rejects conflicting qualified-profile runtime input %s", (name, value, message) => { expect( preflightVllmModelEnv({ diff --git a/src/lib/inference/vllm-models.ts b/src/lib/inference/vllm-models.ts index 5eb9433f04b..45c4c0a0f96 100644 --- a/src/lib/inference/vllm-models.ts +++ b/src/lib/inference/vllm-models.ts @@ -27,6 +27,8 @@ * envelope, and tool-call behaviour validated. */ +import { HOST_GATEWAY_URL, LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV } from "./local-host-url"; + export type VllmPlatform = "spark" | "station" | "linux"; export const VLLM_PROFILE_ENV = "NEMOCLAW_VLLM_PROFILE"; @@ -92,6 +94,8 @@ export interface VllmModelProfileOverride { revision: string; /** Host port that must stay aligned with this profile's exact serve arguments. */ requiredVllmPort: number; + /** Sandbox-facing host route required by this qualified profile. */ + requiredSandboxHostUrl: string; /** Complete ordered vLLM arguments after the model ID. Replaces shared/default args. */ exactServeArgs: readonly string[]; /** Required runtime/container replacement for this profile. */ @@ -331,6 +335,7 @@ export const VLLM_MODELS: readonly VllmModelDef[] = [ servedModelId: "nemotron-ultra", revision: "183968f87ae4cedce3039313cac1fd43d112c578", requiredVllmPort: 8000, + requiredSandboxHostUrl: HOST_GATEWAY_URL, exactServeArgs: [ "--served-model-name", "nemotron-ultra", @@ -506,10 +511,7 @@ function assertVllmModelProfileRuntimeInputs( env: NodeJS.ProcessEnv, ): void { const requestedPort = String(env.NEMOCLAW_VLLM_PORT ?? "").trim(); - if ( - requestedPort && - (!/^[0-9]+$/.test(requestedPort) || Number(requestedPort) !== profile.requiredVllmPort) - ) { + if (requestedPort && requestedPort !== String(profile.requiredVllmPort)) { throw new Error( `${VLLM_PROFILE_ENV}='${profile.id}' requires ` + `NEMOCLAW_VLLM_PORT=${String(profile.requiredVllmPort)}; remove the override or set it to ` + @@ -519,17 +521,27 @@ function assertVllmModelProfileRuntimeInputs( const requiredContextWindow = profile.readiness.maxModelLen; const requestedContextWindow = String(env.NEMOCLAW_CONTEXT_WINDOW ?? "").trim(); - if ( - requestedContextWindow && - (!/^[0-9]+$/.test(requestedContextWindow) || - Number(requestedContextWindow) !== requiredContextWindow) - ) { + if (requestedContextWindow && requestedContextWindow !== String(requiredContextWindow)) { throw new Error( `${VLLM_PROFILE_ENV}='${profile.id}' requires ` + `NEMOCLAW_CONTEXT_WINDOW=${String(requiredContextWindow)}; remove the override or set it to ` + `${String(requiredContextWindow)}.`, ); } + + const requestedSandboxHostUrl = String(env[LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV] ?? "").trim(); + const requiredSandboxHost = profile.requiredSandboxHostUrl.replace(/^http:\/\//, ""); + if ( + requestedSandboxHostUrl && + requestedSandboxHostUrl.replace(/\/+$/, "") !== profile.requiredSandboxHostUrl && + requestedSandboxHostUrl.replace(/\/+$/, "") !== requiredSandboxHost + ) { + throw new Error( + `${VLLM_PROFILE_ENV}='${profile.id}' requires ` + + `${LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV}=${profile.requiredSandboxHostUrl}; remove the ` + + `override or set it to ${profile.requiredSandboxHostUrl}.`, + ); + } } /** diff --git a/src/lib/onboard/setup-nim-flow.test.ts b/src/lib/onboard/setup-nim-flow.test.ts index 910a84107cb..7bb40d38d93 100644 --- a/src/lib/onboard/setup-nim-flow.test.ts +++ b/src/lib/onboard/setup-nim-flow.test.ts @@ -867,12 +867,59 @@ describe("createSetupNim", () => { await expect(setupNim(null)).rejects.toThrow("vLLM is already running on localhost:8000"); + expect(error).toHaveBeenCalledWith(expect.stringContaining("Select Local vLLM")); expect(error).toHaveBeenCalledWith(expect.stringContaining("stop the existing server")); expect(abortNonInteractive).toHaveBeenCalledOnce(); expect(installVllm).not.toHaveBeenCalled(); expect(handleVllmSelection).not.toHaveBeenCalled(); }); + it("requires the exact experimental managed runtime instead of suggesting existing local vLLM", async () => { + vi.stubEnv("NEMOCLAW_VLLM_PROFILE", "experimental-single-user"); + const profile = { + name: "DGX Station experimental single-user", + } as VllmProfile; + const error = vi.fn(); + const abortNonInteractive = vi.fn((message) => { + throw new Error(message); + }); + const installVllm = vi.fn(); + const handleVllmSelection = vi.fn(); + const setupNim = createSetupNim( + makeDeps({ + isNonInteractive: () => true, + getNonInteractiveProvider: () => "install-vllm", + error, + abortNonInteractive, + detectInferenceProviderHostState: () => + makeHostState({ + vllmRunning: true, + vllmProfile: profile, + hasVllmImage: true, + vllmEntries: [ + { + key: "install-vllm", + label: "Start vLLM (DGX Station experimental single-user)", + }, + ], + }), + installVllm, + handleVllmSelection, + }), + ); + + await expect(setupNim(null)).rejects.toThrow( + "The experimental single-user profile requires the qualified managed runtime", + ); + + const renderedError = String(error.mock.calls[0]?.[0]); + expect(renderedError).toContain("stop the existing server"); + expect(renderedError).not.toContain("Select Local vLLM"); + expect(abortNonInteractive).toHaveBeenCalledOnce(); + expect(installVllm).not.toHaveBeenCalled(); + expect(handleVllmSelection).not.toHaveBeenCalled(); + }); + it("returns interactive occupied-port selection to the provider menu", async () => { vi.stubEnv("NEMOCLAW_PROVIDER", ""); const profile = { name: "DGX Spark" } as VllmProfile; diff --git a/src/lib/onboard/setup-nim-flow.ts b/src/lib/onboard/setup-nim-flow.ts index 599371237fe..aea9f21ffd2 100644 --- a/src/lib/onboard/setup-nim-flow.ts +++ b/src/lib/onboard/setup-nim-flow.ts @@ -25,6 +25,7 @@ import { promptForInferenceProviderSelection } from "./provider-selection-prompt import type { RebuildRouteHandoff, RegistryInferenceRoute } from "./rebuild-route-handoff"; import { prepareProviderDiscovery } from "./setup-nim-provider-discovery"; import type { SetupNimSelectionState as BaseSetupNimSelectionState } from "./setup-nim-selection"; +import { requiresExactManagedVllm } from "./vllm-menu"; export type SetupNimGpu = ReturnType; export type SetupNimSelectionState = BaseSetupNimSelectionState; @@ -558,9 +559,11 @@ export function createSetupNim( continue selectionLoop; } if (vllmRunning) { - const message = - `vLLM is already running on localhost:${String(deps.vllmPort)}. ` + - "Select Local vLLM, or stop the existing server before selecting the managed install path."; + const message = requiresExactManagedVllm() + ? `vLLM is already running on localhost:${String(deps.vllmPort)}. ` + + "The experimental single-user profile requires the qualified managed runtime; stop the existing server before selecting the managed install path." + : `vLLM is already running on localhost:${String(deps.vllmPort)}. ` + + "Select Local vLLM, or stop the existing server before selecting the managed install path."; deps.error(` ${message}`); if (deps.isNonInteractive()) { deps.abortNonInteractive(message); diff --git a/src/lib/onboard/vllm-menu.test.ts b/src/lib/onboard/vllm-menu.test.ts index 3564eccce5a..5d35051e8cc 100644 --- a/src/lib/onboard/vllm-menu.test.ts +++ b/src/lib/onboard/vllm-menu.test.ts @@ -169,6 +169,33 @@ describe("buildVllmMenuEntries", () => { assert.match(logs[0], /selecting the running instance/); }); + it("keeps the exact experimental managed-install entry when vLLM is already running", () => { + const logs: string[] = []; + const entries = buildVllmMenuEntries({ + vllmRunning: true, + vllmProfile: { name: "DGX Station experimental single-user" }, + experimental: false, + platform: "station", + hasVllmImage: true, + env: { + NEMOCLAW_PROVIDER: "install-vllm", + NEMOCLAW_VLLM_PROFILE: "EXPERIMENTAL-SINGLE-USER", + }, + log: (message) => logs.push(message), + }); + + assert.deepEqual(entries, [ + { + key: "install-vllm", + label: "Start vLLM (DGX Station experimental single-user)", + }, + ]); + assert.equal(logs.length, 1); + assert.match(logs[0], /requires the qualified managed runtime/); + assert.match(logs[0], /existing vLLM instance on localhost:8000 cannot be reused/); + assert.doesNotMatch(logs[0], /selecting the running instance/); + }); + it("does not log the override note when the user did not request install-vllm", () => { const logs: string[] = []; buildVllmMenuEntries({ diff --git a/src/lib/onboard/vllm-menu.ts b/src/lib/onboard/vllm-menu.ts index 58709a87bee..6476684c4a8 100644 --- a/src/lib/onboard/vllm-menu.ts +++ b/src/lib/onboard/vllm-menu.ts @@ -21,10 +21,13 @@ * available for this host." message. It also lets the caller surface managed * vLLM by default for known DGX platforms while generic Linux stays gated, and * logs a note when running-vLLM takes precedence over the env-var opt-in. + * Exact qualified profiles are the exception: they retain the install entry so + * the FSM can reject an occupied port instead of reusing an unqualified server. */ import { VLLM_PORT } from "../core/ports"; import type { NvidiaPlatform } from "../inference/nim"; +import { EXPERIMENTAL_SINGLE_USER_PROFILE, VLLM_PROFILE_ENV } from "../inference/vllm-models"; interface VllmProfileShape { name: string; @@ -48,6 +51,20 @@ export interface BuildVllmMenuOptions { log?: (message: string) => void; } +/** + * Exact managed-runtime profiles cannot reuse an arbitrary listener on the + * local vLLM port. Their image, arguments, environment, and safety checks are + * part of the qualified contract, so onboarding must keep the managed-install + * transition and let it fail closed until the existing server is stopped. + */ +export function requiresExactManagedVllm(env: NodeJS.ProcessEnv = process.env): boolean { + return ( + String(env[VLLM_PROFILE_ENV] ?? "") + .trim() + .toLowerCase() === EXPERIMENTAL_SINGLE_USER_PROFILE + ); +} + export function buildVllmMenuEntries(opts: BuildVllmMenuOptions): VllmMenuEntry[] { const log = opts.log ?? console.log; // Read NEMOCLAW_PROVIDER directly so interactive runs with an explicit @@ -56,7 +73,8 @@ export function buildVllmMenuEntries(opts: BuildVllmMenuOptions): VllmMenuEntry[ const env = opts.env ?? process.env; const userChoseManagedVllm = (env.NEMOCLAW_PROVIDER || "").trim().toLowerCase() === "install-vllm"; - if (opts.vllmRunning) { + const exactManagedVllmRequired = requiresExactManagedVllm(env); + if (opts.vllmRunning && !exactManagedVllmRequired) { if (userChoseManagedVllm) { log( ` Note: NEMOCLAW_PROVIDER=install-vllm requested, but vLLM is already running on localhost:${VLLM_PORT} — selecting the running instance.`, @@ -71,7 +89,13 @@ export function buildVllmMenuEntries(opts: BuildVllmMenuOptions): VllmMenuEntry[ }, ]; } + if (opts.vllmRunning && exactManagedVllmRequired) { + log( + ` Note: ${VLLM_PROFILE_ENV}=${EXPERIMENTAL_SINGLE_USER_PROFILE} requires the qualified managed runtime; the existing vLLM instance on localhost:${VLLM_PORT} cannot be reused.`, + ); + } if ( + exactManagedVllmRequired || userChoseManagedVllm || (opts.vllmProfile && (opts.experimental || (opts.platform && MANAGED_VLLM_DEFAULT_PLATFORMS.has(opts.platform)))) diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index 23b8355686b..e572960aa73 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -278,6 +278,7 @@ detect_express_platform NEMOCLAW_VLLM_PROFILE: "experimental-single-user", NEMOCLAW_VLLM_PORT: "8000", NEMOCLAW_CONTEXT_WINDOW: "262144", + NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL: "http://host.openshell.internal/", }); const output = `${result.stdout}${result.stderr}`; expect(result.status, output).toBe(0); @@ -344,7 +345,14 @@ detect_express_platform /cannot be combined with NEMOCLAW_VLLM_EXTRA_ARGS_JSON/, ], ["NEMOCLAW_VLLM_PORT", "9000", /conflicts with NEMOCLAW_VLLM_PORT='9000'/], + ["NEMOCLAW_VLLM_PORT", "08000", /conflicts with NEMOCLAW_VLLM_PORT='08000'/], ["NEMOCLAW_CONTEXT_WINDOW", "8192", /conflicts with NEMOCLAW_CONTEXT_WINDOW='8192'/], + ["NEMOCLAW_CONTEXT_WINDOW", "0262144", /conflicts with NEMOCLAW_CONTEXT_WINDOW='0262144'/], + [ + "NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL", + "http://127.0.0.1", + /conflicts with NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL='http:\/\/127\.0\.0\.1'/, + ], ])("rejects conflicting %s for the qualified experimental Station profile", (name, value, message) => { const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { STATION_EXPERIMENTAL_SINGLE_USER: "1", From 151c24c9fa19aec2565a370181f947a2dbf252ef Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 10:27:16 -0700 Subject: [PATCH 17/21] test(inference): require numeric qualified context Signed-off-by: Aaron Erickson --- src/lib/inference/vllm-experimental-single-user.test.ts | 1 + src/lib/inference/vllm.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/inference/vllm-experimental-single-user.test.ts b/src/lib/inference/vllm-experimental-single-user.test.ts index 83aad759d02..181875dbb61 100644 --- a/src/lib/inference/vllm-experimental-single-user.test.ts +++ b/src/lib/inference/vllm-experimental-single-user.test.ts @@ -389,6 +389,7 @@ describe("experimental single-user vLLM install (#6883)", () => { '{"data":[{"id":"nemotron-ultra","max_model_len":262144},{"id":"other","max_model_len":262144}]}', ], ["wrong context", "200", '{"data":[{"id":"nemotron-ultra","max_model_len":131072}]}'], + ["string context", "200", '{"data":[{"id":"nemotron-ultra","max_model_len":"262144"}]}'], ["missing context", "200", '{"data":[{"id":"nemotron-ultra"}]}'], ["malformed catalog", "200", "not-json"], ])("does not accept readiness with %s", async (_name, healthStatus, modelsResponse) => { diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 08b8613c3c4..4e63e725bea 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -694,7 +694,7 @@ function vllmEndpointReady(model: VllmModelDef): boolean { return ( parsed.data.length === 1 && parsed.data[0]?.id === readiness.servedModelId && - Number(parsed.data[0]?.max_model_len) === readiness.maxModelLen + parsed.data[0]?.max_model_len === readiness.maxModelLen ); } catch { return false; From 98691c8c5dee217af179f715b35caa7442b585aa Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 10:31:57 -0700 Subject: [PATCH 18/21] fix(installer): bind experimental profile invariants Signed-off-by: Aaron Erickson --- docs/inference/set-up-vllm.mdx | 1 + scripts/install.sh | 14 ++++++ .../vllm-experimental-single-user.test.ts | 27 ++++++++++- src/lib/inference/vllm.ts | 48 ++++++++++++++++--- test/install-express-prompt.test.ts | 31 ++++++++++++ 5 files changed, 112 insertions(+), 9 deletions(-) diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 53c0b40096c..0e482b22136 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -166,6 +166,7 @@ curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-experimenta ``` The installer maps this flag to `NEMOCLAW_VLLM_PROFILE=experimental-single-user` and rejects conflicting model, profile, extra-argument, DeepSeek, and non-Station selections. +An inherited profile selector cannot silently change the ordinary Station express prompt: use the flag for installer express, or pair the profile and model selectors with `NEMOCLAW_PROVIDER=install-vllm` for advanced direct onboarding. It also rejects ambient `NEMOCLAW_VLLM_PORT` values other than `8000`, `NEMOCLAW_CONTEXT_WINDOW` values other than `262144`, and `NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL` routes other than `http://host.openshell.internal`, so host readiness, gateway routing, and the dashboard cannot drift from the qualified contract. An existing server on port `8000` is not reused for this profile; stop it before opting in so the existing managed-vLLM lifecycle can prove the qualified image, arguments, environment, and hardware limits. diff --git a/scripts/install.sh b/scripts/install.sh index cfbdf925a49..e72866c9632 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2788,6 +2788,19 @@ validate_station_experimental_single_user_override() { fi } +validate_station_express_profile_selector() { + local platform="$1" + if [ "$platform" != "DGX Station" ] || [ "${STATION_EXPERIMENTAL_SINGLE_USER:-}" = "1" ]; then + return 0 + fi + local requested_profile + requested_profile="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_PROFILE:-}")" + if [ -z "$requested_profile" ]; then + return 0 + fi + error "NEMOCLAW_VLLM_PROFILE='${NEMOCLAW_VLLM_PROFILE}' cannot select a DGX Station express profile by itself. Use --station-experimental-single-user for the opt-in recipe, or set NEMOCLAW_PROVIDER=install-vllm for advanced direct onboarding." +} + configure_station_express_model() { local selected_model selected_model="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_MODEL:-}")" @@ -2925,6 +2938,7 @@ maybe_offer_express_install() { info "Detected ${platform}. Skipping express prompt (NEMOCLAW_PROVIDER=${NEMOCLAW_PROVIDER} already set)." return 0 fi + validate_station_express_profile_selector "$platform" local reply="" if [ -t 0 ]; then info "Detected ${platform}." diff --git a/src/lib/inference/vllm-experimental-single-user.test.ts b/src/lib/inference/vllm-experimental-single-user.test.ts index 181875dbb61..7c39bb806d1 100644 --- a/src/lib/inference/vllm-experimental-single-user.test.ts +++ b/src/lib/inference/vllm-experimental-single-user.test.ts @@ -349,16 +349,18 @@ describe("experimental single-user vLLM install (#6883)", () => { const telemetryCalls = mocks.runCapture.mock.calls .map((call) => call[0] as string[]) .filter((args) => args[0] === "nvidia-smi"); - expect(telemetryCalls).toHaveLength(2); + expect(telemetryCalls).toHaveLength(3); expect(telemetryCalls[0]).toEqual([ "nvidia-smi", "--id=0", "--query-gpu=memory.used,ecc.errors.uncorrected.aggregate.total,ecc.errors.uncorrected.volatile.total", "--format=csv,noheader,nounits", ]); + expect(mocks.getGpuIndicesByName).toHaveBeenCalledTimes(2); const [runArgs] = mocks.dockerRunDetached.mock.calls[0] as [string[]]; expect(runArgs).not.toContain("-p"); + expect(runArgs).toEqual(expect.arrayContaining(["--gpus", "device=0"])); expect(runArgs).toEqual( expect.arrayContaining([ "--network", @@ -380,6 +382,27 @@ describe("experimental single-user vLLM install (#6883)", () => { expect(serveCommand).not.toContain("--speculative-config"); }); + it("fails closed when the qualified GPU selection changes before launch", async () => { + mocks.getGpuIndicesByName.mockReturnValueOnce([0]).mockReturnValueOnce([]); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName); + mocks.dockerImageInspectFormat.mockReturnValue( + NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, + ); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: false }); + expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); + expect(errSpy).toHaveBeenCalledWith( + expect.stringContaining("qualified GPU selection changed before launch"), + ); + }); + it.each([ ["non-200 health", "503", '{"data":[{"id":"nemotron-ultra","max_model_len":262144}]}'], ["wrong alias", "200", '{"data":[{"id":"other-model","max_model_len":262144}]}'], @@ -489,7 +512,7 @@ describe("experimental single-user vLLM install (#6883)", () => { mockSuccessfulVllmInstall(profile.containerName, { healthStatus: "200", modelsResponse: '{"data":[{"id":"nemotron-ultra","max_model_len":262144}]}', - qualificationSamples: ["1000, 0, 0", "245000, 0, 0"], + qualificationSamples: ["1000, 0, 0", "1000, 0, 0", "245000, 0, 0"], }); mocks.dockerImageInspectFormat.mockReturnValue( NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 4e63e725bea..b83a9579391 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -73,8 +73,10 @@ export interface VllmProfile { dockerRunFlags: string[]; // Optional dynamic flag builder. When present, its return value replaces // dockerRunFlags at install time. Used by Station to pick the GB300 GPU - // out of a mixed-GPU host instead of using `--gpus all`. - buildDockerRunFlags?: () => string[]; + // out of a mixed-GPU host instead of using `--gpus all`; a qualified profile + // passes its frozen, revalidated GPU index so launch cannot rediscover a + // different device. + buildDockerRunFlags?: (qualifiedGpuIndex?: number) => string[]; // Maximum wall-clock safety budget for image pulls. The Docker adapter uses // a shorter progress watchdog for stalls, so slow-but-moving pulls can keep // going until this last-ditch cap. @@ -261,8 +263,9 @@ const STATION_PROFILE: VllmProfile = { defaultModel: deepseekV4FlashModel(), containerName: NEMOCLAW_VLLM_CONTAINER_NAME, dockerRunFlags: SPARK_PROFILE.dockerRunFlags, - buildDockerRunFlags: () => { - const indices = getGpuIndicesByName(/GB300/i); + buildDockerRunFlags: (qualifiedGpuIndex) => { + const indices = + qualifiedGpuIndex === undefined ? getGpuIndicesByName(/GB300/i) : [qualifiedGpuIndex]; // Docker parses --gpus as CSV, so multi-device values must retain // double quotes inside the argv token to keep the comma in one field. const gpuFlag = @@ -533,7 +536,7 @@ export function resolveVllmRuntimeProfile(profile: VllmProfile, model: VllmModel qualification: runtime.qualification ?? profile.qualification, dockerRunFlags: [...profile.dockerRunFlags, ...extraRunArgs], buildDockerRunFlags: profile.buildDockerRunFlags - ? () => [...profile.buildDockerRunFlags!(), ...extraRunArgs] + ? (qualifiedGpuIndex) => [...profile.buildDockerRunFlags!(qualifiedGpuIndex), ...extraRunArgs] : undefined, }; } @@ -606,10 +609,11 @@ export function isNemoClawManagedVllmRunning(): boolean { function startContainer( profile: VllmProfile, model: VllmModelDef, + qualificationMonitor: VllmQualificationMonitor | null, ): { ok: boolean; reason?: string } { emit(`Starting vLLM container (${profile.containerName})`); const resolvedFlags = profile.buildDockerRunFlags - ? profile.buildDockerRunFlags() + ? profile.buildDockerRunFlags(qualificationMonitor?.gpuIndex) : profile.dockerRunFlags; // The explicit download completed before this long-lived container starts, // so do not retain the host Hugging Face token in the serving process. @@ -774,6 +778,25 @@ function initializeVllmQualification(profile: VllmProfile): VllmQualificationRes return sampleVllmQualification(profile, { gpuIndex: indices[0], peakHbmMiB: 0 }); } +function revalidateVllmQualification( + profile: VllmProfile, + monitor: VllmQualificationMonitor, +): VllmQualificationResult { + const qualification = profile.qualification; + if (!qualification) return { ok: true, monitor }; + const indices = getGpuIndicesByName(literalCaseInsensitivePattern(qualification.gpuNameIncludes)); + if (indices.length !== qualification.gpuCount || indices[0] !== monitor.gpuIndex) { + return { + ok: false, + reason: + `qualified GPU selection changed before launch: expected GPU ${String(monitor.gpuIndex)} ` + + `matching '${qualification.gpuNameIncludes}', detected matching indices ` + + `[${indices.map(String).join(", ")}]`, + }; + } + return sampleVllmQualification(profile, monitor); +} + function readContainerLogTail(profile: VllmProfile, lineCount = 80): string[] { const output = dockerCapture(["logs", "--tail", String(lineCount), profile.containerName], { env: buildVllmDockerEnv(), @@ -1243,7 +1266,18 @@ export async function installVllm( return { ok: false }; } - const start = startContainer(runtimeProfile, model); + if (qualificationMonitor) { + const preLaunchQualification = revalidateVllmQualification( + runtimeProfile, + qualificationMonitor, + ); + if (!preLaunchQualification.ok) { + console.error(` vLLM install failed: ${preLaunchQualification.reason}`); + return { ok: false }; + } + } + + const start = startContainer(runtimeProfile, model, qualificationMonitor); if (!start.ok) { console.error(` vLLM install failed: ${String(start.reason)}`); return { ok: false }; diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index e572960aa73..42d65d53a1b 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -287,6 +287,37 @@ detect_express_platform ); }); + it("rejects an inherited experimental profile from ordinary Station express", () => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { + NEMOCLAW_VLLM_PROFILE: "experimental-single-user", + }); + const output = `${result.stdout}${result.stderr}`; + + expect(result.status, output).not.toBe(0); + expect(output).toMatch( + /NEMOCLAW_VLLM_PROFILE='experimental-single-user' cannot select a DGX Station express profile by itself/, + ); + expect(output).toMatch(/Use --station-experimental-single-user/); + expect(output).not.toMatch(/Run express install/); + }); + + it("preserves the experimental selector for advanced direct onboarding", () => { + const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { + NEMOCLAW_PROVIDER: "install-vllm", + NEMOCLAW_VLLM_MODEL: "nemotron-3-ultra-550b-a55b", + NEMOCLAW_VLLM_PROFILE: "experimental-single-user", + }); + const output = `${result.stdout}${result.stderr}`; + + expect(result.status, output).toBe(0); + expect(output).toMatch( + /Skipping express prompt \(NEMOCLAW_PROVIDER=install-vllm already set\)/, + ); + expect(output).toMatch( + /PROVIDER=install-vllm .*VLLM_MODEL=nemotron-3-ultra-550b-a55b VLLM_PROFILE=experimental-single-user/, + ); + }); + it("rejects combining the experimental Station profile with the DeepSeek override", () => { const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { STATION_EXPERIMENTAL_SINGLE_USER: "1", From 36256535aaddfd12f7277f158c0358e5d689c504 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 12:04:50 -0700 Subject: [PATCH 19/21] fix(installer): require published Station runtimes Signed-off-by: Aaron Erickson --- docs/get-started/quickstart.mdx | 5 - docs/inference/choose-inference-provider.mdx | 7 - docs/inference/set-up-vllm.mdx | 100 +--- docs/reference/commands.mdx | 1 - docs/reference/platform-support.mdx | 4 - install.sh | 2 - scripts/install.sh | 121 +--- .../actions/sandbox/connect-vllm-preflight.ts | 10 +- .../sandbox/rebuild-env-isolation.test.ts | 3 - .../actions/sandbox/rebuild-env-isolation.ts | 3 +- src/lib/inference/local-host-url.ts | 8 - src/lib/inference/local.ts | 4 +- .../vllm-experimental-single-user.test.ts | 537 ------------------ src/lib/inference/vllm-models.test.ts | 253 --------- src/lib/inference/vllm-models.ts | 335 +---------- src/lib/inference/vllm.test.ts | 109 +++- src/lib/inference/vllm.ts | 403 ++++--------- src/lib/onboard/setup-nim-flow.test.ts | 46 -- src/lib/onboard/setup-nim-flow.ts | 9 +- src/lib/onboard/setup-nim-vllm.test.ts | 30 +- src/lib/onboard/setup-nim-vllm.ts | 4 +- src/lib/onboard/vllm-menu.test.ts | 27 - src/lib/onboard/vllm-menu.ts | 26 +- src/lib/onboard/vllm-model-preflight.ts | 4 +- test/cli/dispatch-basics.test.ts | 4 +- test/generate-hermes-config.test.ts | 25 - test/inference-options-docs.test.ts | 4 +- test/install-express-prompt.test.ts | 201 +------ test/onboard-selection-vllm.test.ts | 2 +- test/seed-hermes-dashboard-config.test.ts | 53 -- 30 files changed, 245 insertions(+), 2095 deletions(-) delete mode 100644 src/lib/inference/local-host-url.ts delete mode 100644 src/lib/inference/vllm-experimental-single-user.test.ts diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 35dbdfd58d3..327f0ede767 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -122,8 +122,6 @@ Use these details when your first-run path needs more control. DGX Spark, DGX Station, and Windows WSL offer an interactive express-install path that chooses a managed local inference option for the platform. On DGX Station, accepting the express prompt selects the pinned `nemotron-3-ultra-550b-a55b` managed-vLLM recipe and completes onboarding without more provider, model, policy, or sandbox-name choices. Pass `--station-deepseek` to use DeepSeek V4 Flash for a Station demo instead. - Pass `--station-experimental-single-user` to opt in to the named reclaimed-v1/offload90 Station profile within the same managed-vLLM lifecycle; this flag does not change the default Station recipe. - The experimental profile currently requires its exact qualified runtime image to be preloaded, so a fresh pull remains blocked until a repository-qualified manifest is published and proven. Refer to [Platform Support](../reference/platform-support) and [Choose an Inference Provider](../inference/learn-and-choose/choose-inference-provider) for the current platform behavior. If `nemoclaw` is not found after installation and you use nvm or fnm, open a new terminal or reload your shell profile. @@ -193,8 +191,6 @@ Use these details when your first-run path needs more control. DGX Spark uses managed vLLM with `qwen3.6-35b-a3b-nvfp4` by default. DGX Station express install explicitly selects `nemotron-3-ultra-550b-a55b` instead of the Station managed-vLLM profile default, `deepseek-v4-flash`, and discloses the approximately `352 GB` model download before confirmation. To select DeepSeek V4 Flash while retaining the one-confirmation Station express flow, run `curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-deepseek`. - To opt in to the experimental single-user Station profile, run `curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-experimental-single-user`. - That profile fails closed unless Docker already has the exact qualified local image ID `sha256:e44830d3f0197850e5a63ca895b13a047f358ff4aea15b1655abf61391bb883e`; it does not substitute another vLLM image. Unless `NEMOCLAW_POLICY_TIER` is set, express install applies policy in `suggested` mode with the `balanced` tier, including the base sandbox policy and supported package, model, web-search, and local-inference presets. Express install uses `my-assistant` as the sandbox name across all platforms unless `NEMOCLAW_SANDBOX_NAME` is set. Windows WSL selects the Windows-host Ollama setup path. @@ -203,7 +199,6 @@ Use these details when your first-run path needs more control. Express install automates the configuration but does not change DGX Station's Deferred support status. Full NemoClaw onboarding with this recipe has not completed end-to-end validation on physical DGX Station hardware. - The experimental single-user profile has prior functional qualification evidence, but this change has not completed a fresh, current-PR physical Station qualification. The installer auto-launches `nemoclaw onboard` when it can find the new binary. diff --git a/docs/inference/choose-inference-provider.mdx b/docs/inference/choose-inference-provider.mdx index dd08ca002de..a87cc050137 100644 --- a/docs/inference/choose-inference-provider.mdx +++ b/docs/inference/choose-inference-provider.mdx @@ -35,13 +35,6 @@ Use this status table to distinguish validated provider integrations from adapte | Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and DGX Station. DGX Station remains deferred until its managed-vLLM onboarding path is validated end-to-end on physical hardware. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pins runtime images to immutable digests. DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest published under `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use the matching `linux/arm64` or `linux/amd64` digest published under `nvcr.io/nvidia/vllm:26.03.post1-py3`. The DGX Station express installer selects `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` with a pinned Hugging Face revision and the multi-platform index digest published under `vllm/vllm-openai:v0.22.0`; that index resolves to the `linux/arm64` manifest on Station. Direct managed-vLLM profile defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls from `nvcr.io` require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing. | {/* provider-status:end */} - -DGX Station also has an explicitly opt-in managed-vLLM profile selected by the installer flag `--station-experimental-single-user`. -It serves `nemotron-ultra` through `vllm-local` with a `262144` context window and leaves the canonical Station recipe unchanged when the flag is absent. -This profile is experimental and currently requires the exact qualified local Docker image ID to be preloaded; a fresh pull is blocked until a repository-qualified manifest is published and proven. -Refer to [Set Up vLLM](../local-inference/set-up-vllm#opt-in-to-the-experimental-single-user-station-profile) for the runtime contract and current qualification boundary. - - ## Hosted Providers Hosted providers use provider-managed endpoints and curated model choices. diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 0e482b22136..0e6cd29bde3 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -75,24 +75,20 @@ Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PR In an interactive run, the managed path lists the supported model registry for your host profile. Press **Enter** to use the profile default or select another validated model with its registered `vllm serve` arguments. -NemoClaw resolves the pinned vLLM image, downloads model weights into `~/.cache/huggingface`, starts the `nemoclaw-vllm` container on `localhost:8000`, streams Hugging Face download progress, and polls `/v1/models` until the model is ready. -Registry-backed recipes pull that image; a local-only profile requires its exact image to be preloaded. +NemoClaw pulls the vLLM image, downloads model weights into `~/.cache/huggingface`, starts the `nemoclaw-vllm` container on `localhost:8000`, streams Hugging Face download progress, and polls `/v1/models` until the model is ready. The container carries a NemoClaw ownership label so later onboarding runs can distinguish it from a bring-your-own server without trusting the container name alone. If an existing `nemoclaw-vllm` container does not carry that label, NemoClaw preserves it and asks you to remove or rename it before installing managed vLLM. If another vLLM server already occupies port `8000`, the managed path stops and asks you to select the existing server or stop it first. Managed profiles and model-specific recipes use immutable image digests: - DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest `sha256:9204569b17ee4c0eff75194b8e6e458479c8aee18953b5ab9cf359fcdac659e2` with a compressed layer size of `9.60 GB` under `nvcr.io/nvidia/vllm:26.05.post1-py3`. -- The canonical DGX Station Nemotron 3 Ultra express recipe uses the multi-platform index digest `sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416` under `vllm/vllm-openai:v0.22.0`; on DGX Station, that index selects a `linux/arm64` manifest with a compressed layer size of `10.67 GB`. +- The DGX Station Nemotron 3 Ultra express recipe uses the multi-platform index digest `sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416` under `vllm/vllm-openai:v0.22.0`; on DGX Station, that index selects a `linux/arm64` manifest with a compressed layer size of `10.67 GB`. It also pins Hugging Face revision `183968f87ae4cedce3039313cac1fd43d112c578` for the approximately `352.38 GB` model download. -- The opt-in DGX Station `experimental-single-user` profile requires the preloaded local image ID `sha256:e44830d3f0197850e5a63ca895b13a047f358ff4aea15b1655abf61391bb883e`. - This ID is not currently published as a pullable repository manifest, and no existing registry tag has been proven to resolve to it. - NemoClaw does not substitute the canonical Ultra image or another vLLM image when the qualified image is absent. - Generic Linux `arm64` hosts use `sha256:447995cbb57e6c7cf792cab95e9852e5f62b5fb6d2f39e030fa4eda9a54eadb4` with a compressed layer size of `9.28 GB` under `nvcr.io/nvidia/vllm:26.03.post1-py3`. - Generic Linux `amd64` hosts use `sha256:7be6c2f676c36059a494fe17254e69ae5c677535ba6191044e5fc8e42a91c773` with a compressed layer size of `8.93 GB` under `nvcr.io/nvidia/vllm:26.03.post1-py3`. Image pulls from `nvcr.io` require NGC registry login. -The canonical Station Nemotron 3 Ultra recipe pulls its vLLM image from Docker Hub instead. +The Station Nemotron 3 Ultra recipe pulls its vLLM image from Docker Hub instead. The first run duration depends on image and model size, storage performance, and network throughput. Later runs reuse the cached image and model weights. @@ -107,9 +103,8 @@ For Docker's classic image store, NemoClaw checks the absolute `DockerRootDir` r For Docker's containerd image store, it checks the top-level `root` from `/etc/containerd/config.toml`, or `/var/lib/containerd` when that configuration file is absent, and also checks `DockerRootDir` for pull staging. The check uses the location with the least available capacity. -For registry-backed recipes, NemoClaw recognizes an image cache hit only when Docker contains the exact platform digest. -A registry-backed cache hit skips the capacity check, but the explicit watched `docker pull` still runs against that immutable digest. -The local-only experimental profile performs no pull and verifies the qualified image ID directly. +NemoClaw recognizes an image cache hit only when Docker contains the exact platform digest. +A cache hit skips the capacity check, but the explicit watched `docker pull` still runs against that immutable digest. The model-download helper and the long-lived vLLM container use `--pull=never`, so an implicit pull cannot select another image. NemoClaw treats `/var/run/docker.sock` and `/run/docker.sock` as the local default Docker socket paths. @@ -154,91 +149,6 @@ The registered Ultra recipe tracks the [official DGX Station deployment guide](h NemoClaw intentionally keeps its existing bridge-networked managed-inference topology instead of importing the playbook's host-network setting. The container publishes port `8000` through Docker, so apply the firewall guidance at the top of this page. -### Opt In to the Experimental Single-User Station Profile - -The `experimental-single-user` profile extends the existing managed-vLLM and Station express lifecycle; it is not a separate launcher. -It is explicit opt-in and does not change the canonical Station express recipe above. -The canonical recipe remains the default while Nemotron-team quality validation of this experimental profile is pending. -Use it only on a detected DGX Station: - -```bash -curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -s -- --station-experimental-single-user -``` - -The installer maps this flag to `NEMOCLAW_VLLM_PROFILE=experimental-single-user` and rejects conflicting model, profile, extra-argument, DeepSeek, and non-Station selections. -An inherited profile selector cannot silently change the ordinary Station express prompt: use the flag for installer express, or pair the profile and model selectors with `NEMOCLAW_PROVIDER=install-vllm` for advanced direct onboarding. -It also rejects ambient `NEMOCLAW_VLLM_PORT` values other than `8000`, `NEMOCLAW_CONTEXT_WINDOW` values other than `262144`, and `NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL` routes other than `http://host.openshell.internal`, so host readiness, gateway routing, and the dashboard cannot drift from the qualified contract. -An existing server on port `8000` is not reused for this profile; stop it before opting in so the existing managed-vLLM lifecycle can prove the qualified image, arguments, environment, and hardware limits. - - -The qualified runtime exists today only as local Docker image ID `sha256:e44830d3f0197850e5a63ca895b13a047f358ff4aea15b1655abf61391bb883e`. -Before running the installer, this command must return that exact ID: - -```bash -docker image inspect -f '{{.Id}}' sha256:e44830d3f0197850e5a63ca895b13a047f358ff4aea15b1655abf61391bb883e -``` - -A fresh pull or fresh machine install remains blocked until a repository-qualified manifest pin is published and proven to resolve to the qualified image. -The profile fails closed instead of silently using a different vLLM image. - - -The profile uses the reclaimed-v1/offload90 configuration with this model identity: - -| Setting | Value | -|---|---| -| Hugging Face model | `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` | -| Revision | `183968f87ae4cedce3039313cac1fd43d112c578` | -| Served model | `nemotron-ultra` | -| Maximum context | `262144` | - -Its container runs with host networking and host IPC, `16 GiB` shared memory, a `650 GiB` container memory limit, tensor parallel size `1`, and a `1200`-second startup timeout. -Because the qualified contract binds the unauthenticated vLLM API to `0.0.0.0:8000` on the host network, apply the default-deny firewall rules at the top of this page before opting in; this profile does not install or modify host firewall rules. -The installer fails closed unless exactly one GB300 is available, HBM is below `245000 MiB`, and aggregate and volatile uncorrected ECC are zero before launch; it samples the same exclusive HBM ceiling and ECC contract throughout startup. -Full workload-level HBM/ECC qualification through chat and tool calls remains part of the blocked current-PR physical Station acceptance run. - -After `vllm serve` and the pinned revision's local snapshot path, the profile supplies exactly these vLLM arguments: - -```text ---served-model-name nemotron-ultra ---host 0.0.0.0 ---port 8000 ---tensor-parallel-size 1 ---trust-remote-code ---cpu-offload-gb 90 ---cpu-offload-params experts ---kernel_config '{"enable_flashinfer_autotune": false}' ---max-num-seqs 1 ---gpu-memory-utilization 0.9 ---kv-cache-memory-bytes 4294967296 ---reasoning-parser nemotron_v3 ---enable-auto-tool-choice ---tool-call-parser qwen3_coder -``` - -The required environment is: - -```text -VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY=1 -VLLM_NVFP4_GEMM_BACKEND=flashinfer-trtllm -NEMOTRON_ULTRA_MODEL_OPT_NVFP4_MOE_RECLAIM=1 -HF_HUB_OFFLINE=1 -TRANSFORMERS_OFFLINE=1 -PYTHONHASHSEED=0 -``` - -Prefix caching remains disabled, matching the pinned Station guide. -That setting predates this experimental profile; it is not an optimization introduced by this work. -The profile does not add `--enable-prefix-caching`, does not use MTP speculative decoding, and disables FlashInfer autotuning explicitly. -It also does not use the Gen4B/autotuned result, which measured `44.88197518340701` tokens/s/user but took `2174.438460592006` seconds to start. -The selected reclaimed-v1/offload90 parent configuration measured `41.87059772268699` parser-free AIPerf tokens/s/user and `285.592754706` seconds to API readiness. -The Hermes parser overlay has functional qualification but no separate AIPerf score. - -NemoClaw routes the profile through provider `vllm-local`, model `nemotron-ultra`, and effective context length `262144` using the normal inference and Hermes lifecycle. -Generated gateway configuration and the dashboard mirror receive the same route and context; do not edit sandbox configuration files directly. -Nemotron Ultra requires the `qwen3_coder` tool-call parser, not the Hermes tool parser. - -This profile has prior benchmark and functional evidence, but it has not completed a fresh physical-DGX-Station qualification from the current PR. - DGX Station and its managed-vLLM paths remain Deferred until NemoClaw completes end-to-end validation on physical DGX Station hardware. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index d7fe74064a2..64372265328 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3287,7 +3287,6 @@ Set them before running `$$nemoclaw onboard`. | `NEMOCLAW_INSTALL_REF` | git ref | For internal installer commands: the git ref to install from. Overridden by the `--install-ref` flag. | | `NEMOCLAW_INSTALL_TAG` | release tag | For internal installer commands: the release tag to install. Defaults to the admin-promoted `lkg` tag when unset. Overridden by the `--install-tag` flag. | | `NEMOCLAW_VLLM_MODEL` | registry slug or Hugging Face model id | Selects the model the managed-vLLM install path serves. Recognised slugs: `qwen3.6-27b`, `qwen3.6-35b-a3b-nvfp4`, `nemotron-3-nano-4b`, `deepseek-v4-flash`, `nemotron-3-ultra-550b-a55b`, `deepseek-r1-distill-70b`. Unset uses the per-platform profile default. The DGX Station express installer sets `nemotron-3-ultra-550b-a55b` explicitly. Gated models (e.g. `deepseek-r1-distill-70b`) require `HF_TOKEN` or `HUGGING_FACE_HUB_TOKEN`. | -| `NEMOCLAW_VLLM_PROFILE` | profile name | Selects an explicit managed-vLLM runtime profile. `experimental-single-user` is restricted to DGX Station and selects the reclaimed-v1/offload90 recipe; leaving this variable unset preserves the existing per-model and per-platform behavior. Prefer the installer flag `--station-experimental-single-user`, which validates conflicting selections and sets the matching model and served name. | | `NEMOCLAW_VLLM_EXTRA_ARGS_JSON` | JSON array of non-blank strings | Appends advanced operator-owned tokens to the managed `vllm serve` command after NemoClaw's registry defaults. Example: `["--max-num-seqs","2"]`. Malformed JSON, non-string tokens, or blank tokens fail before Docker work starts. | | `NEMOCLAW_IGNORE_VLLM_DISK_SPACE` | exactly `1` | Allows managed vLLM to continue when verified Docker image-storage or registered model-cache capacity is insufficient; other values do not bypass either check. Inconclusive checks report diagnostics and continue automatically. | diff --git a/docs/reference/platform-support.mdx b/docs/reference/platform-support.mdx index c97ef9084c2..35a31916def 100644 --- a/docs/reference/platform-support.mdx +++ b/docs/reference/platform-support.mdx @@ -107,10 +107,6 @@ NemoClaw routes inference through the OpenShell gateway. Each row below is a pro | Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and DGX Station. DGX Station remains deferred until its managed-vLLM onboarding path is validated end-to-end on physical hardware. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pins runtime images to immutable digests. DGX Spark and DGX Station models without a model-specific runtime use the `linux/arm64` digest published under `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use the matching `linux/arm64` or `linux/amd64` digest published under `nvcr.io/nvidia/vllm:26.03.post1-py3`. The DGX Station express installer selects `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4` with a pinned Hugging Face revision and the multi-platform index digest published under `vllm/vllm-openai:v0.22.0`; that index resolves to the `linux/arm64` manifest on Station. Direct managed-vLLM profile defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls from `nvcr.io` require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing. | {/* provider-status-full:end */} -The DGX Station `experimental-single-user` managed-vLLM profile is available through the explicit installer flag `--station-experimental-single-user`; advanced direct onboarding may opt in with the matching `NEMOCLAW_VLLM_PROFILE` and `NEMOCLAW_VLLM_MODEL` selectors. -It does not change the default Station recipe or promote DGX Station beyond Deferred status. -The profile currently requires its exact qualified runtime image to be preloaded because no pullable repository manifest has been published and proven for that image; do not present it as a fresh-install or current-PR physical-hardware qualification. - ## Messaging Integrations NemoClaw configures messaging channels during onboarding. The OpenShell gateway runs each channel as a supervised process; NemoClaw supplies onboarding, credential delivery, and policy presets for the sandbox egress rules. diff --git a/install.sh b/install.sh index 09bbd00fe1f..ab867b4c891 100755 --- a/install.sh +++ b/install.sh @@ -114,8 +114,6 @@ bootstrap_usage() { printf " Options:\n" printf " --non-interactive Skip prompts (uses env vars / defaults)\n" printf " --station-deepseek Use DeepSeek V4 Flash for DGX Station express install\n" - printf " --station-experimental-single-user\n" - printf " Use the experimental single-user DGX Station profile\n" printf " --yes-i-accept-third-party-software Accept the third-party software notice without prompting\n" printf " --fresh Discard any failed/interrupted onboarding session and start over\n" printf " --version, -v Print installer version and exit\n" diff --git a/scripts/install.sh b/scripts/install.sh index acb30b0e35c..f7212c51398 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -748,8 +748,6 @@ usage() { printf " --yes-i-accept-third-party-software Accept the third-party software notice without prompting\n" printf " --fresh Discard any failed/interrupted onboarding session and start over\n" printf " --station-deepseek Use DeepSeek V4 Flash for DGX Station express install\n" - printf " --station-experimental-single-user\n" - printf " Use the experimental single-user DGX Station profile\n" printf " --version, -v Print installer version and exit\n" printf " --help, -h Show this help message and exit\n\n" printf " ${C_DIM}Environment:${C_RESET}\n" @@ -2858,11 +2856,6 @@ STATION_ULTRA_VLLM_MODEL="nemotron-3-ultra-550b-a55b" STATION_ULTRA_SERVED_MODEL="nvidia/nemotron-3-ultra-550b-a55b" STATION_DEEPSEEK_VLLM_MODEL="deepseek-v4-flash" STATION_DEEPSEEK_SERVED_MODEL="deepseek-ai/DeepSeek-V4-Flash" -STATION_EXPERIMENTAL_SINGLE_USER_PROFILE="experimental-single-user" -STATION_EXPERIMENTAL_SINGLE_USER_SERVED_MODEL="nemotron-ultra" -STATION_EXPERIMENTAL_SINGLE_USER_PORT="8000" -STATION_EXPERIMENTAL_SINGLE_USER_CONTEXT_WINDOW="262144" -STATION_EXPERIMENTAL_SINGLE_USER_SANDBOX_HOST_URL="http://host.openshell.internal" normalize_station_vllm_model() { printf "%s" "${1:-}" | tr '[:upper:]' '[:lower:]' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' @@ -2885,9 +2878,6 @@ validate_station_deepseek_override() { if [ -n "${NEMOCLAW_PROVIDER:-}" ]; then error "--station-deepseek conflicts with NEMOCLAW_PROVIDER=${NEMOCLAW_PROVIDER}. Remove the provider override to use Station express install." fi - if [ -n "$(printf "%s" "${NEMOCLAW_VLLM_PROFILE:-}" | tr -d '[:space:]')" ]; then - error "--station-deepseek conflicts with NEMOCLAW_VLLM_PROFILE='${NEMOCLAW_VLLM_PROFILE}'. Remove the profile override to use the DeepSeek Station recipe." - fi local requested_model requested_model="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_MODEL:-}")" @@ -2899,107 +2889,10 @@ validate_station_deepseek_override() { esac } -validate_station_experimental_single_user_override() { - local platform="$1" - if [ "${STATION_EXPERIMENTAL_SINGLE_USER:-}" != "1" ]; then - return 0 - fi - if [ "${STATION_DEEPSEEK:-}" = "1" ]; then - error "--station-experimental-single-user cannot be combined with --station-deepseek." - fi - if [ "$platform" != "DGX Station" ]; then - error "--station-experimental-single-user requires a detected DGX Station (detected: ${platform:-unsupported platform})." - fi - if [ "${NEMOCLAW_NO_EXPRESS:-}" = "1" ]; then - error "--station-experimental-single-user cannot be combined with NEMOCLAW_NO_EXPRESS=1. Remove one override." - fi - if [ "${NON_INTERACTIVE:-}" = "1" ]; then - error "--station-experimental-single-user selects the DGX Station express prompt and cannot be combined with --non-interactive." - fi - if [ -n "${NEMOCLAW_PROVIDER:-}" ]; then - error "--station-experimental-single-user conflicts with NEMOCLAW_PROVIDER=${NEMOCLAW_PROVIDER}. Remove the provider override to use Station express install." - fi - - local requested_model - requested_model="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_MODEL:-}")" - case "$requested_model" in - "" | "$STATION_ULTRA_VLLM_MODEL" | "$STATION_ULTRA_SERVED_MODEL" | "nvidia/nvidia-nemotron-3-ultra-550b-a55b-nvfp4") ;; - *) - error "--station-experimental-single-user conflicts with NEMOCLAW_VLLM_MODEL='${NEMOCLAW_VLLM_MODEL}'. Remove one override or set NEMOCLAW_VLLM_MODEL=${STATION_ULTRA_VLLM_MODEL}." - ;; - esac - - local requested_served_model - requested_served_model="$(normalize_station_vllm_model "${NEMOCLAW_MODEL:-}")" - case "$requested_served_model" in - "" | "$STATION_EXPERIMENTAL_SINGLE_USER_SERVED_MODEL") ;; - *) - error "--station-experimental-single-user conflicts with NEMOCLAW_MODEL='${NEMOCLAW_MODEL}'. Remove one override or set NEMOCLAW_MODEL=${STATION_EXPERIMENTAL_SINGLE_USER_SERVED_MODEL}." - ;; - esac - - local requested_profile - requested_profile="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_PROFILE:-}")" - case "$requested_profile" in - "" | "$STATION_EXPERIMENTAL_SINGLE_USER_PROFILE") ;; - *) - error "--station-experimental-single-user conflicts with NEMOCLAW_VLLM_PROFILE='${NEMOCLAW_VLLM_PROFILE}'. Remove one override or set NEMOCLAW_VLLM_PROFILE=${STATION_EXPERIMENTAL_SINGLE_USER_PROFILE}." - ;; - esac - - local requested_port - requested_port="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_PORT:-}")" - case "$requested_port" in - "" | "$STATION_EXPERIMENTAL_SINGLE_USER_PORT") ;; - *) - error "--station-experimental-single-user conflicts with NEMOCLAW_VLLM_PORT='${NEMOCLAW_VLLM_PORT}'. Remove the override or set NEMOCLAW_VLLM_PORT=${STATION_EXPERIMENTAL_SINGLE_USER_PORT}." - ;; - esac - - local requested_context_window - requested_context_window="$(normalize_station_vllm_model "${NEMOCLAW_CONTEXT_WINDOW:-}")" - case "$requested_context_window" in - "" | "$STATION_EXPERIMENTAL_SINGLE_USER_CONTEXT_WINDOW") ;; - *) - error "--station-experimental-single-user conflicts with NEMOCLAW_CONTEXT_WINDOW='${NEMOCLAW_CONTEXT_WINDOW}'. Remove the override or set NEMOCLAW_CONTEXT_WINDOW=${STATION_EXPERIMENTAL_SINGLE_USER_CONTEXT_WINDOW}." - ;; - esac - - local requested_sandbox_host_url - requested_sandbox_host_url="$(printf "%s" "${NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL:-}" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//; s:/*$::')" - case "$requested_sandbox_host_url" in - "" | "$STATION_EXPERIMENTAL_SINGLE_USER_SANDBOX_HOST_URL" | "host.openshell.internal") ;; - *) - error "--station-experimental-single-user conflicts with NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL='${NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL}'. Remove the override or set NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL=${STATION_EXPERIMENTAL_SINGLE_USER_SANDBOX_HOST_URL}." - ;; - esac - - if [ -n "$(printf "%s" "${NEMOCLAW_VLLM_EXTRA_ARGS_JSON:-}" | tr -d '[:space:]')" ]; then - error "--station-experimental-single-user cannot be combined with NEMOCLAW_VLLM_EXTRA_ARGS_JSON. The qualified profile supplies the exact vLLM arguments." - fi -} - -validate_station_express_profile_selector() { - local platform="$1" - if [ "$platform" != "DGX Station" ] || [ "${STATION_EXPERIMENTAL_SINGLE_USER:-}" = "1" ]; then - return 0 - fi - local requested_profile - requested_profile="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_PROFILE:-}")" - if [ -z "$requested_profile" ]; then - return 0 - fi - error "NEMOCLAW_VLLM_PROFILE='${NEMOCLAW_VLLM_PROFILE}' cannot select a DGX Station express profile by itself. Use --station-experimental-single-user for the opt-in recipe, or set NEMOCLAW_PROVIDER=install-vllm for advanced direct onboarding." -} - configure_station_express_model() { local selected_model selected_model="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_MODEL:-}")" - if [ "${STATION_EXPERIMENTAL_SINGLE_USER:-}" = "1" ]; then - NEMOCLAW_VLLM_PROFILE="$STATION_EXPERIMENTAL_SINGLE_USER_PROFILE" - NEMOCLAW_VLLM_MODEL="$STATION_ULTRA_VLLM_MODEL" - NEMOCLAW_MODEL="$STATION_EXPERIMENTAL_SINGLE_USER_SERVED_MODEL" - elif [ "${STATION_DEEPSEEK:-}" = "1" ]; then + if [ "${STATION_DEEPSEEK:-}" = "1" ]; then NEMOCLAW_VLLM_MODEL="$STATION_DEEPSEEK_VLLM_MODEL" NEMOCLAW_MODEL="$STATION_DEEPSEEK_SERVED_MODEL" elif [ -z "$selected_model" ]; then @@ -3023,9 +2916,6 @@ configure_station_express_model() { esac fi export NEMOCLAW_VLLM_MODEL - if [ "${STATION_EXPERIMENTAL_SINGLE_USER:-}" = "1" ]; then - export NEMOCLAW_VLLM_PROFILE - fi if [ -n "${NEMOCLAW_MODEL:-}" ]; then export NEMOCLAW_MODEL fi @@ -3054,10 +2944,7 @@ describe_express_install() { sandbox_summary="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" ;; "DGX Station") - if [ "${STATION_EXPERIMENTAL_SINGLE_USER:-}" = "1" ]; then - inference_summary="managed local vLLM with the experimental single-user Nemotron Ultra profile" - inference_disclosure="This opt-in profile uses the qualified single-user Station runtime and model configuration." - elif [ "${STATION_DEEPSEEK:-}" = "1" ]; then + if [ "${STATION_DEEPSEEK:-}" = "1" ]; then inference_summary="managed local vLLM with DeepSeek V4 Flash" inference_disclosure="Managed vLLM pulls the configured Station image/model and runs a local inference container." elif [ -n "$(printf "%s" "${NEMOCLAW_VLLM_MODEL:-}" | tr -d '[:space:]')" ]; then @@ -3110,7 +2997,6 @@ maybe_offer_express_install() { local platform platform="$(detect_express_platform)" validate_station_deepseek_override "$platform" - validate_station_experimental_single_user_override "$platform" # Not on a platform we have an express recipe for — say nothing. if [ -z "$platform" ]; then return 0 @@ -3129,7 +3015,6 @@ maybe_offer_express_install() { info "Detected ${platform}. Skipping express prompt (NEMOCLAW_PROVIDER=${NEMOCLAW_PROVIDER} already set)." return 0 fi - validate_station_express_profile_selector "$platform" local reply="" if [ -t 0 ]; then info "Detected ${platform}." @@ -3199,14 +3084,12 @@ main() { ACCEPT_THIRD_PARTY_SOFTWARE="" FRESH="" STATION_DEEPSEEK="" - STATION_EXPERIMENTAL_SINGLE_USER="" for arg in "$@"; do case "$arg" in --non-interactive) NON_INTERACTIVE=1 ;; --yes-i-accept-third-party-software) ACCEPT_THIRD_PARTY_SOFTWARE=1 ;; --fresh) FRESH=1 ;; --station-deepseek) STATION_DEEPSEEK=1 ;; - --station-experimental-single-user) STATION_EXPERIMENTAL_SINGLE_USER=1 ;; --version | -v) local version_suffix version_suffix="$(installer_version_for_display)" diff --git a/src/lib/actions/sandbox/connect-vllm-preflight.ts b/src/lib/actions/sandbox/connect-vllm-preflight.ts index 77bbe92fcfa..811f91fc483 100644 --- a/src/lib/actions/sandbox/connect-vllm-preflight.ts +++ b/src/lib/actions/sandbox/connect-vllm-preflight.ts @@ -6,17 +6,17 @@ import { preflightVllmModelEnv } from "../../inference/vllm-models"; // Managed-vLLM env vars only steer the express-vLLM install path, but users // often re-export them in the same shell they later run `connect` in. Run the -// installer's validators up-front so model/profile conflicts, malformed extra -// args, or a gated model with no `HF_TOKEN` fail fast on the host — before any -// sandbox readiness probe, inference-route reset, or SSH attach — instead of -// being silently ignored. +// installer's validators up-front so typos, malformed extra args, or a gated +// model with no `HF_TOKEN` fail fast on the host — before any sandbox readiness +// probe, inference-route reset, or SSH attach — instead of being silently +// ignored. export function preflightVllmModelEnvOrExit(): void { const result = preflightVllmModelEnv(); if (result.ok) return; console.error(""); console.error(` Error: ${result.message}`); console.error( - ` Hint: NEMOCLAW_VLLM_MODEL, NEMOCLAW_VLLM_PROFILE, and NEMOCLAW_VLLM_EXTRA_ARGS_JSON are consumed by the managed-vLLM install path; they are not used by \`${CLI_NAME} connect\`.`, + ` Hint: NEMOCLAW_VLLM_MODEL is consumed by the managed-vLLM install path, and NEMOCLAW_VLLM_EXTRA_ARGS_JSON is consumed there too; neither is used by \`${CLI_NAME} connect\`.`, ); console.error( " Unset the managed-vLLM env var before reconnecting, or fix the value and re-run the install path that serves the model.", diff --git a/src/lib/actions/sandbox/rebuild-env-isolation.test.ts b/src/lib/actions/sandbox/rebuild-env-isolation.test.ts index d432fba0a76..746715bc63a 100644 --- a/src/lib/actions/sandbox/rebuild-env-isolation.test.ts +++ b/src/lib/actions/sandbox/rebuild-env-isolation.test.ts @@ -52,7 +52,6 @@ describe("AMBIENT_RECREATE_ENV_VARS contract PRA-4 (#5735)", () => { "NEMOCLAW_PREFERRED_API", "NEMOCLAW_REASONING", "NEMOCLAW_VLLM_MODEL", - "NEMOCLAW_VLLM_PROFILE", "NEMOCLAW_VLLM_EXTRA_ARGS_JSON", "NEMOCLAW_FROM_DOCKERFILE", "NEMOCLAW_WEB_SEARCH_PROVIDER", @@ -123,7 +122,6 @@ describe("isolateAmbientRecreateEnv", () => { NEMOCLAW_PREFERRED_API: "openai-responses", NEMOCLAW_REASONING: "false", NEMOCLAW_VLLM_MODEL: "ambient-vllm-model", - NEMOCLAW_VLLM_PROFILE: "experimental-single-user", NEMOCLAW_VLLM_EXTRA_ARGS_JSON: '{"ambient":true}', NEMOCLAW_FROM_DOCKERFILE: "/tmp/unrelated.Dockerfile", NEMOCLAW_WEB_SEARCH_PROVIDER: "tavily", @@ -154,7 +152,6 @@ describe("isolateAmbientRecreateEnv", () => { expect(env.NEMOCLAW_PREFERRED_API).toBe("openai-responses"); expect(env.NEMOCLAW_REASONING).toBe("false"); expect(env.NEMOCLAW_VLLM_MODEL).toBe("ambient-vllm-model"); - expect(env.NEMOCLAW_VLLM_PROFILE).toBe("experimental-single-user"); expect(env.NEMOCLAW_VLLM_EXTRA_ARGS_JSON).toBe('{"ambient":true}'); expect(env.NEMOCLAW_FROM_DOCKERFILE).toBe("/tmp/unrelated.Dockerfile"); expect(env.NEMOCLAW_WEB_SEARCH_PROVIDER).toBe("tavily"); diff --git a/src/lib/actions/sandbox/rebuild-env-isolation.ts b/src/lib/actions/sandbox/rebuild-env-isolation.ts index a6d486f0e4c..b0d456f7a25 100644 --- a/src/lib/actions/sandbox/rebuild-env-isolation.ts +++ b/src/lib/actions/sandbox/rebuild-env-isolation.ts @@ -22,7 +22,7 @@ // → src/lib/onboard/providers.ts (hosted model aliases) // - NEMOCLAW_PREFERRED_API → src/lib/onboard/setup-nim-selection.ts // - NEMOCLAW_REASONING → src/lib/onboard/reasoning-mode.ts -// - NEMOCLAW_VLLM_MODEL / NEMOCLAW_VLLM_PROFILE / NEMOCLAW_VLLM_EXTRA_ARGS_JSON +// - NEMOCLAW_VLLM_MODEL / NEMOCLAW_VLLM_EXTRA_ARGS_JSON // → src/lib/onboard/setup-nim-vllm.ts // - NEMOCLAW_FROM_DOCKERFILE → src/lib/onboard/entry-options.ts // - NEMOCLAW_POLICY_TIER / NEMOCLAW_POLICY_MODE / NEMOCLAW_POLICY_PRESETS @@ -48,7 +48,6 @@ export const AMBIENT_RECREATE_ENV_VARS = [ "NEMOCLAW_PREFERRED_API", "NEMOCLAW_REASONING", "NEMOCLAW_VLLM_MODEL", - "NEMOCLAW_VLLM_PROFILE", "NEMOCLAW_VLLM_EXTRA_ARGS_JSON", "NEMOCLAW_FROM_DOCKERFILE", "NEMOCLAW_WEB_SEARCH_PROVIDER", diff --git a/src/lib/inference/local-host-url.ts b/src/lib/inference/local-host-url.ts deleted file mode 100644 index 9ad83d60bc1..00000000000 --- a/src/lib/inference/local-host-url.ts +++ /dev/null @@ -1,8 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -/** Sandbox-to-host route used by NemoClaw's managed local inference providers. */ -export const HOST_GATEWAY_URL = "http://host.openshell.internal"; - -/** Advanced override for the sandbox-facing local inference host route. */ -export const LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV = "NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL"; diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index e8808e6e6e9..e2bcb0eab80 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -20,7 +20,6 @@ import { containerCanReachHostLoopback, isWsl } from "../platform"; import { type CaptureResult, runCapture, runCaptureEx, shellQuote } from "../runner"; import { nemoclawStateRoot } from "../state/state-root"; import { buildSubprocessEnv } from "../subprocess-env"; -import { HOST_GATEWAY_URL, LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV } from "./local-host-url"; import { detectNvidiaPlatform } from "./nim"; import { anyRegistryModelFits, @@ -67,7 +66,8 @@ export function resetOllamaContainerPortCache(): void { _ollamaContainerPort = null; } -export { HOST_GATEWAY_URL, LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV } from "./local-host-url"; +export const HOST_GATEWAY_URL = "http://host.openshell.internal"; +export const LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV = "NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL"; export const CONTAINER_REACHABILITY_IMAGE = "curlimages/curl:8.10.1"; // These tags are convenience aliases for callers that want to refer to a // specific bootstrap model by role rather than by string. The canonical diff --git a/src/lib/inference/vllm-experimental-single-user.test.ts b/src/lib/inference/vllm-experimental-single-user.test.ts deleted file mode 100644 index 7c39bb806d1..00000000000 --- a/src/lib/inference/vllm-experimental-single-user.test.ts +++ /dev/null @@ -1,537 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { EventEmitter } from "node:events"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - -const mocks = vi.hoisted(() => ({ - dockerCapture: vi.fn(), - dockerForceRm: vi.fn(), - dockerImageInspectFormat: vi.fn(), - dockerPullWithProgressWatchdog: vi.fn(), - dockerRunDetached: vi.fn(), - dockerSpawn: vi.fn(), - dockerStop: vi.fn(), - findUnwritableTreePath: vi.fn(), - getGpuIndicesByName: vi.fn<(_pattern: RegExp) => number[]>(() => []), - measureDirectorySizeBytes: vi.fn(), - probeDockerStorage: vi.fn(), - probeHostStorage: vi.fn(), - runCapture: vi.fn(), -})); - -vi.mock("../runner", () => ({ - runCapture: mocks.runCapture, -})); - -vi.mock("../adapters/docker", () => ({ - dockerCapture: mocks.dockerCapture, - dockerForceRm: mocks.dockerForceRm, - dockerImageInspectFormat: mocks.dockerImageInspectFormat, - dockerPullWithProgressWatchdog: mocks.dockerPullWithProgressWatchdog, - dockerRunDetached: mocks.dockerRunDetached, - dockerSpawn: mocks.dockerSpawn, - dockerStop: mocks.dockerStop, -})); - -vi.mock("./nim", () => ({ - getGpuIndicesByName: mocks.getGpuIndicesByName, -})); - -vi.mock("./vllm-storage", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - findUnwritableTreePath: mocks.findUnwritableTreePath, - measureDirectorySizeBytes: mocks.measureDirectorySizeBytes, - probeDockerStorage: mocks.probeDockerStorage, - probeHostStorage: mocks.probeHostStorage, - }; -}); - -import { - buildVllmRunArgs, - detectVllmProfile, - installVllm, - NEMOCLAW_VLLM_CONTAINER_NAME, - NEMOCLAW_VLLM_MANAGED_LABEL, - resolveVllmRuntimeProfile, -} from "./vllm"; -import { - buildVllmServeCommand, - EXPERIMENTAL_SINGLE_USER_PROFILE, - NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE, - resolveVllmModelProfile, - VLLM_MODELS, -} from "./vllm-models"; - -beforeEach(() => { - mocks.dockerImageInspectFormat.mockReturnValue(""); - mocks.findUnwritableTreePath.mockReturnValue(null); - mocks.measureDirectorySizeBytes.mockReturnValue(0n); - mocks.probeDockerStorage.mockReturnValue({ - ok: true, - capacity: { availableBytes: 1_000_000_000_000n, path: "/docker", source: "Docker" }, - }); - mocks.probeHostStorage.mockReturnValue({ - ok: true, - capacity: { - availableBytes: 1_000_000_000_000n, - path: path.join(os.homedir(), ".cache", "huggingface"), - source: "Hugging Face cache", - }, - }); -}); - -function mockDockerSpawnSuccess(): EventEmitter & { - stdout: EventEmitter; - stderr: EventEmitter; -} { - const proc = new EventEmitter() as EventEmitter & { - stdout: EventEmitter; - stderr: EventEmitter; - }; - proc.stdout = new EventEmitter(); - proc.stderr = new EventEmitter(); - process.nextTick(() => proc.emit("exit", 0)); - return proc; -} - -function mockSuccessfulVllmInstall( - containerName: string, - endpoints: { - healthStatus?: string; - modelsResponse?: string; - qualificationSamples?: readonly string[]; - } = {}, -): void { - const qualificationSamples = [...(endpoints.qualificationSamples ?? ["1000, 0, 0"])]; - let lastQualificationSample = qualificationSamples.at(-1) ?? ""; - mocks.runCapture.mockImplementation((cmd: readonly string[]) => { - switch (cmd[0]) { - case "sh": - return "/usr/bin/tool\n"; - case "nvidia-smi": { - const sample = qualificationSamples.shift() ?? lastQualificationSample; - lastQualificationSample = sample; - return sample; - } - case "curl": { - const url = cmd.at(-1) ?? ""; - return url.endsWith("/health") - ? (endpoints.healthStatus ?? "") - : (endpoints.modelsResponse ?? '{"data":[]}'); - } - default: - return ""; - } - }); - mocks.dockerPullWithProgressWatchdog.mockResolvedValue({ - status: 0, - signal: null, - output: "", - timedOut: false, - timeoutKind: null, - }); - mocks.dockerSpawn.mockReturnValue(mockDockerSpawnSuccess()); - mocks.dockerRunDetached.mockReturnValue({ status: 0, stdout: "", stderr: "", error: null }); - const ownershipResponses: (() => string)[] = [() => "", () => ""]; - mocks.dockerCapture.mockImplementation((args: readonly string[]) => { - switch (args[0]) { - case "container": - return (ownershipResponses.shift() ?? (() => ""))(); - case "ps": - return `${containerName}\n`; - default: - return ""; - } - }); -} - -describe("experimental single-user vLLM runtime profile (#6883)", () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it("resolves through the existing Station runtime with exact generated Docker arguments", () => { - mocks.getGpuIndicesByName.mockReturnValue([0]); - const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); - const ultra = VLLM_MODELS.find((model) => model.envValue === "nemotron-3-ultra-550b-a55b"); - - expect(profile).not.toBeNull(); - expect(ultra).toBeDefined(); - const effective = resolveVllmModelProfile(ultra!, "station", { - NEMOCLAW_VLLM_PROFILE: EXPERIMENTAL_SINGLE_USER_PROFILE, - }); - const runtime = resolveVllmRuntimeProfile(profile!, effective.model); - const flags = runtime.buildDockerRunFlags!(); - - expect(runtime).toEqual( - expect.objectContaining({ - image: NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, - loadTimeoutSec: 1200, - networkMode: "host", - imagePullPolicy: "local-only", - expectedImageId: NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, - qualification: { - gpuNameIncludes: "GB300", - gpuCount: 1, - hbmSafetyCeilingMiB: 245000, - requireZeroUncorrectedEcc: true, - }, - }), - ); - expect(flags).toEqual([ - "--gpus", - "device=0", - "--ipc=host", - "-v", - `${path.join(os.homedir(), ".cache", "huggingface")}:/root/.cache/huggingface`, - "-e", - "HF_HOME=/root/.cache/huggingface", - "--network", - "host", - "--shm-size", - "16g", - "--memory", - "650g", - "-e", - "VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY=1", - "-e", - "VLLM_NVFP4_GEMM_BACKEND=flashinfer-trtllm", - "-e", - "NEMOTRON_ULTRA_MODEL_OPT_NVFP4_MOE_RECLAIM=1", - "-e", - "HF_HUB_OFFLINE=1", - "-e", - "TRANSFORMERS_OFFLINE=1", - "-e", - "PYTHONHASHSEED=0", - ]); - - expect(buildVllmRunArgs(runtime, effective.model, flags, {} as NodeJS.ProcessEnv)).toEqual([ - "--pull=never", - "--restart", - "unless-stopped", - ...flags, - "--label", - `${NEMOCLAW_VLLM_MANAGED_LABEL}=true`, - "--name", - NEMOCLAW_VLLM_CONTAINER_NAME, - "--entrypoint", - "/bin/bash", - NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, - "-lc", - buildVllmServeCommand(effective.model, {} as NodeJS.ProcessEnv), - ]); - }); -}); - -describe("experimental single-user vLLM install (#6883)", () => { - let errSpy: ReturnType; - let mkdirSpy: ReturnType; - let stdoutWrite: ReturnType; - const originalEnv = { ...process.env }; - - beforeEach(() => { - vi.clearAllMocks(); - vi.spyOn(console, "log").mockImplementation(() => {}); - errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - mkdirSpy = vi.spyOn(fs, "mkdirSync").mockImplementation(() => undefined); - stdoutWrite = vi.spyOn(process.stdout, "write").mockImplementation(() => true); - process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; - process.env.NEMOCLAW_VLLM_PROFILE = EXPERIMENTAL_SINGLE_USER_PROFILE; - delete process.env.NEMOCLAW_VLLM_EXTRA_ARGS_JSON; - delete process.env.NEMOCLAW_VLLM_PORT; - delete process.env.NEMOCLAW_CONTEXT_WINDOW; - delete process.env.NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL; - delete process.env.NEMOCLAW_IGNORE_VLLM_DISK_SPACE; - delete process.env.HF_TOKEN; - delete process.env.HUGGING_FACE_HUB_TOKEN; - }); - - afterEach(() => { - vi.restoreAllMocks(); - mkdirSpy.mockRestore(); - stdoutWrite.mockRestore(); - process.env = { ...originalEnv }; - }); - - it.each([ - ["NEMOCLAW_VLLM_PORT", "9000", "requires NEMOCLAW_VLLM_PORT=8000"], - ["NEMOCLAW_VLLM_PORT", "08000", "requires NEMOCLAW_VLLM_PORT=8000"], - ["NEMOCLAW_CONTEXT_WINDOW", "8192", "requires NEMOCLAW_CONTEXT_WINDOW=262144"], - ["NEMOCLAW_CONTEXT_WINDOW", "0262144", "requires NEMOCLAW_CONTEXT_WINDOW=262144"], - [ - "NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL", - "http://127.0.0.1", - "requires NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL=http://host.openshell.internal", - ], - ])("rejects conflicting %s before managed-vLLM side effects", async (name, value, message) => { - process.env[name] = value; - const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; - const beforeInstall = vi.fn(); - mockSuccessfulVllmInstall(profile.containerName); - - const result = await installVllm(profile, { - hasImage: true, - nonInteractive: true, - promptFn: vi.fn(), - beforeInstall, - }); - - expect(result).toEqual({ ok: false }); - expect(beforeInstall).not.toHaveBeenCalled(); - expect(mocks.probeHostStorage).not.toHaveBeenCalled(); - expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); - expect(errSpy.mock.calls.flat().join("\n")).toContain(message); - }); - - it("fails closed before downloads when the qualified image is absent", async () => { - const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; - const beforeInstall = vi.fn(); - mockSuccessfulVllmInstall(profile.containerName); - mocks.dockerImageInspectFormat.mockReturnValue(""); - - const result = await installVllm(profile, { - hasImage: false, - nonInteractive: true, - promptFn: vi.fn(), - beforeInstall, - }); - - expect(result).toEqual({ ok: false }); - expect(beforeInstall).toHaveBeenCalledWith("nemotron-ultra"); - expect(mocks.probeHostStorage).not.toHaveBeenCalled(); - expect(mocks.probeDockerStorage).not.toHaveBeenCalled(); - expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); - expect(mocks.dockerSpawn).not.toHaveBeenCalled(); - expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); - const errors = errSpy.mock.calls.map((call: unknown[]) => String(call[0])).join("\n"); - expect(errors).toContain("qualified experimental runtime image is not published"); - expect(errors).toContain(NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId); - expect(errors).toContain("No fallback image was substituted"); - }); - - it("runs the exact preloaded profile and validates both readiness endpoints", async () => { - mocks.getGpuIndicesByName.mockReturnValue([0]); - const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; - const beforeInstall = vi.fn(); - mockSuccessfulVllmInstall(profile.containerName, { - healthStatus: "200", - modelsResponse: '{"data":[{"id":"nemotron-ultra","max_model_len":262144}]}', - }); - mocks.dockerImageInspectFormat.mockReturnValue( - NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, - ); - - const result = await installVllm(profile, { - hasImage: false, - nonInteractive: true, - promptFn: vi.fn(), - beforeInstall, - }); - - expect(result).toEqual({ ok: true }); - expect(beforeInstall).toHaveBeenCalledWith("nemotron-ultra"); - expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); - const curlCalls = mocks.runCapture.mock.calls - .map((call) => call[0] as string[]) - .filter((args) => args[0] === "curl"); - expect(curlCalls.map((args) => args.at(-1))).toEqual([ - "http://127.0.0.1:8000/health", - "http://127.0.0.1:8000/v1/models", - ]); - expect(curlCalls[0]).toEqual(expect.arrayContaining(["-o", "/dev/null", "-w", "%{http_code}"])); - const telemetryCalls = mocks.runCapture.mock.calls - .map((call) => call[0] as string[]) - .filter((args) => args[0] === "nvidia-smi"); - expect(telemetryCalls).toHaveLength(3); - expect(telemetryCalls[0]).toEqual([ - "nvidia-smi", - "--id=0", - "--query-gpu=memory.used,ecc.errors.uncorrected.aggregate.total,ecc.errors.uncorrected.volatile.total", - "--format=csv,noheader,nounits", - ]); - expect(mocks.getGpuIndicesByName).toHaveBeenCalledTimes(2); - - const [runArgs] = mocks.dockerRunDetached.mock.calls[0] as [string[]]; - expect(runArgs).not.toContain("-p"); - expect(runArgs).toEqual(expect.arrayContaining(["--gpus", "device=0"])); - expect(runArgs).toEqual( - expect.arrayContaining([ - "--network", - "host", - "--ipc=host", - "--shm-size", - "16g", - "--memory", - "650g", - NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, - ]), - ); - const serveCommand = runArgs.at(-1) ?? ""; - expect(serveCommand).toContain( - "vllm serve /root/.cache/huggingface/hub/models--nvidia--NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4/snapshots/183968f87ae4cedce3039313cac1fd43d112c578", - ); - expect(serveCommand).toContain("--served-model-name nemotron-ultra"); - expect(serveCommand).not.toContain("--enable-prefix-caching"); - expect(serveCommand).not.toContain("--speculative-config"); - }); - - it("fails closed when the qualified GPU selection changes before launch", async () => { - mocks.getGpuIndicesByName.mockReturnValueOnce([0]).mockReturnValueOnce([]); - const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; - mockSuccessfulVllmInstall(profile.containerName); - mocks.dockerImageInspectFormat.mockReturnValue( - NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, - ); - - const result = await installVllm(profile, { - hasImage: true, - nonInteractive: true, - promptFn: vi.fn(), - }); - - expect(result).toEqual({ ok: false }); - expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); - expect(errSpy).toHaveBeenCalledWith( - expect.stringContaining("qualified GPU selection changed before launch"), - ); - }); - - it.each([ - ["non-200 health", "503", '{"data":[{"id":"nemotron-ultra","max_model_len":262144}]}'], - ["wrong alias", "200", '{"data":[{"id":"other-model","max_model_len":262144}]}'], - [ - "extra model", - "200", - '{"data":[{"id":"nemotron-ultra","max_model_len":262144},{"id":"other","max_model_len":262144}]}', - ], - ["wrong context", "200", '{"data":[{"id":"nemotron-ultra","max_model_len":131072}]}'], - ["string context", "200", '{"data":[{"id":"nemotron-ultra","max_model_len":"262144"}]}'], - ["missing context", "200", '{"data":[{"id":"nemotron-ultra"}]}'], - ["malformed catalog", "200", "not-json"], - ])("does not accept readiness with %s", async (_name, healthStatus, modelsResponse) => { - mocks.getGpuIndicesByName.mockReturnValue([0]); - const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; - mockSuccessfulVllmInstall(profile.containerName, { healthStatus, modelsResponse }); - mocks.dockerImageInspectFormat.mockReturnValue( - NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, - ); - mocks.dockerCapture.mockImplementation(() => ""); - - const result = await installVllm(profile, { - hasImage: true, - nonInteractive: true, - promptFn: vi.fn(), - }); - - expect(result).toEqual({ ok: false }); - expect(mocks.dockerRunDetached).toHaveBeenCalledTimes(1); - expect(mocks.dockerStop).toHaveBeenCalledWith( - profile.containerName, - expect.objectContaining({ ignoreError: true, suppressOutput: true }), - ); - }); - - it("rejects an image identity mismatch before storage or downloads", async () => { - const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; - mockSuccessfulVllmInstall(profile.containerName); - mocks.dockerImageInspectFormat.mockReturnValue("sha256:not-qualified"); - - const result = await installVllm(profile, { - hasImage: true, - nonInteractive: true, - promptFn: vi.fn(), - }); - - expect(result).toEqual({ ok: false }); - expect(mocks.probeHostStorage).not.toHaveBeenCalled(); - expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); - expect(mocks.dockerSpawn).not.toHaveBeenCalled(); - expect(errSpy).toHaveBeenCalledWith( - expect.stringContaining("qualified image identity mismatch"), - ); - }); - - it("fails before downloads when the qualified Station GPU is unavailable", async () => { - const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; - mockSuccessfulVllmInstall(profile.containerName); - mocks.dockerImageInspectFormat.mockReturnValue( - NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, - ); - mocks.getGpuIndicesByName.mockReturnValue([]); - - const result = await installVllm(profile, { - hasImage: true, - nonInteractive: true, - promptFn: vi.fn(), - }); - - expect(result).toEqual({ ok: false }); - expect(mocks.probeHostStorage).not.toHaveBeenCalled(); - expect(mocks.dockerSpawn).not.toHaveBeenCalled(); - expect(errSpy).toHaveBeenCalledWith( - expect.stringContaining("requires exactly 1 GPU matching 'GB300'; detected 0"), - ); - }); - - it.each([ - ["HBM ceiling", "245000, 0, 0", "qualified HBM safety ceiling exceeded"], - ["aggregate ECC", "1000, 1, 0", "uncorrected ECC is nonzero"], - ["volatile ECC", "1000, 0, 1", "uncorrected ECC is nonzero"], - ["malformed telemetry", "N/A, 0, 0", "telemetry is unavailable or malformed"], - ])("fails before downloads when qualified %s preflight fails", async (_name, sample, message) => { - mocks.getGpuIndicesByName.mockReturnValue([0]); - const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; - mockSuccessfulVllmInstall(profile.containerName, { qualificationSamples: [sample] }); - mocks.dockerImageInspectFormat.mockReturnValue( - NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, - ); - - const result = await installVllm(profile, { - hasImage: true, - nonInteractive: true, - promptFn: vi.fn(), - }); - - expect(result).toEqual({ ok: false }); - expect(mocks.probeHostStorage).not.toHaveBeenCalled(); - expect(mocks.dockerSpawn).not.toHaveBeenCalled(); - expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); - expect(errSpy).toHaveBeenCalledWith(expect.stringContaining(message)); - }); - - it("stops the managed container when startup HBM reaches the exclusive ceiling", async () => { - mocks.getGpuIndicesByName.mockReturnValue([0]); - const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; - mockSuccessfulVllmInstall(profile.containerName, { - healthStatus: "200", - modelsResponse: '{"data":[{"id":"nemotron-ultra","max_model_len":262144}]}', - qualificationSamples: ["1000, 0, 0", "1000, 0, 0", "245000, 0, 0"], - }); - mocks.dockerImageInspectFormat.mockReturnValue( - NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, - ); - - const result = await installVllm(profile, { - hasImage: true, - nonInteractive: true, - promptFn: vi.fn(), - }); - - expect(result).toEqual({ ok: false }); - expect(mocks.dockerRunDetached).toHaveBeenCalledTimes(1); - expect(mocks.dockerStop).toHaveBeenCalledWith( - profile.containerName, - expect.objectContaining({ ignoreError: true, suppressOutput: true }), - ); - expect(errSpy).toHaveBeenCalledWith( - expect.stringContaining("qualified HBM safety ceiling exceeded"), - ); - }); -}); diff --git a/src/lib/inference/vllm-models.test.ts b/src/lib/inference/vllm-models.test.ts index 57b9bb2d564..aa8d33a9e64 100644 --- a/src/lib/inference/vllm-models.test.ts +++ b/src/lib/inference/vllm-models.test.ts @@ -7,17 +7,12 @@ import { assertGatedModelAccess, buildVllmServeCommand, DEFAULT_VLLM_MODEL, - EXPERIMENTAL_SINGLE_USER_PROFILE, modelsForPlatform, - NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE, parseVllmExtraServeArgs, preflightVllmModelEnv, - resolveVllmModelProfile, selectVllmModelFromEnv, - selectVllmModelProfileFromEnv, VLLM_EXTRA_ARGS_ENV, VLLM_MODELS, - VLLM_PROFILE_ENV, } from "./vllm-models"; describe("vllm model registry", () => { @@ -106,174 +101,6 @@ describe("vllm model registry", () => { ); }); - it("keeps the qualified single-user recipe explicit, exact, and separate from the canonical Ultra recipe", () => { - const ultra = VLLM_MODELS.find((m) => m.envValue === "nemotron-3-ultra-550b-a55b"); - expect(ultra).toBeDefined(); - - const canonical = resolveVllmModelProfile(ultra!, "station", {} as NodeJS.ProcessEnv); - expect(canonical).toEqual({ model: ultra, profile: null }); - expect(canonical.model).toBe(ultra); - - const resolved = resolveVllmModelProfile(ultra!, "station", { - [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, - } as NodeJS.ProcessEnv); - const profile = resolved.profile; - expect(profile).toBeDefined(); - expect(profile).toEqual({ - id: "experimental-single-user", - label: "Experimental single-user", - experimental: true, - optIn: true, - platforms: ["station"], - servedModelId: "nemotron-ultra", - revision: "183968f87ae4cedce3039313cac1fd43d112c578", - requiredVllmPort: 8000, - requiredSandboxHostUrl: "http://host.openshell.internal", - exactServeArgs: [ - "--served-model-name", - "nemotron-ultra", - "--host", - "0.0.0.0", - "--port", - "8000", - "--tensor-parallel-size", - "1", - "--trust-remote-code", - "--cpu-offload-gb", - "90", - "--cpu-offload-params", - "experts", - "--kernel_config", - `'{"enable_flashinfer_autotune": false}'`, - "--max-num-seqs", - "1", - "--gpu-memory-utilization", - "0.9", - "--kv-cache-memory-bytes", - "4294967296", - "--reasoning-parser", - "nemotron_v3", - "--enable-auto-tool-choice", - "--tool-call-parser", - "qwen3_coder", - ], - runtime: { - image: NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, - imageDownloadSizeBytes: 10_670_087_425, - modelDownloadSizeBytes: 352_381_245_521, - loadTimeoutSec: 1200, - dockerRunArgs: ["--shm-size", "16g", "--memory", "650g"], - networkMode: "host", - containerEnv: { - VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: "1", - VLLM_NVFP4_GEMM_BACKEND: "flashinfer-trtllm", - NEMOTRON_ULTRA_MODEL_OPT_NVFP4_MOE_RECLAIM: "1", - HF_HUB_OFFLINE: "1", - TRANSFORMERS_OFFLINE: "1", - PYTHONHASHSEED: "0", - }, - imagePullPolicy: "local-only", - expectedImageId: "sha256:e44830d3f0197850e5a63ca895b13a047f358ff4aea15b1655abf61391bb883e", - qualification: { - gpuNameIncludes: "GB300", - gpuCount: 1, - hbmSafetyCeilingMiB: 245000, - requireZeroUncorrectedEcc: true, - }, - }, - readiness: { - healthEndpoint: true, - servedModelId: "nemotron-ultra", - maxModelLen: 262144, - }, - }); - - expect(resolved.model.revision).toBe("183968f87ae4cedce3039313cac1fd43d112c578"); - expect(resolved.model.servedModelId).toBe("nemotron-ultra"); - expect(resolved.model.serveModelPath).toMatch( - /\/snapshots\/183968f87ae4cedce3039313cac1fd43d112c578$/, - ); - expect(resolved.model.readiness).toEqual({ - healthEndpoint: true, - servedModelId: "nemotron-ultra", - maxModelLen: 262144, - }); - expect(resolved.model.serveEnv).toBeUndefined(); - - const cmd = buildVllmServeCommand(resolved.model, {} as NodeJS.ProcessEnv); - expect(cmd).toBe( - [ - "vllm serve", - "/root/.cache/huggingface/hub/models--nvidia--NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4/snapshots/183968f87ae4cedce3039313cac1fd43d112c578", - "--served-model-name nemotron-ultra", - "--host 0.0.0.0", - "--port 8000", - "--tensor-parallel-size 1", - "--trust-remote-code", - "--cpu-offload-gb 90", - "--cpu-offload-params experts", - `--kernel_config '{"enable_flashinfer_autotune": false}'`, - "--max-num-seqs 1", - "--gpu-memory-utilization 0.9", - "--kv-cache-memory-bytes 4294967296", - "--reasoning-parser nemotron_v3", - "--enable-auto-tool-choice", - "--tool-call-parser qwen3_coder", - ].join(" "), - ); - for (const forbidden of [ - "--enable-prefix-caching", - "--speculative-config", - "--default-chat-template-kwargs", - "--pipeline-parallel-size", - "--data-parallel-size", - "--cpu-offload-gb 150", - "--max-num-seqs 256", - "--tool-call-parser hermes", - `{"enable_flashinfer_autotune": true}`, - ]) { - expect(cmd).not.toContain(forbidden); - } - - const driftedCanonical = { ...ultra!, revision: "future-canonical-revision" }; - const independentlyPinned = resolveVllmModelProfile(driftedCanonical, "station", { - [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, - } as NodeJS.ProcessEnv).model; - expect(independentlyPinned.revision).toBe("183968f87ae4cedce3039313cac1fd43d112c578"); - expect(independentlyPinned.serveModelPath).toMatch( - /\/snapshots\/183968f87ae4cedce3039313cac1fd43d112c578$/, - ); - }); - - it("rejects invalid or incompatible explicit model profiles before runtime work", () => { - const ultra = VLLM_MODELS.find((m) => m.envValue === "nemotron-3-ultra-550b-a55b"); - const qwen = VLLM_MODELS.find((m) => m.envValue === "qwen3.6-27b"); - expect(ultra).toBeDefined(); - expect(qwen).toBeDefined(); - - expect(() => - selectVllmModelProfileFromEnv({ - [VLLM_PROFILE_ENV]: "unknown-profile", - } as NodeJS.ProcessEnv), - ).toThrow(/Unknown NEMOCLAW_VLLM_PROFILE='unknown-profile'/); - expect(() => - resolveVllmModelProfile(qwen!, "station", { - [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, - } as NodeJS.ProcessEnv), - ).toThrow(/requires NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b'/); - expect(() => - resolveVllmModelProfile(ultra!, "linux", { - [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, - } as NodeJS.ProcessEnv), - ).toThrow(/is not supported on linux/); - expect(() => - resolveVllmModelProfile(ultra!, "station", { - [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, - [VLLM_EXTRA_ARGS_ENV]: '["--enable-prefix-caching"]', - } as NodeJS.ProcessEnv), - ).toThrow(/cannot be combined with NEMOCLAW_VLLM_EXTRA_ARGS_JSON/); - }); - it("rejects an unknown NEMOCLAW_VLLM_MODEL with a helpful message", () => { expect(() => selectVllmModelFromEnv({ NEMOCLAW_VLLM_MODEL: "made-up-model" } as NodeJS.ProcessEnv), @@ -559,86 +386,6 @@ describe("preflightVllmModelEnv", () => { ).toEqual({ ok: true }); }); - it("succeeds for the explicit qualified Station profile and matching Ultra model", () => { - expect( - preflightVllmModelEnv({ - NEMOCLAW_VLLM_MODEL: "nemotron-3-ultra-550b-a55b", - [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, - NEMOCLAW_VLLM_PORT: "8000", - NEMOCLAW_CONTEXT_WINDOW: "262144", - NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL: "host.openshell.internal", - } as NodeJS.ProcessEnv), - ).toEqual({ ok: true }); - }); - - it("rejects invalid or incompatible profile input", () => { - expect( - preflightVllmModelEnv({ - [VLLM_PROFILE_ENV]: "unknown-profile", - } as NodeJS.ProcessEnv), - ).toMatchObject({ - ok: false, - message: expect.stringMatching(/Unknown NEMOCLAW_VLLM_PROFILE='unknown-profile'/), - }); - expect( - preflightVllmModelEnv({ - [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, - } as NodeJS.ProcessEnv), - ).toEqual({ - ok: false, - message: - "NEMOCLAW_VLLM_PROFILE='experimental-single-user' requires " + - "NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b'.", - }); - expect( - preflightVllmModelEnv({ - NEMOCLAW_VLLM_MODEL: "qwen3.6-27b", - [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, - } as NodeJS.ProcessEnv), - ).toEqual({ - ok: false, - message: - "NEMOCLAW_VLLM_PROFILE='experimental-single-user' requires " + - "NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b'.", - }); - expect( - preflightVllmModelEnv({ - NEMOCLAW_VLLM_MODEL: "nemotron-3-ultra-550b-a55b", - [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, - [VLLM_EXTRA_ARGS_ENV]: '["--enable-prefix-caching"]', - } as NodeJS.ProcessEnv), - ).toEqual({ - ok: false, - message: - "NEMOCLAW_VLLM_PROFILE='experimental-single-user' cannot be combined with " + - "NEMOCLAW_VLLM_EXTRA_ARGS_JSON; extra arguments would invalidate the qualified " + - "runtime contract.", - }); - }); - - it.each([ - ["NEMOCLAW_VLLM_PORT", "9000", /requires NEMOCLAW_VLLM_PORT=8000/], - ["NEMOCLAW_VLLM_PORT", "08000", /requires NEMOCLAW_VLLM_PORT=8000/], - ["NEMOCLAW_CONTEXT_WINDOW", "8192", /requires NEMOCLAW_CONTEXT_WINDOW=262144/], - ["NEMOCLAW_CONTEXT_WINDOW", "0262144", /requires NEMOCLAW_CONTEXT_WINDOW=262144/], - [ - "NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL", - "http://127.0.0.1", - /requires NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL=http:\/\/host\.openshell\.internal/, - ], - ])("rejects conflicting qualified-profile runtime input %s", (name, value, message) => { - expect( - preflightVllmModelEnv({ - NEMOCLAW_VLLM_MODEL: "nemotron-3-ultra-550b-a55b", - [VLLM_PROFILE_ENV]: EXPERIMENTAL_SINGLE_USER_PROFILE, - [name]: value, - } as NodeJS.ProcessEnv), - ).toEqual({ - ok: false, - message: expect.stringMatching(message), - }); - }); - it("succeeds for a gated slug when HF_TOKEN is set", () => { expect( preflightVllmModelEnv({ diff --git a/src/lib/inference/vllm-models.ts b/src/lib/inference/vllm-models.ts index 45c4c0a0f96..e0b927e61e9 100644 --- a/src/lib/inference/vllm-models.ts +++ b/src/lib/inference/vllm-models.ts @@ -27,34 +27,8 @@ * envelope, and tool-call behaviour validated. */ -import { HOST_GATEWAY_URL, LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV } from "./local-host-url"; - export type VllmPlatform = "spark" | "station" | "linux"; -export const VLLM_PROFILE_ENV = "NEMOCLAW_VLLM_PROFILE"; -export const EXPERIMENTAL_SINGLE_USER_PROFILE = "experimental-single-user" as const; -export type VllmModelProfileId = typeof EXPERIMENTAL_SINGLE_USER_PROFILE; - -export interface VllmRuntimeQualification { - /** GPU model fragment used to select the qualified accelerator. */ - gpuNameIncludes: string; - /** Exact number of matching accelerators required by this single-user recipe. */ - gpuCount: number; - /** Exclusive HBM ceiling used by the Station qualification procedure. */ - hbmSafetyCeilingMiB: number; - /** The qualification procedure must fail closed on non-zero uncorrected ECC. */ - requireZeroUncorrectedEcc: boolean; -} - -export interface VllmReadinessContract { - /** Require the health endpoint in addition to the OpenAI model catalog. */ - healthEndpoint: boolean; - /** Require one exact served alias rather than accepting any non-empty catalog. */ - servedModelId: string; - /** Exact context length that the model catalog must advertise. */ - maxModelLen: number; -} - export interface VllmRuntimeOverride { /** Model-specific runtime image, pinned by digest. */ image: string; @@ -66,42 +40,6 @@ export interface VllmRuntimeOverride { loadTimeoutSec?: number; /** Additional `docker run` arguments required by this recipe. */ dockerRunArgs?: readonly string[]; - /** Networking mode for the long-lived inference container. Defaults to bridge. */ - networkMode?: "bridge" | "host"; - /** Non-secret environment passed to the long-lived inference container. */ - containerEnv?: Readonly>; - /** Registry images are pulled; local-only images must already exist exactly. */ - imagePullPolicy?: "registry" | "local-only"; - /** Docker image/config ID that must back image, even when a manifest ref is used. */ - expectedImageId?: string; - /** Hardware qualification limits associated with this reviewed recipe. */ - qualification?: VllmRuntimeQualification; -} - -export interface VllmModelProfileOverride { - /** Stable opt-in selector consumed from NEMOCLAW_VLLM_PROFILE. */ - id: VllmModelProfileId; - /** Human label for summaries and documentation. */ - label: string; - /** Profiles in this registry are never selected implicitly. */ - experimental: true; - optIn: true; - /** Platforms on which this exact recipe was qualified. */ - platforms: readonly VllmPlatform[]; - /** Served alias used by the gateway and dashboard. */ - servedModelId: string; - /** Immutable Hugging Face revision owned by this profile. */ - revision: string; - /** Host port that must stay aligned with this profile's exact serve arguments. */ - requiredVllmPort: number; - /** Sandbox-facing host route required by this qualified profile. */ - requiredSandboxHostUrl: string; - /** Complete ordered vLLM arguments after the model ID. Replaces shared/default args. */ - exactServeArgs: readonly string[]; - /** Required runtime/container replacement for this profile. */ - runtime: VllmRuntimeOverride; - /** Strict endpoint contract checked before onboarding continues. */ - readiness: VllmReadinessContract; } export const NEMOTRON_ULTRA_STATION_IMAGE = { @@ -112,14 +50,6 @@ export const NEMOTRON_ULTRA_STATION_IMAGE = { }, } as const; -// Exact Docker image/config ID qualified by the reclaimed-v1 offload90 study. -// It is local-only until a repository-qualified manifest publishes these bytes. -// Once published, image can become that manifest reference while expectedImageId -// continues to prove that the pulled image resolves to this exact ID. -export const NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE = { - localImageId: "sha256:e44830d3f0197850e5a63ca895b13a047f358ff4aea15b1655abf61391bb883e", -} as const; - export interface VllmModelDef { /** Hugging Face model id (also passed to `vllm serve`). */ id: string; @@ -133,8 +63,6 @@ export interface VllmModelDef { revision?: string; /** Stable model name exposed by the local OpenAI-compatible endpoint. */ servedModelId?: string; - /** Exact snapshot path used by a resolved profile instead of the repository ID. */ - serveModelPath?: string; /** Model-specific flags appended after the shared serving flags. */ modelArgs: string[]; /** True when the upstream HF repo requires accepting a licence. */ @@ -157,12 +85,6 @@ export interface VllmModelDef { serveEnv?: Record; /** Runtime overrides for recipes that cannot use the platform image. */ runtime?: VllmRuntimeOverride; - /** Complete serve args for a resolved model profile; base entries use shared args. */ - exactServeArgs?: readonly string[]; - /** Strict readiness contract for a resolved model profile. */ - readiness?: VllmReadinessContract; - /** Explicit model-scoped profiles. These are never interactive/default choices. */ - profiles?: Partial>; /** Whether startup must install vLLM's fastsafetensors extra. Defaults to true. */ installFastSafetensors?: boolean; } @@ -325,78 +247,6 @@ export const VLLM_MODELS: readonly VllmModelDef[] = [ // importing the playbook's host-network setting. dockerRunArgs: ["--shm-size", "16g", "--ulimit", "memlock=-1", "--ulimit", "stack=67108864"], }, - profiles: { - [EXPERIMENTAL_SINGLE_USER_PROFILE]: { - id: EXPERIMENTAL_SINGLE_USER_PROFILE, - label: "Experimental single-user", - experimental: true, - optIn: true, - platforms: ["station"], - servedModelId: "nemotron-ultra", - revision: "183968f87ae4cedce3039313cac1fd43d112c578", - requiredVllmPort: 8000, - requiredSandboxHostUrl: HOST_GATEWAY_URL, - exactServeArgs: [ - "--served-model-name", - "nemotron-ultra", - "--host", - "0.0.0.0", - "--port", - "8000", - "--tensor-parallel-size", - "1", - "--trust-remote-code", - "--cpu-offload-gb", - "90", - "--cpu-offload-params", - "experts", - "--kernel_config", - "'{\"enable_flashinfer_autotune\": false}'", - "--max-num-seqs", - "1", - "--gpu-memory-utilization", - "0.9", - "--kv-cache-memory-bytes", - "4294967296", - "--reasoning-parser", - "nemotron_v3", - "--enable-auto-tool-choice", - "--tool-call-parser", - "qwen3_coder", - ], - runtime: { - image: NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, - // Unused while local-only. A future manifest publication must make - // its compressed pull accounting explicit before this policy changes. - imageDownloadSizeBytes: NEMOTRON_ULTRA_STATION_IMAGE.arm64.downloadSizeBytes, - modelDownloadSizeBytes: 352_381_245_521, - loadTimeoutSec: 1200, - dockerRunArgs: ["--shm-size", "16g", "--memory", "650g"], - networkMode: "host", - containerEnv: { - VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: "1", - VLLM_NVFP4_GEMM_BACKEND: "flashinfer-trtllm", - NEMOTRON_ULTRA_MODEL_OPT_NVFP4_MOE_RECLAIM: "1", - HF_HUB_OFFLINE: "1", - TRANSFORMERS_OFFLINE: "1", - PYTHONHASHSEED: "0", - }, - imagePullPolicy: "local-only", - expectedImageId: NEMOTRON_ULTRA_EXPERIMENTAL_SINGLE_USER_IMAGE.localImageId, - qualification: { - gpuNameIncludes: "GB300", - gpuCount: 1, - hbmSafetyCeilingMiB: 245000, - requireZeroUncorrectedEcc: true, - }, - }, - readiness: { - healthEndpoint: true, - servedModelId: "nemotron-ultra", - maxModelLen: 262144, - }, - }, - }, // The digest-pinned vLLM image already contains the serving package, and // this recipe does not use the fastsafetensors load format. Avoid mutating // the reviewed runtime from the package index when the container starts. @@ -472,133 +322,6 @@ export function modelsForPlatform(platform: VllmPlatform): readonly VllmModelDef const HF_TOKEN_ENV_KEYS = ["HF_TOKEN", "HUGGING_FACE_HUB_TOKEN"] as const; export const VLLM_EXTRA_ARGS_ENV = "NEMOCLAW_VLLM_EXTRA_ARGS_JSON"; -/** Parse and validate the explicit model-profile selector. */ -export function selectVllmModelProfileFromEnv( - env: NodeJS.ProcessEnv = process.env, -): VllmModelProfileId | null { - const requested = String(env[VLLM_PROFILE_ENV] ?? "") - .trim() - .toLowerCase(); - if (!requested) return null; - if (requested === EXPERIMENTAL_SINGLE_USER_PROFILE) return requested; - throw new Error( - "Unknown " + - VLLM_PROFILE_ENV + - "='" + - String(env[VLLM_PROFILE_ENV]) + - "'. Recognised values: '" + - EXPERIMENTAL_SINGLE_USER_PROFILE + - "'.", - ); -} - -export interface ResolvedVllmModelProfile { - model: VllmModelDef; - profile: VllmModelProfileOverride | null; -} - -function huggingFaceSnapshotContainerPath(modelId: string, revision: string): string { - return ( - "/root/.cache/huggingface/hub/models--" + - modelId.replaceAll("/", "--") + - "/snapshots/" + - revision - ); -} - -function assertVllmModelProfileRuntimeInputs( - profile: VllmModelProfileOverride, - env: NodeJS.ProcessEnv, -): void { - const requestedPort = String(env.NEMOCLAW_VLLM_PORT ?? "").trim(); - if (requestedPort && requestedPort !== String(profile.requiredVllmPort)) { - throw new Error( - `${VLLM_PROFILE_ENV}='${profile.id}' requires ` + - `NEMOCLAW_VLLM_PORT=${String(profile.requiredVllmPort)}; remove the override or set it to ` + - `${String(profile.requiredVllmPort)}.`, - ); - } - - const requiredContextWindow = profile.readiness.maxModelLen; - const requestedContextWindow = String(env.NEMOCLAW_CONTEXT_WINDOW ?? "").trim(); - if (requestedContextWindow && requestedContextWindow !== String(requiredContextWindow)) { - throw new Error( - `${VLLM_PROFILE_ENV}='${profile.id}' requires ` + - `NEMOCLAW_CONTEXT_WINDOW=${String(requiredContextWindow)}; remove the override or set it to ` + - `${String(requiredContextWindow)}.`, - ); - } - - const requestedSandboxHostUrl = String(env[LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV] ?? "").trim(); - const requiredSandboxHost = profile.requiredSandboxHostUrl.replace(/^http:\/\//, ""); - if ( - requestedSandboxHostUrl && - requestedSandboxHostUrl.replace(/\/+$/, "") !== profile.requiredSandboxHostUrl && - requestedSandboxHostUrl.replace(/\/+$/, "") !== requiredSandboxHost - ) { - throw new Error( - `${VLLM_PROFILE_ENV}='${profile.id}' requires ` + - `${LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV}=${profile.requiredSandboxHostUrl}; remove the ` + - `override or set it to ${profile.requiredSandboxHostUrl}.`, - ); - } -} - -/** - * Resolve an explicit model-scoped profile into the effective recipe consumed - * by the existing managed-vLLM lifecycle. With no selector this returns the - * original model object unchanged, which freezes the canonical Station output. - */ -export function resolveVllmModelProfile( - model: VllmModelDef, - platform: VllmPlatform, - env: NodeJS.ProcessEnv = process.env, -): ResolvedVllmModelProfile { - const profileId = selectVllmModelProfileFromEnv(env); - if (!profileId) return { model, profile: null }; - - const profile = model.profiles?.[profileId]; - if (!profile) { - throw new Error( - VLLM_PROFILE_ENV + - "='" + - profileId + - "' requires NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b'.", - ); - } - if (!profile.platforms.includes(platform)) { - throw new Error( - VLLM_PROFILE_ENV + "='" + profileId + "' is not supported on " + platform + ".", - ); - } - if (parseVllmExtraServeArgs(env).length > 0) { - throw new Error( - VLLM_PROFILE_ENV + - "='" + - profileId + - "' cannot be combined with " + - VLLM_EXTRA_ARGS_ENV + - "; extra arguments would invalidate the qualified runtime contract.", - ); - } - assertVllmModelProfileRuntimeInputs(profile, env); - - return { - profile, - model: { - ...model, - revision: profile.revision, - servedModelId: profile.servedModelId, - serveModelPath: huggingFaceSnapshotContainerPath(model.id, profile.revision), - modelArgs: [], - serveEnv: undefined, - runtime: profile.runtime, - exactServeArgs: [...profile.exactServeArgs], - readiness: profile.readiness, - }, - }; -} - /** * Look up the requested express-vLLM model from `NEMOCLAW_VLLM_MODEL`. * Returns `null` when the env var is empty so the caller can fall back to @@ -673,40 +396,9 @@ export function preflightVllmModelEnv( env: NodeJS.ProcessEnv = process.env, ): PreflightVllmModelResult { try { - const profileId = selectVllmModelProfileFromEnv(env); - const extraArgs = parseVllmExtraServeArgs(env); - if (profileId && extraArgs.length > 0) { - throw new Error( - VLLM_PROFILE_ENV + - "='" + - profileId + - "' cannot be combined with " + - VLLM_EXTRA_ARGS_ENV + - "; extra arguments would invalidate the qualified runtime contract.", - ); - } + parseVllmExtraServeArgs(env); const model = selectVllmModelFromEnv(env); - if (!model) { - if (profileId) { - throw new Error( - VLLM_PROFILE_ENV + - "='" + - profileId + - "' requires NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b'.", - ); - } - return { ok: true }; - } - const modelProfile = profileId ? model.profiles?.[profileId] : undefined; - if (profileId && !modelProfile) { - throw new Error( - VLLM_PROFILE_ENV + - "='" + - profileId + - "' requires NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b'.", - ); - } - if (modelProfile) assertVllmModelProfileRuntimeInputs(modelProfile, env); + if (!model) return { ok: true }; assertGatedModelAccess(model, env); return { ok: true }; } catch (err) { @@ -786,21 +478,16 @@ export function buildVllmServeCommand( .map(([key, value]) => `export ${key}=${value}`) .join(" && ")} && ` : ""; - const args = model.exactServeArgs - ? [...model.exactServeArgs] - : [ - ...SHARED_VLLM_ARGS, - "--max-model-len", - String(model.maxModelLen), - ...(model.revision ? ["--revision", model.revision] : []), - ...(model.servedModelId ? ["--served-model-name", model.servedModelId] : []), - ...model.modelArgs, - ]; + const args = [ + ...SHARED_VLLM_ARGS, + "--max-model-len", + String(model.maxModelLen), + ...(model.revision ? ["--revision", model.revision] : []), + ...(model.servedModelId ? ["--served-model-name", model.servedModelId] : []), + ...model.modelArgs, + ]; const extraArgs = parseVllmExtraServeArgs(env).map(shellQuote); const setup = model.installFastSafetensors === false ? "" : "pip install vllm[fastsafetensors] && "; - return `${envPrefix}${setup}vllm serve ${model.serveModelPath ?? model.id} ${[ - ...args, - ...extraArgs, - ].join(" ")}`; + return `${envPrefix}${setup}vllm serve ${model.id} ${[...args, ...extraArgs].join(" ")}`; } diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index 3c0ea886d05..b924d35ae71 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -54,6 +54,7 @@ vi.mock("./vllm-storage", async (importOriginal) => { }); import { + assertVllmRegistryDigestRef, buildVllmRunArgs, detectVllmProfile, installVllm, @@ -63,6 +64,7 @@ import { pullImage, resolveVllmRuntimeProfile, resolveVllmServedModelId, + VLLM_IMAGES, } from "./vllm"; import { buildVllmServeCommand, VLLM_MODELS } from "./vllm-models"; @@ -116,25 +118,14 @@ function vllmContainerRow( function mockSuccessfulVllmInstall( containerName: string, ownershipResponses: readonly (() => string)[] = [() => "", () => ""], - endpoints: { - healthStatus?: string; - modelsResponse?: string; - } = {}, ): void { - mocks.runCapture.mockImplementation((cmd: readonly string[]) => { - switch (cmd[0]) { - case "sh": - return "/usr/bin/tool\n"; - case "curl": { - const url = cmd.at(-1) ?? ""; - return url.endsWith("/health") - ? (endpoints.healthStatus ?? "") - : (endpoints.modelsResponse ?? '{"data":[]}'); - } - default: - return ""; - } - }); + const runCaptureByCommand: Record = { + curl: '{"data":[]}', + sh: "/usr/bin/tool\n", + }; + mocks.runCapture.mockImplementation( + (cmd: readonly string[]) => runCaptureByCommand[cmd[0] ?? ""] ?? "", + ); mocks.dockerPullWithProgressWatchdog.mockResolvedValue({ status: 0, signal: null, @@ -177,6 +168,64 @@ describe("vLLM served route identity", () => { }); }); +describe("managed vLLM image distribution boundary", () => { + const digest = `sha256:${"a".repeat(64)}`; + + it("accepts repository-qualified immutable registry digests", () => { + expect(() => assertVllmRegistryDigestRef(`vllm/vllm-openai@${digest}`)).not.toThrow(); + expect(() => + assertVllmRegistryDigestRef(`registry.example.test:5000/team/runtime@${digest}`), + ).not.toThrow(); + }); + + it.each([ + `sha256:${"a".repeat(64)}`, + "vllm/vllm-openai:latest", + `ubuntu@${digest}`, + `vllm/vllm-openai@sha256:${"A".repeat(64)}`, + `vllm/vllm-openai@${digest}suffix`, + ` vllm/vllm-openai@${digest}`, + `vllm/vllm-openai@${digest} `, + ])("rejects an unpullable or mutable product image reference %j", (image) => { + expect(() => assertVllmRegistryDigestRef(image)).toThrow( + /pullable immutable registry reference/, + ); + }); + + it("keeps every shipped managed-vLLM image on a registry digest", () => { + const refs = new Set(); + for (const imageSet of Object.values(VLLM_IMAGES)) { + for (const value of Object.values(imageSet)) { + if (typeof value === "object" && value !== null && "ref" in value) { + refs.add(String(value.ref)); + } + } + } + for (const model of VLLM_MODELS) { + if (model.runtime) refs.add(model.runtime.image); + } + + expect(refs.size).toBeGreaterThan(0); + for (const ref of refs) { + expect(() => assertVllmRegistryDigestRef(ref), ref).not.toThrow(); + } + }); + + it("refuses a local image ID before invoking Docker pull", async () => { + mocks.dockerPullWithProgressWatchdog.mockClear(); + const profile = { + ...detectVllmProfile({ platform: "station", type: "nvidia" })!, + image: `sha256:${"a".repeat(64)}`, + }; + + await expect(pullImage(profile)).resolves.toEqual({ + ok: false, + reason: expect.stringContaining("Local image IDs"), + }); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + }); +}); + describe("vLLM profile detection", () => { beforeEach(() => { vi.clearAllMocks(); @@ -513,7 +562,6 @@ describe("installVllm model resolution", () => { mkdirSpy = vi.spyOn(fs, "mkdirSync").mockImplementation(() => undefined); stdoutWrite = vi.spyOn(process.stdout, "write").mockImplementation(() => true); delete process.env.NEMOCLAW_VLLM_MODEL; - delete process.env.NEMOCLAW_VLLM_PROFILE; delete process.env.NEMOCLAW_VLLM_EXTRA_ARGS_JSON; delete process.env.NEMOCLAW_IGNORE_VLLM_DISK_SPACE; delete process.env.HF_TOKEN; @@ -549,6 +597,29 @@ describe("installVllm model resolution", () => { expect(summary).not.toContain("NEMOCLAW_VLLM_MODEL override"); }); + it("rejects a local image ID before callbacks, prompts, or Docker work", async () => { + const profile = { + ...detectVllmProfile({ platform: "station", type: "nvidia" })!, + image: `sha256:${"a".repeat(64)}`, + }; + const promptFn = vi.fn<(q: string) => Promise>(); + const beforeInstall = vi.fn(); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn, + beforeInstall, + }); + + expect(result).toEqual({ ok: false }); + expect(promptFn).not.toHaveBeenCalled(); + expect(beforeInstall).not.toHaveBeenCalled(); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); + expect(errSpy).toHaveBeenCalledWith(expect.stringContaining("Local image IDs")); + }); + it("annotates the summary as a NEMOCLAW_VLLM_MODEL override when the env var resolves", async () => { process.env.NEMOCLAW_VLLM_MODEL = "qwen3.6-27b"; const profile = detectVllmProfile({ platform: "spark", type: "nvidia" })!; diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index b83a9579391..c4d454a1658 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -17,7 +17,7 @@ import { dockerSpawn, dockerStop, } from "../adapters/docker"; -import { buildCurlProbeSpawnArgs, buildValidatedCurlCommandArgs } from "../adapters/http/curl-args"; +import { buildValidatedCurlCommandArgs } from "../adapters/http/curl-args"; import { VLLM_PORT } from "../core/ports"; import { shellQuote } from "../core/shell-quote"; import { isAffirmativeAnswer } from "../onboard/prompt-helpers"; @@ -29,12 +29,10 @@ import { buildVllmServeCommand, NEMOTRON_ULTRA_STATION_IMAGE, parseVllmExtraServeArgs, - resolveVllmModelProfile, VLLM_EXTRA_ARGS_ENV, VLLM_MODELS, type VllmModelDef, type VllmPlatform, - type VllmRuntimeQualification, } from "./vllm-models"; import { resolveVllmInstallModel } from "./vllm-prompt"; import { @@ -73,10 +71,8 @@ export interface VllmProfile { dockerRunFlags: string[]; // Optional dynamic flag builder. When present, its return value replaces // dockerRunFlags at install time. Used by Station to pick the GB300 GPU - // out of a mixed-GPU host instead of using `--gpus all`; a qualified profile - // passes its frozen, revalidated GPU index so launch cannot rediscover a - // different device. - buildDockerRunFlags?: (qualifiedGpuIndex?: number) => string[]; + // out of a mixed-GPU host instead of using `--gpus all`. + buildDockerRunFlags?: () => string[]; // Maximum wall-clock safety budget for image pulls. The Docker adapter uses // a shorter progress watchdog for stalls, so slow-but-moving pulls can keep // going until this last-ditch cap. @@ -86,14 +82,6 @@ export interface VllmProfile { // Optional pinned model snapshot size. Model-specific runtime overrides use // this to guard the host Hugging Face cache before a cold download. modelDownloadSizeBytes?: number; - // Host mode is profile-scoped because it also suppresses Docker port - // publication; raw run args cannot represent that invariant safely. - networkMode?: "bridge" | "host"; - // Registry is the historical default. Local-only fails closed unless the - // exact qualified Docker image/config ID is already present. - imagePullPolicy?: "registry" | "local-only"; - expectedImageId?: string; - qualification?: VllmRuntimeQualification; } // Platform manifests and decimal compressed sizes published by NGC for the @@ -263,9 +251,8 @@ const STATION_PROFILE: VllmProfile = { defaultModel: deepseekV4FlashModel(), containerName: NEMOCLAW_VLLM_CONTAINER_NAME, dockerRunFlags: SPARK_PROFILE.dockerRunFlags, - buildDockerRunFlags: (qualifiedGpuIndex) => { - const indices = - qualifiedGpuIndex === undefined ? getGpuIndicesByName(/GB300/i) : [qualifiedGpuIndex]; + buildDockerRunFlags: () => { + const indices = getGpuIndicesByName(/GB300/i); // Docker parses --gpus as CSV, so multi-device values must retain // double quotes inside the argv token to keep the comma in one field. const gpuFlag = @@ -346,6 +333,11 @@ function dockerPrereqsOk(): { ok: boolean; reason?: string } { } export async function pullImage(profile: VllmProfile): Promise<{ ok: boolean; reason?: string }> { + try { + assertVllmRegistryDigestRef(profile.image); + } catch (err) { + return { ok: false, reason: (err as Error).message }; + } emit(`Pulling vLLM image: ${profile.image}`); // Docker can be quiet while finalizing large layers on every supported vLLM // profile, so all profiles intentionally share the 15-minute stall default. @@ -492,6 +484,7 @@ export function buildVllmRunArgs( runFlags: readonly string[], env: NodeJS.ProcessEnv = process.env, ): string[] { + assertVllmRegistryDigestRef(profile.image); const image = validateDockerArg(profile.image, "vLLM image"); const containerName = validateDockerArg(profile.containerName, "vLLM container name"); const safeRunFlags = validateDockerArgs(runFlags, "vLLM docker run flags"); @@ -502,7 +495,8 @@ export function buildVllmRunArgs( ...safeRunFlags, "--label", `${NEMOCLAW_VLLM_MANAGED_LABEL}=true`, - ...(profile.networkMode === "host" ? [] : ["-p", String(VLLM_PORT) + ":8000"]), + "-p", + `${String(VLLM_PORT)}:8000`, "--name", containerName, "--entrypoint", @@ -515,30 +509,60 @@ export function buildVllmRunArgs( export function resolveVllmRuntimeProfile(profile: VllmProfile, model: VllmModelDef): VllmProfile { const runtime = model.runtime; - if (!runtime) return profile; - const extraRunArgs = [ - ...(runtime.networkMode === "host" ? ["--network", "host"] : []), - ...(runtime.dockerRunArgs ?? []), - ...Object.entries(runtime.containerEnv ?? {}).flatMap(([key, value]) => [ - "-e", - key + "=" + value, - ]), - ]; - return { - ...profile, - image: runtime.image, - imageDownloadSizeBytes: runtime.imageDownloadSizeBytes, - modelDownloadSizeBytes: runtime.modelDownloadSizeBytes ?? profile.modelDownloadSizeBytes, - loadTimeoutSec: runtime.loadTimeoutSec ?? profile.loadTimeoutSec, - networkMode: runtime.networkMode ?? profile.networkMode, - imagePullPolicy: runtime.imagePullPolicy ?? profile.imagePullPolicy, - expectedImageId: runtime.expectedImageId ?? profile.expectedImageId, - qualification: runtime.qualification ?? profile.qualification, - dockerRunFlags: [...profile.dockerRunFlags, ...extraRunArgs], - buildDockerRunFlags: profile.buildDockerRunFlags - ? (qualifiedGpuIndex) => [...profile.buildDockerRunFlags!(qualifiedGpuIndex), ...extraRunArgs] - : undefined, - }; + let resolved = profile; + if (runtime) { + const extraRunArgs = [...(runtime.dockerRunArgs ?? [])]; + resolved = { + ...profile, + image: runtime.image, + imageDownloadSizeBytes: runtime.imageDownloadSizeBytes, + modelDownloadSizeBytes: runtime.modelDownloadSizeBytes ?? profile.modelDownloadSizeBytes, + loadTimeoutSec: runtime.loadTimeoutSec ?? profile.loadTimeoutSec, + dockerRunFlags: [...profile.dockerRunFlags, ...extraRunArgs], + buildDockerRunFlags: profile.buildDockerRunFlags + ? () => [...profile.buildDockerRunFlags!(), ...extraRunArgs] + : undefined, + }; + } + assertVllmRegistryDigestRef(resolved.image); + return resolved; +} + +const SHA256_IMAGE_DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; +const IMAGE_REPOSITORY_COMPONENT_PATTERN = /^[a-z0-9]+(?:[._-][a-z0-9]+)*$/; + +/** + * Managed vLLM is a product install path, so every effective runtime must be + * downloadable by immutable registry digest. A bare Docker image/config ID + * only identifies bytes already present in one daemon and is never a valid + * product dependency. + */ +export function assertVllmRegistryDigestRef(image: string): void { + const separator = image.lastIndexOf("@"); + const repository = separator > 0 ? image.slice(0, separator) : ""; + const digest = separator > 0 ? image.slice(separator + 1) : ""; + const components = repository.split("/"); + const firstComponent = components[0] ?? ""; + const portSeparator = firstComponent.lastIndexOf(":"); + const registryOrNamespace = + portSeparator > 0 && /^\d+$/.test(firstComponent.slice(portSeparator + 1)) + ? firstComponent.slice(0, portSeparator) + : firstComponent; + const hasInvalidPort = firstComponent.includes(":") && registryOrNamespace === firstComponent; + const validRepository = + separator === image.indexOf("@") && + components.length >= 2 && + !hasInvalidPort && + IMAGE_REPOSITORY_COMPONENT_PATTERN.test(registryOrNamespace) && + components.slice(1).every((component) => IMAGE_REPOSITORY_COMPONENT_PATTERN.test(component)); + + if (!validRepository || !SHA256_IMAGE_DIGEST_PATTERN.test(digest)) { + throw new Error( + "vLLM image must be a pullable immutable registry reference in " + + `repository@sha256:<64 lowercase hex> form; got '${image}'. ` + + "Local image IDs and mutable tags are not supported.", + ); + } } type VllmContainerOwnership = @@ -609,11 +633,10 @@ export function isNemoClawManagedVllmRunning(): boolean { function startContainer( profile: VllmProfile, model: VllmModelDef, - qualificationMonitor: VllmQualificationMonitor | null, ): { ok: boolean; reason?: string } { emit(`Starting vLLM container (${profile.containerName})`); const resolvedFlags = profile.buildDockerRunFlags - ? profile.buildDockerRunFlags(qualificationMonitor?.gpuIndex) + ? profile.buildDockerRunFlags() : profile.dockerRunFlags; // The explicit download completed before this long-lived container starts, // so do not retain the host Hugging Face token in the serving process. @@ -649,154 +672,30 @@ function vllmModelsEndpoint(): string { return `http://127.0.0.1:${String(VLLM_PORT)}/v1/models`; } -function vllmHealthEndpoint(): string { - return `http://127.0.0.1:${String(VLLM_PORT)}/health`; -} - -function queryVllmEndpoint(url: string): string { +function vllmEndpointReady(): boolean { const response = runCapture( [ "curl", - ...buildValidatedCurlCommandArgs(["-sf", "--connect-timeout", "2", "--max-time", "5", url]), + ...buildValidatedCurlCommandArgs([ + "-sf", + "--connect-timeout", + "2", + "--max-time", + "5", + vllmModelsEndpoint(), + ]), ], { ignoreError: true }, ).trim(); - return response; -} - -function vllmHealthReady(): boolean { - const validated = buildValidatedCurlCommandArgs([ - "-sf", - "--connect-timeout", - "2", - "--max-time", - "5", - vllmHealthEndpoint(), - ]); - const url = validated.at(-1); - if (!url) return false; - const response = runCapture( - ["curl", ...buildCurlProbeSpawnArgs(validated.slice(0, -1), url, "/dev/null", "json")], - { ignoreError: true }, - ).trim(); - return response === "200"; -} - -function vllmEndpointReady(model: VllmModelDef): boolean { - const readiness = model.readiness; - // vLLM's /health endpoint commonly returns an empty 200 response, so check - // its status code instead of treating a non-empty body as the success signal. - if (readiness?.healthEndpoint && !vllmHealthReady()) return false; - const response = queryVllmEndpoint(vllmModelsEndpoint()); if (!response) return false; try { - const parsed = JSON.parse(response) as { - data?: Array<{ id?: unknown; max_model_len?: unknown }>; - }; - if (!Array.isArray(parsed.data)) return false; - if (!readiness) return true; - return ( - parsed.data.length === 1 && - parsed.data[0]?.id === readiness.servedModelId && - parsed.data[0]?.max_model_len === readiness.maxModelLen - ); + const parsed = JSON.parse(response) as { data?: unknown }; + return Array.isArray(parsed.data); } catch { return false; } } -interface VllmQualificationMonitor { - gpuIndex: number; - peakHbmMiB: number; -} - -type VllmQualificationResult = - | { ok: true; monitor: VllmQualificationMonitor } - | { ok: false; reason: string }; - -function literalCaseInsensitivePattern(value: string): RegExp { - return new RegExp(value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i"); -} - -function sampleVllmQualification( - profile: VllmProfile, - monitor: VllmQualificationMonitor, -): VllmQualificationResult { - const qualification = profile.qualification; - if (!qualification) return { ok: true, monitor }; - const raw = runCapture( - [ - "nvidia-smi", - `--id=${String(monitor.gpuIndex)}`, - "--query-gpu=memory.used,ecc.errors.uncorrected.aggregate.total,ecc.errors.uncorrected.volatile.total", - "--format=csv,noheader,nounits", - ], - { ignoreError: true }, - ).trim(); - const rows = raw.split(/\r?\n/).filter((row) => row.trim().length > 0); - const values = rows.length === 1 ? rows[0].split(",").map((value) => value.trim()) : []; - if (values.length !== 3 || values.some((value) => !/^\d+$/.test(value))) { - return { - ok: false, - reason: `qualified GPU telemetry is unavailable or malformed for GPU ${String(monitor.gpuIndex)}`, - }; - } - - const [hbmMiB, aggregateEcc, volatileEcc] = values.map(Number); - monitor.peakHbmMiB = Math.max(monitor.peakHbmMiB, hbmMiB); - if (hbmMiB >= qualification.hbmSafetyCeilingMiB) { - return { - ok: false, - reason: - `qualified HBM safety ceiling exceeded on GPU ${String(monitor.gpuIndex)}: ` + - `${String(hbmMiB)} MiB (required <${String(qualification.hbmSafetyCeilingMiB)} MiB)`, - }; - } - if (qualification.requireZeroUncorrectedEcc && (aggregateEcc !== 0 || volatileEcc !== 0)) { - return { - ok: false, - reason: - `uncorrected ECC is nonzero on GPU ${String(monitor.gpuIndex)}: ` + - `aggregate=${String(aggregateEcc)}, volatile=${String(volatileEcc)}`, - }; - } - return { ok: true, monitor }; -} - -function initializeVllmQualification(profile: VllmProfile): VllmQualificationResult | null { - const qualification = profile.qualification; - if (!qualification) return null; - const indices = getGpuIndicesByName(literalCaseInsensitivePattern(qualification.gpuNameIncludes)); - if (indices.length !== qualification.gpuCount) { - return { - ok: false, - reason: - `qualified runtime requires exactly ${String(qualification.gpuCount)} GPU matching ` + - `'${qualification.gpuNameIncludes}'; detected ${String(indices.length)}`, - }; - } - return sampleVllmQualification(profile, { gpuIndex: indices[0], peakHbmMiB: 0 }); -} - -function revalidateVllmQualification( - profile: VllmProfile, - monitor: VllmQualificationMonitor, -): VllmQualificationResult { - const qualification = profile.qualification; - if (!qualification) return { ok: true, monitor }; - const indices = getGpuIndicesByName(literalCaseInsensitivePattern(qualification.gpuNameIncludes)); - if (indices.length !== qualification.gpuCount || indices[0] !== monitor.gpuIndex) { - return { - ok: false, - reason: - `qualified GPU selection changed before launch: expected GPU ${String(monitor.gpuIndex)} ` + - `matching '${qualification.gpuNameIncludes}', detected matching indices ` + - `[${indices.map(String).join(", ")}]`, - }; - } - return sampleVllmQualification(profile, monitor); -} - function readContainerLogTail(profile: VllmProfile, lineCount = 80): string[] { const output = dockerCapture(["logs", "--tail", String(lineCount), profile.containerName], { env: buildVllmDockerEnv(), @@ -817,11 +716,7 @@ function printContainerLogTail(profile: VllmProfile): void { // Poll the real OpenAI-compatible models endpoint instead of interpreting // vLLM startup logs. Logs stay quiet on the happy path and print only on // failure. -function waitForVllmReady( - profile: VllmProfile, - model: VllmModelDef, - qualificationMonitor: VllmQualificationMonitor | null, -): Promise<{ ok: boolean; reason?: string }> { +function waitForVllmReady(profile: VllmProfile): Promise<{ ok: boolean; reason?: string }> { return new Promise((resolve) => { let resolved = false; const start = Date.now(); @@ -841,20 +736,7 @@ function waitForVllmReady( function poll(): void { if (resolved) return; - if (qualificationMonitor) { - const qualification = sampleVllmQualification(profile, qualificationMonitor); - if (!qualification.ok) { - done({ ok: false, reason: qualification.reason }); - return; - } - } - if (vllmEndpointReady(model)) { - if (qualificationMonitor && profile.qualification) { - emit( - `Qualified startup telemetry: peak HBM ${String(qualificationMonitor.peakHbmMiB)} MiB ` + - `(<${String(profile.qualification.hbmSafetyCeilingMiB)} MiB), uncorrected ECC 0`, - ); - } + if (vllmEndpointReady()) { emit(`vLLM is serving on :${String(VLLM_PORT)}`); done({ ok: true }); return; @@ -1054,33 +936,14 @@ interface InstallVllmOptions { beforeInstall?: (modelId: string) => void; } -function inspectImageId(profile: VllmProfile): string { - return dockerImageInspectFormat("{{.Id}}", profile.image, { - env: buildVllmDockerEnv(), - ignoreError: true, - timeout: 10_000, - }).trim(); -} - function imageIsCached(profile: VllmProfile): boolean { - return Boolean(inspectImageId(profile)); -} - -function verifyExpectedImageId( - profile: VllmProfile, - observedImageId: string, -): { ok: true } | { ok: false; reason: string } { - if (!profile.expectedImageId || observedImageId === profile.expectedImageId) { - return { ok: true }; - } - return { - ok: false, - reason: - "qualified image identity mismatch: expected " + - profile.expectedImageId + - ", resolved " + - (observedImageId || "no local image"), - }; + return Boolean( + dockerImageInspectFormat("{{.Id}}", profile.image, { + env: buildVllmDockerEnv(), + ignoreError: true, + timeout: 10_000, + }).trim(), + ); } export function resolveVllmServedModelId(modelId: string, extraServeArgs: string[]): string { @@ -1119,22 +982,18 @@ export async function installVllm( promptFn: opts.promptFn, }); if (!resolved) return { ok: false }; - const { model: selectedModel, source: modelSource } = resolved; - let model: VllmModelDef; - let activeModelProfile: ReturnType["profile"]; + const { model, source: modelSource } = resolved; + if (model.runtime && !model.platforms.includes(profile.platform)) { + console.error(` vLLM install failed: ${model.label} is not supported on ${profile.name}`); + return { ok: false }; + } + let runtimeProfile: VllmProfile; try { - const effective = resolveVllmModelProfile(selectedModel, profile.platform); - model = effective.model; - activeModelProfile = effective.profile; + runtimeProfile = resolveVllmRuntimeProfile(profile, model); } catch (err) { console.error(` vLLM install failed: ${(err as Error).message}`); return { ok: false }; } - if (model.runtime && !model.platforms.includes(profile.platform)) { - console.error(` vLLM install failed: ${model.label} is not supported on ${profile.name}`); - return { ok: false }; - } - const runtimeProfile = resolveVllmRuntimeProfile(profile, model); let extraServeArgs: string[]; let servedModelId: string; @@ -1153,19 +1012,12 @@ export async function installVllm( console.log( ` Model: ${model.id}${modelSource === "env" ? " (NEMOCLAW_VLLM_MODEL override)" : ""}`, ); - if (activeModelProfile) { - console.log(` Profile: ${activeModelProfile.id} (experimental, opt-in)`); - } if (extraServeArgs.length > 0) { console.log( ` Extra serve args: ${String(extraServeArgs.length)} token(s) from ${VLLM_EXTRA_ARGS_ENV}`, ); } - if (runtimeProfile.imagePullPolicy === "local-only") { - console.log(" Qualified local image required; no pull or fallback is available"); - } else if (!opts.hasImage) { - console.log(" Image download on first run, cached after"); - } + if (!opts.hasImage) console.log(" Image download on first run, cached after"); console.log(" Model download on first run, cached after"); console.log(""); @@ -1191,35 +1043,6 @@ export async function installVllm( return { ok: false }; } - let localOnlyImageId: string | null = null; - if (runtimeProfile.imagePullPolicy === "local-only") { - localOnlyImageId = inspectImageId(runtimeProfile); - if (!localOnlyImageId) { - console.error( - " vLLM install blocked: the qualified experimental runtime image is not published by the configured image source.", - ); - console.error( - ` Required local image ID: ${runtimeProfile.expectedImageId ?? runtimeProfile.image}`, - ); - console.error( - " Publish a repository-qualified manifest for these exact bytes and prove its pulled .Id before enabling a fresh install. No fallback image was substituted.", - ); - return { ok: false }; - } - const identity = verifyExpectedImageId(runtimeProfile, localOnlyImageId); - if (!identity.ok) { - console.error(` vLLM install failed: ${identity.reason}`); - return { ok: false }; - } - } - - const qualification = initializeVllmQualification(runtimeProfile); - if (qualification && !qualification.ok) { - console.error(` vLLM install failed: ${qualification.reason}`); - return { ok: false }; - } - const qualificationMonitor = qualification?.monitor ?? null; - // Guard the host filesystem before an image pull or model-download // container can start. The cache path itself is created only after both // storage decisions pass, so Docker never creates it as root. @@ -1227,7 +1050,7 @@ export async function installVllm( return { ok: false }; } - const hasImage = localOnlyImageId !== null || imageIsCached(runtimeProfile); + const hasImage = imageIsCached(runtimeProfile); if (!hasImage && !(await imageStorageAccepted(runtimeProfile, opts))) { return { ok: false }; } @@ -1238,19 +1061,10 @@ export async function installVllm( return { ok: false }; } - if (runtimeProfile.imagePullPolicy !== "local-only") { - const pull = await pullImage(runtimeProfile); - if (!pull.ok) { - console.error(` vLLM install failed: ${String(pull.reason)}`); - return { ok: false }; - } - if (runtimeProfile.expectedImageId) { - const identity = verifyExpectedImageId(runtimeProfile, inspectImageId(runtimeProfile)); - if (!identity.ok) { - console.error(` vLLM install failed: ${identity.reason}`); - return { ok: false }; - } - } + const pull = await pullImage(runtimeProfile); + if (!pull.ok) { + console.error(` vLLM install failed: ${String(pull.reason)}`); + return { ok: false }; } // A cold image pull can consume the same host filesystem that backs the @@ -1266,18 +1080,7 @@ export async function installVllm( return { ok: false }; } - if (qualificationMonitor) { - const preLaunchQualification = revalidateVllmQualification( - runtimeProfile, - qualificationMonitor, - ); - if (!preLaunchQualification.ok) { - console.error(` vLLM install failed: ${preLaunchQualification.reason}`); - return { ok: false }; - } - } - - const start = startContainer(runtimeProfile, model, qualificationMonitor); + const start = startContainer(runtimeProfile, model); if (!start.ok) { console.error(` vLLM install failed: ${String(start.reason)}`); return { ok: false }; @@ -1288,7 +1091,7 @@ export async function installVllm( `Launch can take 5 minutes to ${String(Math.ceil(runtimeProfile.loadTimeoutSec / 60))} minutes`, ); - const ready = await waitForVllmReady(runtimeProfile, model, qualificationMonitor); + const ready = await waitForVllmReady(runtimeProfile); if (!ready.ok) { printContainerLogTail(runtimeProfile); dockerStop(runtimeProfile.containerName, { diff --git a/src/lib/onboard/setup-nim-flow.test.ts b/src/lib/onboard/setup-nim-flow.test.ts index c6e850baf14..47db126cf83 100644 --- a/src/lib/onboard/setup-nim-flow.test.ts +++ b/src/lib/onboard/setup-nim-flow.test.ts @@ -904,52 +904,6 @@ describe("createSetupNim", () => { expect(handleVllmSelection).not.toHaveBeenCalled(); }); - it("requires the exact experimental managed runtime instead of suggesting existing local vLLM", async () => { - vi.stubEnv("NEMOCLAW_VLLM_PROFILE", "experimental-single-user"); - const profile = { - name: "DGX Station experimental single-user", - } as VllmProfile; - const error = vi.fn(); - const abortNonInteractive = vi.fn((message) => { - throw new Error(message); - }); - const installVllm = vi.fn(); - const handleVllmSelection = vi.fn(); - const setupNim = createSetupNim( - makeDeps({ - isNonInteractive: () => true, - getNonInteractiveProvider: () => "install-vllm", - error, - abortNonInteractive, - detectInferenceProviderHostState: () => - makeHostState({ - vllmRunning: true, - vllmProfile: profile, - hasVllmImage: true, - vllmEntries: [ - { - key: "install-vllm", - label: "Start vLLM (DGX Station experimental single-user)", - }, - ], - }), - installVllm, - handleVllmSelection, - }), - ); - - await expect(setupNim(null)).rejects.toThrow( - "The experimental single-user profile requires the qualified managed runtime", - ); - - const renderedError = String(error.mock.calls[0]?.[0]); - expect(renderedError).toContain("stop the existing server"); - expect(renderedError).not.toContain("Select Local vLLM"); - expect(abortNonInteractive).toHaveBeenCalledOnce(); - expect(installVllm).not.toHaveBeenCalled(); - expect(handleVllmSelection).not.toHaveBeenCalled(); - }); - it("returns interactive occupied-port selection to the provider menu", async () => { vi.stubEnv("NEMOCLAW_PROVIDER", ""); const profile = { name: "DGX Spark" } as VllmProfile; diff --git a/src/lib/onboard/setup-nim-flow.ts b/src/lib/onboard/setup-nim-flow.ts index 8a80dd1dec4..6495aec12e3 100644 --- a/src/lib/onboard/setup-nim-flow.ts +++ b/src/lib/onboard/setup-nim-flow.ts @@ -27,7 +27,6 @@ import { promptForInferenceProviderSelection } from "./provider-selection-prompt import type { RebuildRouteHandoff, RegistryInferenceRoute } from "./rebuild-route-handoff"; import { prepareProviderDiscovery } from "./setup-nim-provider-discovery"; import type { SetupNimSelectionState as BaseSetupNimSelectionState } from "./setup-nim-selection"; -import { requiresExactManagedVllm } from "./vllm-menu"; export type SetupNimGpu = ReturnType; export type SetupNimSelectionState = BaseSetupNimSelectionState; @@ -566,11 +565,9 @@ export function createSetupNim( continue selectionLoop; } if (vllmRunning) { - const message = requiresExactManagedVllm() - ? `vLLM is already running on localhost:${String(deps.vllmPort)}. ` + - "The experimental single-user profile requires the qualified managed runtime; stop the existing server before selecting the managed install path." - : `vLLM is already running on localhost:${String(deps.vllmPort)}. ` + - "Select Local vLLM, or stop the existing server before selecting the managed install path."; + const message = + `vLLM is already running on localhost:${String(deps.vllmPort)}. ` + + "Select Local vLLM, or stop the existing server before selecting the managed install path."; deps.error(` ${message}`); if (deps.isNonInteractive()) { deps.abortNonInteractive(message); diff --git a/src/lib/onboard/setup-nim-vllm.test.ts b/src/lib/onboard/setup-nim-vllm.test.ts index 84b84cf856e..9f45d705e73 100644 --- a/src/lib/onboard/setup-nim-vllm.test.ts +++ b/src/lib/onboard/setup-nim-vllm.test.ts @@ -3,7 +3,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { requireValue } from "../core/require-value"; -import { applyVllmRuntimeContextWindow } from "../inference/vllm-runtime-context"; import type { SetupNimSelectionState } from "./setup-nim-flow"; import { buildDgxSparkExistingVllmHeadroomWarning, @@ -99,39 +98,12 @@ describe("setupNim vLLM route containment", () => { " To keep 'served/model' instead, start detailed setup:", ); expect(console.error).toHaveBeenCalledWith( - " unset NEMOCLAW_PROVIDER NEMOCLAW_MODEL NEMOCLAW_VLLM_MODEL NEMOCLAW_VLLM_PROFILE", + " unset NEMOCLAW_PROVIDER NEMOCLAW_MODEL NEMOCLAW_VLLM_MODEL", ); expect(console.error).toHaveBeenCalledWith(" nemoclaw onboard --fresh"); expect(console.error).toHaveBeenCalledWith(" Then select Local vLLM when prompted."); }); - it("carries the experimental Ultra route and 262144 context through the supported lifecycle", async () => { - const selection = state("nemotron-ultra"); - const runtimeEnv: NodeJS.ProcessEnv = {}; - const models = { - data: [{ id: "nemotron-ultra", max_model_len: 262144 }], - }; - const handler = createSetupNimVllmHandler( - deps({ - runCapture: () => JSON.stringify(models), - applyVllmRuntimeContextWindow: (response, model) => - applyVllmRuntimeContextWindow(response, model, { - env: runtimeEnv, - logger: { log: vi.fn(), warn: vi.fn() }, - }), - }), - ); - - await expect(handler(selection, { managedInstall: true })).resolves.toBe("selected"); - expect(selection).toMatchObject({ - provider: "vllm-local", - model: "nemotron-ultra", - endpointUrl: "http://host.openshell.internal:8000/v1", - preferredInferenceApi: "openai-completions", - }); - expect(runtimeEnv.NEMOCLAW_CONTEXT_WINDOW).toBe("262144"); - }); - it("warns on DGX Spark identified via GPU name (firmware-unknown GB10 host)", async () => { const selection = state(null); const handler = createSetupNimVllmHandler( diff --git a/src/lib/onboard/setup-nim-vllm.ts b/src/lib/onboard/setup-nim-vllm.ts index 00aebeb618b..ead86ae7133 100644 --- a/src/lib/onboard/setup-nim-vllm.ts +++ b/src/lib/onboard/setup-nim-vllm.ts @@ -199,9 +199,7 @@ export function createSetupNimVllmHandler( ` To install '${requiredModel}', stop the existing vLLM server on localhost:${deps.VLLM_PORT}, then rerun the original install/onboard command.`, ); console.error(` To keep '${detectedModel}' instead, start detailed setup:`); - console.error( - " unset NEMOCLAW_PROVIDER NEMOCLAW_MODEL NEMOCLAW_VLLM_MODEL NEMOCLAW_VLLM_PROFILE", - ); + console.error(" unset NEMOCLAW_PROVIDER NEMOCLAW_MODEL NEMOCLAW_VLLM_MODEL"); console.error(` ${cliName()} onboard --fresh`); console.error(" Then select Local vLLM when prompted."); deps.exitProcess(1); diff --git a/src/lib/onboard/vllm-menu.test.ts b/src/lib/onboard/vllm-menu.test.ts index 5d35051e8cc..3564eccce5a 100644 --- a/src/lib/onboard/vllm-menu.test.ts +++ b/src/lib/onboard/vllm-menu.test.ts @@ -169,33 +169,6 @@ describe("buildVllmMenuEntries", () => { assert.match(logs[0], /selecting the running instance/); }); - it("keeps the exact experimental managed-install entry when vLLM is already running", () => { - const logs: string[] = []; - const entries = buildVllmMenuEntries({ - vllmRunning: true, - vllmProfile: { name: "DGX Station experimental single-user" }, - experimental: false, - platform: "station", - hasVllmImage: true, - env: { - NEMOCLAW_PROVIDER: "install-vllm", - NEMOCLAW_VLLM_PROFILE: "EXPERIMENTAL-SINGLE-USER", - }, - log: (message) => logs.push(message), - }); - - assert.deepEqual(entries, [ - { - key: "install-vllm", - label: "Start vLLM (DGX Station experimental single-user)", - }, - ]); - assert.equal(logs.length, 1); - assert.match(logs[0], /requires the qualified managed runtime/); - assert.match(logs[0], /existing vLLM instance on localhost:8000 cannot be reused/); - assert.doesNotMatch(logs[0], /selecting the running instance/); - }); - it("does not log the override note when the user did not request install-vllm", () => { const logs: string[] = []; buildVllmMenuEntries({ diff --git a/src/lib/onboard/vllm-menu.ts b/src/lib/onboard/vllm-menu.ts index 6476684c4a8..58709a87bee 100644 --- a/src/lib/onboard/vllm-menu.ts +++ b/src/lib/onboard/vllm-menu.ts @@ -21,13 +21,10 @@ * available for this host." message. It also lets the caller surface managed * vLLM by default for known DGX platforms while generic Linux stays gated, and * logs a note when running-vLLM takes precedence over the env-var opt-in. - * Exact qualified profiles are the exception: they retain the install entry so - * the FSM can reject an occupied port instead of reusing an unqualified server. */ import { VLLM_PORT } from "../core/ports"; import type { NvidiaPlatform } from "../inference/nim"; -import { EXPERIMENTAL_SINGLE_USER_PROFILE, VLLM_PROFILE_ENV } from "../inference/vllm-models"; interface VllmProfileShape { name: string; @@ -51,20 +48,6 @@ export interface BuildVllmMenuOptions { log?: (message: string) => void; } -/** - * Exact managed-runtime profiles cannot reuse an arbitrary listener on the - * local vLLM port. Their image, arguments, environment, and safety checks are - * part of the qualified contract, so onboarding must keep the managed-install - * transition and let it fail closed until the existing server is stopped. - */ -export function requiresExactManagedVllm(env: NodeJS.ProcessEnv = process.env): boolean { - return ( - String(env[VLLM_PROFILE_ENV] ?? "") - .trim() - .toLowerCase() === EXPERIMENTAL_SINGLE_USER_PROFILE - ); -} - export function buildVllmMenuEntries(opts: BuildVllmMenuOptions): VllmMenuEntry[] { const log = opts.log ?? console.log; // Read NEMOCLAW_PROVIDER directly so interactive runs with an explicit @@ -73,8 +56,7 @@ export function buildVllmMenuEntries(opts: BuildVllmMenuOptions): VllmMenuEntry[ const env = opts.env ?? process.env; const userChoseManagedVllm = (env.NEMOCLAW_PROVIDER || "").trim().toLowerCase() === "install-vllm"; - const exactManagedVllmRequired = requiresExactManagedVllm(env); - if (opts.vllmRunning && !exactManagedVllmRequired) { + if (opts.vllmRunning) { if (userChoseManagedVllm) { log( ` Note: NEMOCLAW_PROVIDER=install-vllm requested, but vLLM is already running on localhost:${VLLM_PORT} — selecting the running instance.`, @@ -89,13 +71,7 @@ export function buildVllmMenuEntries(opts: BuildVllmMenuOptions): VllmMenuEntry[ }, ]; } - if (opts.vllmRunning && exactManagedVllmRequired) { - log( - ` Note: ${VLLM_PROFILE_ENV}=${EXPERIMENTAL_SINGLE_USER_PROFILE} requires the qualified managed runtime; the existing vLLM instance on localhost:${VLLM_PORT} cannot be reused.`, - ); - } if ( - exactManagedVllmRequired || userChoseManagedVllm || (opts.vllmProfile && (opts.experimental || (opts.platform && MANAGED_VLLM_DEFAULT_PLATFORMS.has(opts.platform)))) diff --git a/src/lib/onboard/vllm-model-preflight.ts b/src/lib/onboard/vllm-model-preflight.ts index a9c87a8c78c..6abebaade96 100644 --- a/src/lib/onboard/vllm-model-preflight.ts +++ b/src/lib/onboard/vllm-model-preflight.ts @@ -4,8 +4,8 @@ import { preflightVllmModelEnv } from "../inference/vllm-models"; /** - * Validate managed-vLLM model/profile input up front, before onboarding runs - * preflight or touches Docker, mirroring the `connect` preflight added in #4567. + * Validate `NEMOCLAW_VLLM_MODEL` up front, before onboarding runs preflight or + * touches Docker, mirroring the `connect` preflight added in #4567. * * The variable steers the express-vLLM installer, but it is only consumed deep * in the `[3/8]` provider step. That made it validated late and path-dependent: diff --git a/test/cli/dispatch-basics.test.ts b/test/cli/dispatch-basics.test.ts index 31f543620ca..4959e57ab00 100644 --- a/test/cli/dispatch-basics.test.ts +++ b/test/cli/dispatch-basics.test.ts @@ -486,9 +486,7 @@ describe("CLI dispatch", () => { expect(out).toMatch(/gated on Hugging Face/); expect(out).toMatch(/HF_TOKEN/); expect(out).toMatch(/HUGGING_FACE_HUB_TOKEN/); - expect(out).toContain( - "NEMOCLAW_VLLM_MODEL, NEMOCLAW_VLLM_PROFILE, and NEMOCLAW_VLLM_EXTRA_ARGS_JSON are consumed by the managed-vLLM install path", - ); + expect(out).toContain("NEMOCLAW_VLLM_MODEL is consumed by the managed-vLLM install path"); const calls = fs.existsSync(openshellLog) ? fs.readFileSync(openshellLog, "utf8") : ""; expect(calls).not.toMatch(/\bsandbox\s+(get|connect|list)\b/); } finally { diff --git a/test/generate-hermes-config.test.ts b/test/generate-hermes-config.test.ts index 8b09e8b182c..7c0d30da46d 100644 --- a/test/generate-hermes-config.test.ts +++ b/test/generate-hermes-config.test.ts @@ -529,31 +529,6 @@ describe("agents/hermes/generate-config.ts", () => { expect(config.model.context_window).toBeUndefined(); }); - it("routes the experimental Station Ultra profile through vllm-local at 262144 context", () => { - const { config } = runConfigScript({ - NEMOCLAW_PROVIDER_KEY: "vllm-local", - NEMOCLAW_MODEL: "nemotron-ultra", - NEMOCLAW_CONTEXT_WINDOW: "262144", - }); - - expect(config._nemoclaw_upstream).toEqual({ - provider: "vllm-local", - model: "nemotron-ultra", - }); - expect(config.model).toMatchObject({ - default: "nemotron-ultra", - provider: "custom", - base_url: "https://inference.local/v1", - context_length: 262144, - }); - expect(config.model.context_window).toBeUndefined(); - expect(config.providers["vllm-local"]).toMatchObject({ - name: "vllm-local", - default_model: "nemotron-ultra", - discover_models: true, - }); - }); - it("accepts Hermes' minimum context window as model.context_length (#6760)", () => { const settings = readHermesBuildSettings( buildHermesTestEnv({ NEMOCLAW_CONTEXT_WINDOW: String(MIN_HERMES_CONTEXT_WINDOW) }), diff --git a/test/inference-options-docs.test.ts b/test/inference-options-docs.test.ts index 498a1abf509..ba375fb9e34 100644 --- a/test/inference-options-docs.test.ts +++ b/test/inference-options-docs.test.ts @@ -283,7 +283,7 @@ describe("inference setup navigation", () => { (match) => match[1], ); - expect(hostValues).toEqual(["0.0.0.0", "0.0.0.0"]); + expect(hostValues).toEqual(["0.0.0.0"]); expect(markdown).toContain("default-deny inbound rules"); expect(markdown).toContain("only from the OpenShell Docker subnet to its gateway address"); }); @@ -297,7 +297,7 @@ describe("inference setup navigation", () => { tag: VLLM_IMAGES.ngc2605Post1.tag, }, { - prefix: "- The canonical DGX Station Nemotron 3 Ultra express recipe", + prefix: "- The DGX Station Nemotron 3 Ultra express recipe", image: VLLM_IMAGES.vllm022.arm64, tag: VLLM_IMAGES.vllm022.tag, }, diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index 42d65d53a1b..2deba97beb3 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -39,9 +39,9 @@ NON_INTERACTIVE="\${NON_INTERACTIVE:-}" NEMOCLAW_PROVIDER="\${NEMOCLAW_PROVIDER:-}" NEMOCLAW_NO_EXPRESS="\${NEMOCLAW_NO_EXPRESS:-}" maybe_offer_express_install -printf "RESULT NON_INTERACTIVE=%s SUDO_MODE=%s PROVIDER=%s MODEL=%s VLLM_MODEL=%s VLLM_PROFILE=%s POLICY=%s YES=%s SANDBOX=%s\\n" \\ +printf "RESULT NON_INTERACTIVE=%s SUDO_MODE=%s PROVIDER=%s MODEL=%s VLLM_MODEL=%s POLICY=%s YES=%s SANDBOX=%s\\n" \\ "\${NON_INTERACTIVE:-}" "\${NEMOCLAW_NON_INTERACTIVE_SUDO_MODE:-}" "\${NEMOCLAW_PROVIDER:-}" "\${NEMOCLAW_MODEL:-}" \\ - "\${NEMOCLAW_VLLM_MODEL:-}" "\${NEMOCLAW_VLLM_PROFILE:-}" "\${NEMOCLAW_POLICY_MODE:-}" "\${NEMOCLAW_YES:-}" "\${NEMOCLAW_SANDBOX_NAME:-}" + "\${NEMOCLAW_VLLM_MODEL:-}" "\${NEMOCLAW_POLICY_MODE:-}" "\${NEMOCLAW_YES:-}" "\${NEMOCLAW_SANDBOX_NAME:-}" ''' env = dict(os.environ) env["INSTALLER_UNDER_TEST"] = installer @@ -158,23 +158,6 @@ detect_express_platform ); }); - it("parses and documents the experimental single-user Station profile", () => { - const result = spawnSync( - "bash", - [INSTALLER_PAYLOAD, "--station-experimental-single-user", "--help"], - { - cwd: path.join(import.meta.dirname, ".."), - encoding: "utf-8", - }, - ); - const output = `${result.stdout}${result.stderr}`; - - expect(result.status, output).toBe(0); - expect(output).toMatch( - /--station-experimental-single-user\s+Use the experimental single-user DGX Station profile/, - ); - }); - it("offers express install when curl-piped stdin still has a controlling TTY", () => { const result = runExpressPromptWithTty("y\n", "pipe"); const output = `${result.stdout}${result.stderr}`; @@ -191,7 +174,7 @@ detect_express_platform expect(output).toMatch(/Run express install/); expect(output).toMatch(/Using express install for DGX Spark/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL= VLLM_PROFILE= POLICY=suggested YES=1 SANDBOX=my-assistant/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL= POLICY=suggested YES=1 SANDBOX=my-assistant/, ); }); @@ -209,7 +192,7 @@ detect_express_platform /Managed vLLM pulls the configured vLLM image\/model and runs a local vLLM inference container/, ); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL=custom-qwen3\.6 VLLM_PROFILE= POLICY=suggested YES=1 SANDBOX=my-assistant/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL=custom-qwen3\.6 POLICY=suggested YES=1 SANDBOX=my-assistant/, ); }); @@ -222,7 +205,7 @@ detect_express_platform expect(output).toMatch(/Detected DGX Spark/); expect(output).toMatch(/Sandbox name: custom-spark/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL= VLLM_PROFILE= POLICY=suggested YES=1 SANDBOX=custom-spark/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL= POLICY=suggested YES=1 SANDBOX=custom-spark/, ); }); @@ -237,7 +220,7 @@ detect_express_platform expect(output).toMatch(/approximately 352 GB model/); expect(output).toMatch(/Using express install for DGX Station/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b VLLM_PROFILE= POLICY=suggested YES=1 SANDBOX=my-assistant/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b POLICY=suggested YES=1 SANDBOX=my-assistant/, ); }); @@ -246,155 +229,12 @@ detect_express_platform NEMOCLAW_VLLM_MODEL: "nvidia/nemotron-3-ultra-550b-a55b", }); const output = `${result.stdout}${result.stderr}`; - - expect(result.status, output).toBe(0); - expect(output).toMatch( - /MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b VLLM_PROFILE=/, - ); - }); - - it("uses the opt-in experimental single-user Station profile with one confirmation", () => { - const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { - STATION_EXPERIMENTAL_SINGLE_USER: "1", - }); - const output = `${result.stdout}${result.stderr}`; expect(result.status, output).toBe(0); expect(output).toMatch( - /managed local vLLM with the experimental single-user Nemotron Ultra profile/, - ); - expect(output).toMatch(/This opt-in profile uses the qualified single-user Station runtime/); - expect(output.match(/Run express install with these settings\?/g)).toHaveLength(1); - expect(output).toMatch(/Using express install for DGX Station/); - expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=nemotron-ultra VLLM_MODEL=nemotron-3-ultra-550b-a55b VLLM_PROFILE=experimental-single-user POLICY=suggested YES=1 SANDBOX=my-assistant/, + /MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b POLICY=/, ); }); - it("accepts matching experimental Station model and profile selectors", () => { - const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { - STATION_EXPERIMENTAL_SINGLE_USER: "1", - NEMOCLAW_VLLM_MODEL: "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4", - NEMOCLAW_MODEL: "nemotron-ultra", - NEMOCLAW_VLLM_PROFILE: "experimental-single-user", - NEMOCLAW_VLLM_PORT: "8000", - NEMOCLAW_CONTEXT_WINDOW: "262144", - NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL: "http://host.openshell.internal/", - }); - const output = `${result.stdout}${result.stderr}`; - expect(result.status, output).toBe(0); - expect(output).toMatch( - /MODEL=nemotron-ultra VLLM_MODEL=nemotron-3-ultra-550b-a55b VLLM_PROFILE=experimental-single-user/, - ); - }); - - it("rejects an inherited experimental profile from ordinary Station express", () => { - const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { - NEMOCLAW_VLLM_PROFILE: "experimental-single-user", - }); - const output = `${result.stdout}${result.stderr}`; - - expect(result.status, output).not.toBe(0); - expect(output).toMatch( - /NEMOCLAW_VLLM_PROFILE='experimental-single-user' cannot select a DGX Station express profile by itself/, - ); - expect(output).toMatch(/Use --station-experimental-single-user/); - expect(output).not.toMatch(/Run express install/); - }); - - it("preserves the experimental selector for advanced direct onboarding", () => { - const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { - NEMOCLAW_PROVIDER: "install-vllm", - NEMOCLAW_VLLM_MODEL: "nemotron-3-ultra-550b-a55b", - NEMOCLAW_VLLM_PROFILE: "experimental-single-user", - }); - const output = `${result.stdout}${result.stderr}`; - - expect(result.status, output).toBe(0); - expect(output).toMatch( - /Skipping express prompt \(NEMOCLAW_PROVIDER=install-vllm already set\)/, - ); - expect(output).toMatch( - /PROVIDER=install-vllm .*VLLM_MODEL=nemotron-3-ultra-550b-a55b VLLM_PROFILE=experimental-single-user/, - ); - }); - - it("rejects combining the experimental Station profile with the DeepSeek override", () => { - const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { - STATION_EXPERIMENTAL_SINGLE_USER: "1", - STATION_DEEPSEEK: "1", - }); - const output = `${result.stdout}${result.stderr}`; - expect(result.status, output).not.toBe(0); - expect(output).toMatch( - /--station-experimental-single-user cannot be combined with --station-deepseek/, - ); - expect(output).not.toMatch(/Run express install/); - }); - - it("rejects the experimental single-user profile on non-Station platforms", () => { - const result = runExpressPromptWithTty("\n", "pipe", "DGX Spark", { - STATION_EXPERIMENTAL_SINGLE_USER: "1", - }); - const output = `${result.stdout}${result.stderr}`; - expect(result.status, output).not.toBe(0); - expect(output).toMatch( - /--station-experimental-single-user requires a detected DGX Station \(detected: DGX Spark\)/, - ); - expect(output).not.toMatch(/Run express install/); - }); - - it.each([ - ["NEMOCLAW_NO_EXPRESS", "1", /cannot be combined with NEMOCLAW_NO_EXPRESS=1/], - ["NON_INTERACTIVE", "1", /cannot be combined with --non-interactive/], - ["NEMOCLAW_PROVIDER", "install-vllm", /conflicts with NEMOCLAW_PROVIDER=install-vllm/], - ])("rejects %s when the experimental Station profile would otherwise be ignored", (name, value, message) => { - const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { - STATION_EXPERIMENTAL_SINGLE_USER: "1", - [name]: value, - }); - const output = `${result.stdout}${result.stderr}`; - expect(result.status, output).not.toBe(0); - expect(output).toMatch(message); - expect(output).not.toMatch(/Run express install/); - }); - - it.each([ - [ - "NEMOCLAW_VLLM_MODEL", - "deepseek-v4-flash", - /conflicts with NEMOCLAW_VLLM_MODEL='deepseek-v4-flash'/, - ], - ["NEMOCLAW_MODEL", "other-served-model", /conflicts with NEMOCLAW_MODEL='other-served-model'/], - [ - "NEMOCLAW_VLLM_PROFILE", - "another-profile", - /conflicts with NEMOCLAW_VLLM_PROFILE='another-profile'/, - ], - [ - "NEMOCLAW_VLLM_EXTRA_ARGS_JSON", - '["--enable-prefix-caching"]', - /cannot be combined with NEMOCLAW_VLLM_EXTRA_ARGS_JSON/, - ], - ["NEMOCLAW_VLLM_PORT", "9000", /conflicts with NEMOCLAW_VLLM_PORT='9000'/], - ["NEMOCLAW_VLLM_PORT", "08000", /conflicts with NEMOCLAW_VLLM_PORT='08000'/], - ["NEMOCLAW_CONTEXT_WINDOW", "8192", /conflicts with NEMOCLAW_CONTEXT_WINDOW='8192'/], - ["NEMOCLAW_CONTEXT_WINDOW", "0262144", /conflicts with NEMOCLAW_CONTEXT_WINDOW='0262144'/], - [ - "NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL", - "http://127.0.0.1", - /conflicts with NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL='http:\/\/127\.0\.0\.1'/, - ], - ])("rejects conflicting %s for the qualified experimental Station profile", (name, value, message) => { - const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { - STATION_EXPERIMENTAL_SINGLE_USER: "1", - [name]: value, - }); - const output = `${result.stdout}${result.stderr}`; - expect(result.status, output).not.toBe(0); - expect(output).toMatch(message); - expect(output).not.toMatch(/Run express install/); - }); - it("uses DeepSeek V4 Flash for the Station demo override with one confirmation", () => { const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { STATION_DEEPSEEK: "1", @@ -407,7 +247,7 @@ detect_express_platform expect(output.match(/Run express install with these settings\?/g)).toHaveLength(1); expect(output).toMatch(/Using express install for DGX Station/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=deepseek-ai\/DeepSeek-V4-Flash VLLM_MODEL=deepseek-v4-flash VLLM_PROFILE= POLICY=suggested YES=1 SANDBOX=my-assistant/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=deepseek-ai\/DeepSeek-V4-Flash VLLM_MODEL=deepseek-v4-flash POLICY=suggested YES=1 SANDBOX=my-assistant/, ); }); @@ -435,19 +275,6 @@ detect_express_platform expect(output).not.toMatch(/Run express install/); }); - it("rejects an inherited vLLM profile with the Station demo override", () => { - const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { - STATION_DEEPSEEK: "1", - NEMOCLAW_VLLM_PROFILE: "experimental-single-user", - }); - const output = `${result.stdout}${result.stderr}`; - expect(result.status, output).not.toBe(0); - expect(output).toMatch( - /--station-deepseek conflicts with NEMOCLAW_VLLM_PROFILE='experimental-single-user'/, - ); - expect(output).not.toMatch(/Run express install/); - }); - it("rejects the Station demo override on non-Station platforms", () => { const result = runExpressPromptWithTty("\n", "pipe", "DGX Spark", { STATION_DEEPSEEK: "1", @@ -485,7 +312,7 @@ detect_express_platform expect(output).toMatch(/pulls the configured vLLM image\/model/); expect(output).not.toMatch(/approximately 352 GB model/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL=custom-station-model VLLM_PROFILE= POLICY=suggested YES=1 SANDBOX=my-assistant/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL= VLLM_MODEL=custom-station-model POLICY=suggested YES=1 SANDBOX=my-assistant/, ); }); @@ -498,7 +325,7 @@ detect_express_platform expect(output).toMatch(/managed local vLLM with NVIDIA Nemotron 3 Ultra 550B/); expect(output).toMatch(/approximately 352 GB model/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b VLLM_PROFILE= POLICY=suggested YES=1 SANDBOX=my-assistant/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-vllm MODEL=nvidia\/nemotron-3-ultra-550b-a55b VLLM_MODEL=nemotron-3-ultra-550b-a55b POLICY=suggested YES=1 SANDBOX=my-assistant/, ); }); @@ -556,7 +383,7 @@ detect_express_platform expect(output).toMatch(/Run express install/); expect(output).toMatch(/Using express install for Windows WSL/); expect(output).toMatch( - /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-windows-ollama MODEL= VLLM_MODEL= VLLM_PROFILE= POLICY=suggested YES=1 SANDBOX=/, + /RESULT NON_INTERACTIVE=1 SUDO_MODE=prompt PROVIDER=install-windows-ollama MODEL= VLLM_MODEL= POLICY=suggested YES=1 SANDBOX=/, ); }); @@ -576,9 +403,9 @@ NON_INTERACTIVE="" NEMOCLAW_PROVIDER="" NEMOCLAW_NO_EXPRESS="" maybe_offer_express_install -printf "RESULT NON_INTERACTIVE=%s SUDO_MODE=%s PROVIDER=%s MODEL=%s VLLM_MODEL=%s VLLM_PROFILE=%s POLICY=%s YES=%s SANDBOX=%s\\n" \\ +printf "RESULT NON_INTERACTIVE=%s SUDO_MODE=%s PROVIDER=%s MODEL=%s VLLM_MODEL=%s POLICY=%s YES=%s SANDBOX=%s\\n" \\ "\${NON_INTERACTIVE:-}" "\${NEMOCLAW_NON_INTERACTIVE_SUDO_MODE:-}" "\${NEMOCLAW_PROVIDER:-}" "\${NEMOCLAW_MODEL:-}" \\ - "\${NEMOCLAW_VLLM_MODEL:-}" "\${NEMOCLAW_VLLM_PROFILE:-}" "\${NEMOCLAW_POLICY_MODE:-}" "\${NEMOCLAW_YES:-}" "\${NEMOCLAW_SANDBOX_NAME:-}" + "\${NEMOCLAW_VLLM_MODEL:-}" "\${NEMOCLAW_POLICY_MODE:-}" "\${NEMOCLAW_YES:-}" "\${NEMOCLAW_SANDBOX_NAME:-}" `, ], { @@ -598,7 +425,7 @@ printf "RESULT NON_INTERACTIVE=%s SUDO_MODE=%s PROVIDER=%s MODEL=%s VLLM_MODEL=% expect(output).toMatch(/Skipping express prompt \(no TTY\)/); expect(output).not.toMatch(/Run express install/); expect(output).toMatch( - /RESULT NON_INTERACTIVE= SUDO_MODE= PROVIDER= MODEL= VLLM_MODEL= VLLM_PROFILE= POLICY= YES= SANDBOX=/, + /RESULT NON_INTERACTIVE= SUDO_MODE= PROVIDER= MODEL= VLLM_MODEL= POLICY= YES= SANDBOX=/, ); }, ); diff --git a/test/onboard-selection-vllm.test.ts b/test/onboard-selection-vllm.test.ts index e926aa53462..f6beebf0e6f 100644 --- a/test/onboard-selection-vllm.test.ts +++ b/test/onboard-selection-vllm.test.ts @@ -703,7 +703,7 @@ const { setupNim } = require(${onboardPath}); ); assert.match( result.stderr, - /unset NEMOCLAW_PROVIDER NEMOCLAW_MODEL NEMOCLAW_VLLM_MODEL NEMOCLAW_VLLM_PROFILE\s+nemoclaw onboard --fresh\s+Then select Local vLLM when prompted\./, + /unset NEMOCLAW_PROVIDER NEMOCLAW_MODEL NEMOCLAW_VLLM_MODEL\s+nemoclaw onboard --fresh\s+Then select Local vLLM when prompted\./, ); }); }); diff --git a/test/seed-hermes-dashboard-config.test.ts b/test/seed-hermes-dashboard-config.test.ts index f2506fc3130..52d56e6167e 100644 --- a/test/seed-hermes-dashboard-config.test.ts +++ b/test/seed-hermes-dashboard-config.test.ts @@ -122,59 +122,6 @@ describe.skipIf(!PY_YAML_AVAILABLE)("seed-dashboard-config.py", () => { expect(dash._nemoclaw_upstream).toEqual(GATEWAY_CONFIG._nemoclaw_upstream); }); - it("mirrors the experimental Station Ultra route and effective context", () => { - const stationUltraGateway = { - _config_version: 32, - _nemoclaw_upstream: { provider: "vllm-local", model: "nemotron-ultra" }, - model: { - default: "nemotron-ultra", - provider: "custom", - base_url: "https://inference.local/v1", - api_key: "sk-OPENSHELL-PROXY-REWRITE", - context_length: 262144, - }, - providers: { - "vllm-local": { - name: "vllm-local", - api: "https://inference.local/v1", - api_key: "sk-OPENSHELL-PROXY-REWRITE", - default_model: "nemotron-ultra", - discover_models: true, - }, - }, - custom_providers: [ - { - name: "vllm-local", - base_url: "https://inference.local/v1", - api_key: "sk-OPENSHELL-PROXY-REWRITE", - discover_models: true, - }, - ], - }; - const src = writeYaml("station-ultra-gateway.yaml", stationUltraGateway); - const dst = path.join(tmpDir, "station-ultra-dashboard.yaml"); - - const res = runSeed(src, dst); - expect(res.status).toBe(0); - - const dash = readYaml(dst) as Record; - expect(dash._nemoclaw_upstream).toEqual({ - provider: "vllm-local", - model: "nemotron-ultra", - }); - expect(dash.model).toMatchObject({ - default: "nemotron-ultra", - provider: "vllm-local", - base_url: "https://inference.local/v1", - context_length: 262144, - }); - expect(dash.providers["vllm-local"]).toMatchObject({ - name: "vllm-local", - default_model: "nemotron-ultra", - discover_models: true, - }); - }); - it("mirrors only the exact native Tavily backend into dashboard config", () => { const src = writeYaml("gw.yaml", { ...GATEWAY_CONFIG, From 0f42a7557c8424b5a90d3bf8238dc0a00e557822 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 12:08:56 -0700 Subject: [PATCH 20/21] test(inference): keep registry inventory linear Signed-off-by: Aaron Erickson --- src/lib/inference/vllm.test.ts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index b924d35ae71..1490c6f41c9 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -193,17 +193,17 @@ describe("managed vLLM image distribution boundary", () => { }); it("keeps every shipped managed-vLLM image on a registry digest", () => { - const refs = new Set(); - for (const imageSet of Object.values(VLLM_IMAGES)) { - for (const value of Object.values(imageSet)) { - if (typeof value === "object" && value !== null && "ref" in value) { - refs.add(String(value.ref)); - } - } - } - for (const model of VLLM_MODELS) { - if (model.runtime) refs.add(model.runtime.image); - } + const platformRefs = Object.values(VLLM_IMAGES).flatMap((imageSet) => + Object.values(imageSet) + .map((value) => + typeof value === "object" && value !== null && "ref" in value ? String(value.ref) : null, + ) + .filter((ref): ref is string => ref !== null), + ); + const runtimeRefs = VLLM_MODELS.map((model) => model.runtime?.image).filter( + (ref): ref is string => typeof ref === "string", + ); + const refs = new Set([...platformRefs, ...runtimeRefs]); expect(refs.size).toBeGreaterThan(0); for (const ref of refs) { From 7624d02c7da6d96bb49058bd49474941740e9bd1 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Wed, 15 Jul 2026 12:19:37 -0700 Subject: [PATCH 21/21] fix(installer): fail closed on Station preflights Signed-off-by: Aaron Erickson --- docs/inference/set-up-vllm.mdx | 10 ++- docs/reference/commands.mdx | 2 +- scripts/install.sh | 12 +++ src/lib/inference/vllm.test.ts | 117 ++++++++++++++++++++++++++++ src/lib/inference/vllm.ts | 4 - test/install-express-prompt.test.ts | 69 ++++++++++++++++ 6 files changed, 205 insertions(+), 9 deletions(-) diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index 0e6cd29bde3..fdab181047e 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -111,7 +111,8 @@ NemoClaw treats `/var/run/docker.sock` and `/run/docker.sock` as the local defau It cannot inspect daemon storage through a remote endpoint, a named non-default Docker context, or another Unix socket. When both `DOCKER_CONTEXT` and `DOCKER_HOST` are set, NemoClaw follows Docker CLI precedence and uses `DOCKER_CONTEXT`. Before a registered model with pinned size metadata is downloaded, NemoClaw also checks the filesystem that contains the host Hugging Face cache. -When either check is inconclusive, NemoClaw prints the reason, approximate requirement, and diagnostic commands, then continues without claiming that capacity is sufficient. +When a Docker image-storage check is inconclusive, NemoClaw prints the reason, approximate requirement, and diagnostic commands, then continues without claiming that capacity is sufficient. +An inconclusive model-cache check is guarded like an insufficient result because a registered model download can be hundreds of gigabytes: interactive setup requires explicit confirmation, and non-interactive setup stops unless the disk-space override is set. When verified capacity is insufficient, the warning identifies the image, available capacity, approximate requirement, checked path and source, remediation, and diagnostic commands. Interactive setup continues only when you enter `y` or `yes` explicitly. @@ -119,10 +120,11 @@ Pressing **Enter**, entering `n` or `no`, or entering any other response stops b `--yes` and `NEMOCLAW_YES=1` do not accept a verified insufficient-storage warning. -Non-interactive setup stops unless you set `NEMOCLAW_IGNORE_VLLM_DISK_SPACE=1` exactly. -The override applies only when NemoClaw verifies that Docker image storage or model-cache storage is insufficient. +They also do not accept an inconclusive model-cache warning. +Non-interactive setup stops for verified insufficient capacity or inconclusive model-cache capacity unless you set `NEMOCLAW_IGNORE_VLLM_DISK_SPACE=1` exactly. +The override applies when NemoClaw verifies that Docker image storage or model-cache storage is insufficient and when model-cache capacity cannot be verified. Use it only after you independently verify that the relevant filesystem can complete the download. -Inconclusive checks continue automatically and do not require the override. +An inconclusive Docker image-storage check retains its existing automatic-continue behavior and does not require the override. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 64372265328..035cec8f89b 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3288,7 +3288,7 @@ Set them before running `$$nemoclaw onboard`. | `NEMOCLAW_INSTALL_TAG` | release tag | For internal installer commands: the release tag to install. Defaults to the admin-promoted `lkg` tag when unset. Overridden by the `--install-tag` flag. | | `NEMOCLAW_VLLM_MODEL` | registry slug or Hugging Face model id | Selects the model the managed-vLLM install path serves. Recognised slugs: `qwen3.6-27b`, `qwen3.6-35b-a3b-nvfp4`, `nemotron-3-nano-4b`, `deepseek-v4-flash`, `nemotron-3-ultra-550b-a55b`, `deepseek-r1-distill-70b`. Unset uses the per-platform profile default. The DGX Station express installer sets `nemotron-3-ultra-550b-a55b` explicitly. Gated models (e.g. `deepseek-r1-distill-70b`) require `HF_TOKEN` or `HUGGING_FACE_HUB_TOKEN`. | | `NEMOCLAW_VLLM_EXTRA_ARGS_JSON` | JSON array of non-blank strings | Appends advanced operator-owned tokens to the managed `vllm serve` command after NemoClaw's registry defaults. Example: `["--max-num-seqs","2"]`. Malformed JSON, non-string tokens, or blank tokens fail before Docker work starts. | -| `NEMOCLAW_IGNORE_VLLM_DISK_SPACE` | exactly `1` | Allows managed vLLM to continue when verified Docker image-storage or registered model-cache capacity is insufficient; other values do not bypass either check. Inconclusive checks report diagnostics and continue automatically. | +| `NEMOCLAW_IGNORE_VLLM_DISK_SPACE` | exactly `1` | Allows managed vLLM to continue when verified Docker image-storage or registered model-cache capacity is insufficient, or when registered model-cache capacity cannot be measured; other values do not bypass either check. An inconclusive Docker image-storage check reports diagnostics and retains its existing automatic-continue behavior. | | `NEMOCLAW_MINIMAL_BOOTSTRAP` | `1` to enable | Skips default OpenClaw workspace-template seeding for new pristine workspaces. Existing files are not deleted; refer to [Understand Runtime Changes](../manage-sandboxes/configure-sandboxes/understand-runtime-changes). | diff --git a/scripts/install.sh b/scripts/install.sh index f7212c51398..efd5c0eb0a6 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -2889,6 +2889,12 @@ validate_station_deepseek_override() { esac } +preflight_explicit_express_flags() { + local platform + platform="$(detect_express_platform)" + validate_station_deepseek_override "$platform" +} + configure_station_express_model() { local selected_model selected_model="$(normalize_station_vllm_model "${NEMOCLAW_VLLM_MODEL:-}")" @@ -3128,6 +3134,12 @@ main() { # or any other installer side effect. resolve_nemoclaw_gateway_port >/dev/null + # Explicit express-only flags must fail before license state, Docker, build + # dependencies, or any other host mutation. maybe_offer_express_install + # repeats the same authoritative validation at the prompt boundary because + # it is also exercised directly by sourced-installer callers and tests. + preflight_explicit_express_flags + print_banner # Fail-fast license-acceptance check (#2671). Headless curl|bash still exits diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index 1490c6f41c9..7812f4a1b93 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -92,6 +92,15 @@ function currentHostIdentity(): string | null { return uid === undefined || gid === undefined ? null : `${String(uid)}:${String(gid)}`; } +function inconclusiveModelStorage(reason = "statfs unavailable") { + return { + ok: false as const, + reason, + path: path.join(os.homedir(), ".cache", "huggingface"), + source: "Hugging Face cache", + }; +} + function mockDockerSpawnSuccess(): EventEmitter & { stdout: EventEmitter; stderr: EventEmitter; @@ -875,6 +884,83 @@ describe("installVllm model resolution", () => { expect(errors).toContain("Non-interactive setup stops before the guarded download"); }); + it("fails closed before downloads when non-interactive model-cache capacity is inconclusive", async () => { + process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName); + mocks.probeHostStorage.mockReturnValue(inconclusiveModelStorage()); + const result = await installVllm(profile, { + hasImage: false, + nonInteractive: true, + promptFn: vi.fn(), + }); + expect(result).toEqual({ ok: false }); + expect(mocks.probeHostStorage).toHaveBeenCalledTimes(1); + expect(mocks.dockerImageInspectFormat).not.toHaveBeenCalled(); + expect(mocks.probeDockerStorage).not.toHaveBeenCalled(); + expect(mocks.dockerPullWithProgressWatchdog).not.toHaveBeenCalled(); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); + expect(mkdirSpy).not.toHaveBeenCalled(); + const errors = errSpy.mock.calls.map((call: unknown[]) => String(call[0])).join("\n"); + expect(errors).toContain("Unable to verify storage for the managed vLLM model cache"); + expect(errors).toContain("statfs unavailable"); + expect(errors).toContain("Non-interactive setup stops before the guarded download"); + expect(errors).toContain("NEMOCLAW_IGNORE_VLLM_DISK_SPACE=1"); + }); + + it.each([ + { reply: "y", expected: { ok: true }, pulls: 1, downloads: 1 }, + { reply: "n", expected: { ok: false }, pulls: 0, downloads: 0 }, + { reply: "", expected: { ok: false }, pulls: 0, downloads: 0 }, + ])("requires an explicit interactive '$reply' for an inconclusive model-cache probe", async ({ + reply, + expected, + pulls, + downloads, + }) => { + process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName); + mocks.dockerImageInspectFormat.mockReturnValue("sha256:cached-image"); + mocks.probeHostStorage.mockReturnValue(inconclusiveModelStorage()); + const replies = ["y", reply]; + const promptFn = vi.fn(async () => replies.shift() ?? ""); + + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: false, + promptFn, + }); + + expect(result).toEqual(expected); + expect(promptFn).toHaveBeenCalledTimes(2); + expect(promptFn).toHaveBeenLastCalledWith(" Continue with the model download anyway? [y/N]: "); + expect(mocks.dockerPullWithProgressWatchdog).toHaveBeenCalledTimes(pulls); + expect(mocks.dockerSpawn).toHaveBeenCalledTimes(downloads); + }); + + it("allows an inconclusive model-cache probe only with the exact disk-space override", async () => { + process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + process.env.NEMOCLAW_IGNORE_VLLM_DISK_SPACE = "1"; + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName); + mocks.dockerImageInspectFormat.mockReturnValue("sha256:cached-image"); + mocks.probeHostStorage.mockReturnValue(inconclusiveModelStorage()); + const result = await installVllm(profile, { + hasImage: true, + nonInteractive: true, + promptFn: vi.fn(), + }); + expect(result).toEqual({ ok: true }); + expect(mocks.probeHostStorage).toHaveBeenCalledTimes(1); + expect(mocks.dockerPullWithProgressWatchdog).toHaveBeenCalledTimes(1); + expect(mocks.dockerSpawn).toHaveBeenCalledTimes(1); + expect(errSpy).toHaveBeenCalledWith( + expect.stringContaining("Continuing because NEMOCLAW_IGNORE_VLLM_DISK_SPACE=1"), + ); + }); + it("re-probes after a cold image pull to prevent shared-filesystem overcommit", async () => { process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; @@ -916,6 +1002,37 @@ describe("installVllm model resolution", () => { ); }); + it("stops when the post-pull model-cache capacity re-probe is inconclusive", async () => { + process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; + mockSuccessfulVllmInstall(profile.containerName); + mocks.probeHostStorage + .mockReturnValueOnce({ + ok: true, + capacity: { + availableBytes: 1_000_000_000_000n, + path: path.join(os.homedir(), ".cache", "huggingface"), + source: "Hugging Face cache", + }, + }) + .mockReturnValueOnce(inconclusiveModelStorage("statfs unavailable after image pull")); + + const result = await installVllm(profile, { + hasImage: false, + nonInteractive: true, + promptFn: vi.fn(), + }); + + expect(result).toEqual({ ok: false }); + expect(mocks.probeHostStorage).toHaveBeenCalledTimes(2); + expect(mocks.dockerPullWithProgressWatchdog).toHaveBeenCalledTimes(1); + expect(mocks.dockerSpawn).not.toHaveBeenCalled(); + expect(mocks.dockerRunDetached).not.toHaveBeenCalled(); + expect(errSpy).toHaveBeenCalledWith( + expect.stringContaining("Unable to verify storage for the managed vLLM model cache"), + ); + }); + it("requires only missing snapshot bytes plus headroom for a partial Ultra cache", async () => { process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index c4d454a1658..a98de81b455 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -886,10 +886,6 @@ async function modelStorageAccepted( const requiredBytes = modelStorageRequirementBytes(Number(remainingBytes)); if (probe.ok && probe.capacity.availableBytes >= requiredBytes) return true; printModelStorageWarning(model, probe, requiredBytes, cachedBytes, snapshotBytes); - if (!probe.ok) { - console.error(" Continuing because model-cache capacity could not be verified."); - return true; - } if (env[VLLM_STORAGE_OVERRIDE_ENV] === "1") { console.error(` Continuing because ${VLLM_STORAGE_OVERRIDE_ENV}=1.`); return true; diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index 2deba97beb3..65ba31f39fd 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -287,6 +287,75 @@ detect_express_platform expect(output).not.toMatch(/Run express install/); }); + it.each([ + { + name: "a Station-only flag on DGX Spark", + args: ["--station-deepseek"], + platform: "DGX Spark", + env: {}, + message: /--station-deepseek requires a detected DGX Station \(detected: DGX Spark\)/, + }, + { + name: "a conflicting non-interactive flag", + args: ["--station-deepseek", "--non-interactive"], + platform: "DGX Station", + env: {}, + message: + /--station-deepseek selects the DGX Station express prompt and cannot be combined with --non-interactive/, + }, + { + name: "a conflicting Station model", + args: ["--station-deepseek"], + platform: "DGX Station", + env: { NEMOCLAW_VLLM_MODEL: "nemotron-3-ultra-550b-a55b" }, + message: /--station-deepseek conflicts with NEMOCLAW_VLLM_MODEL='nemotron-3-ultra-550b-a55b'/, + }, + ])("rejects $name before Docker or build-dependency mutation", ({ + args, + platform, + env, + message, + }) => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-flag-preflight-")); + const mutationLog = path.join(tmp, "host-mutations.log"); + const result = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + ` +source "$INSTALLER_UNDER_TEST" >/dev/null +detect_express_platform() { printf "%s" "$EXPRESS_PLATFORM"; } +ensure_docker() { printf "ensure_docker\\n" >>"$MUTATION_LOG"; } +ensure_openshell_build_deps() { printf "ensure_openshell_build_deps\\n" >>"$MUTATION_LOG"; } +main "$@" +`, + "_", + ...args, + ], + { + cwd: tmp, + encoding: "utf-8", + env: { + HOME: tmp, + PATH: TEST_SYSTEM_PATH, + INSTALLER_UNDER_TEST: INSTALLER_PAYLOAD, + MUTATION_LOG: mutationLog, + EXPRESS_PLATFORM: platform, + ...env, + }, + }, + ); + const output = `${result.stdout}${result.stderr}`; + const mutations = fs.existsSync(mutationLog) ? fs.readFileSync(mutationLog, "utf-8") : ""; + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(message); + expect(mutations).toBe(""); + fs.rmSync(tmp, { recursive: true, force: true }); + }); + it.each([ ["NEMOCLAW_NO_EXPRESS", "1", /cannot be combined with NEMOCLAW_NO_EXPRESS=1/], ["NON_INTERACTIVE", "1", /cannot be combined with --non-interactive/],