diff --git a/docs/inference/set-up-vllm.mdx b/docs/inference/set-up-vllm.mdx index ac1f484ca76..fcd0ded5c94 100644 --- a/docs/inference/set-up-vllm.mdx +++ b/docs/inference/set-up-vllm.mdx @@ -267,6 +267,24 @@ When you start managed vLLM outside the installer express flow, NemoClaw uses th For the managed single-node DGX Spark `nvidia/Qwen3.6-35B-A3B-NVFP4` profile, NemoClaw enables async scheduling and does not enable multi-token prediction (MTP) speculative decoding. The profile retains `--gpu-memory-utilization 0.4`, a `262144`-token context window, `4` concurrent sequences, an `8192`-token batch limit, chunked prefill, prefix caching, and its registered parsers and acceleration backends. +NVIDIA Nemotron 3.5 Lightning is an additional explicit-only single-DGX Spark profile and does not change the Qwen default. +Select its stable catalog ID after confirming that `$$nemoclaw profiles list` reports it as compatible. + +```bash +$$nemoclaw onboard --profile vllm.dgx-spark-gb10.single.nemotron-3.5-lightning-30b-a3b-nvfp4 +``` + +The profile serves the public [`nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4`](https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4) checkpoint through a pinned ARM64 vLLM image. +Its profile limits for one concurrent sequence are a 65,536-token context window, a 4,096-token batch limit, 0.7 GPU-memory utilization, FP8 KV cache, and one-token MTP speculative decoding. +It uses vLLM's Python frontend because the pinned runtime's Rust frontend does not register the required `step3p5` reasoning and tool-call parsers. +Physical single-DGX Spark validation covered plain chat, automatic and named structured tool calls, the 10-turn 32K-input/2K-output release workload, OpenClaw chat and file tools, cached restart, cleanup, and restoration of the original Qwen route. + + +This profile is an explicit opt-in for one DGX Spark and remains Experimental. +Promotion requires broader validation of the pinned Python frontend and the `step3p5` reasoning and tool-call parsers. +The fixed recipe intentionally rejects model and serve-argument overrides; use another catalog profile instead of modifying its validated resource boundary. + + Muse Glimmer is an additional single-DGX Spark choice and does not change the Qwen default. Select `muse-glimmer-30b` to serve `Inferact/Muse-Glimmer-30B-NVFP4-W4A4` with the `muse-glimmer` alias. The registered recipe configures a 32,768-token context window, 1 concurrent sequence, a 4,096-token batch limit, and 0.75 GPU-memory utilization. diff --git a/managed-inference/presets/vllm.dgx-spark-gb10.single.nemotron-3.5-lightning-30b-a3b-nvfp4.yaml b/managed-inference/presets/vllm.dgx-spark-gb10.single.nemotron-3.5-lightning-30b-a3b-nvfp4.yaml new file mode 100644 index 00000000000..778238f29d0 --- /dev/null +++ b/managed-inference/presets/vllm.dgx-spark-gb10.single.nemotron-3.5-lightning-30b-a3b-nvfp4.yaml @@ -0,0 +1,106 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nemoclaw.nvidia.com/managed-inference/v1 +kind: ServingPreset + +metadata: + id: vllm.dgx-spark-gb10.single.nemotron-3.5-lightning-30b-a3b-nvfp4 + displayName: NVIDIA Nemotron 3.5 Lightning 30B-A3B NVFP4 on one DGX Spark + supportState: experimental + +spec: + selection: explicit-only + priority: 450 + + requirements: + all: + - readiness: + scope: everyNode + kind: qualification + id: host.platform.dgx_spark + status: qualified + - readiness: + scope: everyNode + kind: capability + id: host.platform.supported + state: present + - readiness: + scope: everyNode + kind: capability + id: host.platform.dgx_spark + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.available + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.daemon_reachable + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.runtime_supported + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.storage_compatible + state: present + - readiness: + scope: everyNode + kind: capability + id: host.gpu.nvidia_available + state: present + - readiness: + scope: everyNode + kind: capability + id: host.gpu.container_toolkit_available + state: present + - readiness: + scope: everyNode + kind: capability + id: host.gpu.cdi_healthy + state: present + - readiness: + scope: everyNode + kind: observation + id: host.os.platform + comparison: + operator: equals + value: linux + - readiness: + scope: everyNode + kind: observation + id: host.os.architecture + comparison: + operator: equals + value: arm64 + - readiness: + scope: everyNode + kind: observation + id: host.docker.runtime + comparison: + operator: equals + value: docker + - readiness: + scope: everyNode + kind: observation + id: host.gpu.count + comparison: + operator: at-least + value: 1 + - readiness: + scope: everyNode + kind: observation + id: host.gpu.driver_version + comparison: + operator: version-at-least + value: 580.65.06 + + plan: + backend: vllm + recipeRef: vllm.nemotron-3.5-lightning-30b-a3b-nvfp4.spark-single.v1 diff --git a/managed-inference/recipes/vllm.nemotron-3.5-lightning-30b-a3b-nvfp4.spark-single.v1.yaml b/managed-inference/recipes/vllm.nemotron-3.5-lightning-30b-a3b-nvfp4.spark-single.v1.yaml new file mode 100644 index 00000000000..bb4d06e630f --- /dev/null +++ b/managed-inference/recipes/vllm.nemotron-3.5-lightning-30b-a3b-nvfp4.spark-single.v1.yaml @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nemoclaw.nvidia.com/managed-inference/v1 +kind: ServingRecipe + +metadata: + id: vllm.nemotron-3.5-lightning-30b-a3b-nvfp4.spark-single.v1 + displayName: NVIDIA Nemotron 3.5 Lightning 30B-A3B NVFP4 with vLLM + +spec: + backend: vllm + + model: + id: nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + revision: 0dcd680e5585c791728c83342b311d0a0026dbeb + servedName: nvidia-nemotron-3.5-lightning-30b-a3b-nvfp4 + downloadSizeBytes: 21561882284 + gated: false + installFastSafetensors: false + preparation: + ref: none/v1 + + runtime: + image: vllm/vllm-openai@sha256:3af90144a0926e5c5fe46ee16e5201e763dd854538b9d7ce433755f11dadaf78 + imageDownloadSizeBytes: 12691539532 + pullTimeoutSeconds: 3600 + architecture: arm64 + networkMode: bridge + ipcMode: host + sharedMemoryBytes: 34359738368 + gpuRequest: all + devices: [] + ulimits: + memlock: -1 + stackBytes: 67108864 + modelCache: + source: huggingface-cache + target: /root/.cache/huggingface + temporaryFilesystems: [] + environment: + # The Lightning 3.5 Spark validation used vLLM's Python frontend; + # `step3p5` parsers are not available in the Rust frontend. + VLLM_USE_RUST_FRONTEND: "0" + + execution: + materializerRef: vllm.host-local/v1 + lifecycleRef: vllm.host-local.lifecycle/v1 + + serve: + authentication: bearer + executable: /usr/local/bin/vllm + arguments: + - name: --max-model-len + value: 65536 + - name: --max-num-seqs + value: 1 + - name: --gpu-memory-utilization + value: 0.7 + - name: --kv-cache-dtype + value: fp8 + - name: --moe-backend + value: marlin + - name: --max-num-batched-tokens + value: 4096 + - name: --speculative-config + value: '{"method":"mtp","num_speculative_tokens":1,"moe_backend":"flashinfer_cutlass"}' + - name: --mamba-backend + value: flashinfer + - name: --mamba-ssm-cache-dtype + value: float16 + - name: --enable-mamba-cache-stochastic-rounding + - name: --mamba-cache-philox-rounds + value: 5 + - name: --enable-prefix-caching + - name: --mamba-cache-mode + value: align + - name: --compilation-config + value: '{"cudagraph_capture_sizes":[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,1024,2048,4096]}' + - name: --enable-auto-tool-choice + - name: --tool-call-parser + value: step3p5 + - name: --reasoning-parser + value: step3p5 + + readiness: + timeoutSeconds: 1800 + expectedModel: nvidia-nemotron-3.5-lightning-30b-a3b-nvfp4 diff --git a/src/lib/inference/serving/adapter-registry.test.ts b/src/lib/inference/serving/adapter-registry.test.ts index c520136d049..0580bf9d477 100644 --- a/src/lib/inference/serving/adapter-registry.test.ts +++ b/src/lib/inference/serving/adapter-registry.test.ts @@ -36,6 +36,14 @@ function shippedRecipe(): ManagedInferenceServingRecipe { return structuredClone(recipe as ManagedInferenceServingRecipe); } +function shippedHostLocalRecipe(): ManagedInferenceServingRecipe { + const recipe = loadManagedInferenceCatalog().recipes.find( + ({ spec }) => spec.execution.materializerRef === HOST_LOCAL_VLLM_MATERIALIZER_REF, + ); + expect(recipe).toBeDefined(); + return structuredClone(recipe as ManagedInferenceServingRecipe); +} + describe("managed inference adapter registries", () => { it("registers one versioned descriptor for each shipped managed cluster mechanic", () => { expect(listManagedInferenceTopologyQualificationDescriptors()).toMatchObject([ @@ -143,6 +151,45 @@ describe("managed inference adapter registries", () => { expect(getManagedInferenceRecipeRegistrationError(wrongShape)).toMatch(/TP times PP/u); }); + it("accepts shell-quoted JSON values for host-local vLLM arguments", () => { + const recipe = shippedHostLocalRecipe(); + const structuredArgument = { + ...recipe, + spec: { + ...recipe.spec, + serve: { + ...recipe.spec.serve, + arguments: [ + ...recipe.spec.serve.arguments, + { + name: "--speculative-config", + value: '{"method":"mtp","num_speculative_tokens":1}', + }, + ], + }, + }, + } as ManagedInferenceServingRecipe; + const unsafeArgument = { + ...structuredArgument, + spec: { + ...structuredArgument.spec, + serve: { + ...structuredArgument.spec.serve, + arguments: structuredArgument.spec.serve.arguments.map((argument) => + argument.name === "--speculative-config" + ? { ...argument, value: '{"method":"mtp"};touch/tmp/unsafe' } + : argument, + ), + }, + }, + } as ManagedInferenceServingRecipe; + + expect(getManagedInferenceRecipeRegistrationError(structuredArgument)).toBeUndefined(); + expect(getManagedInferenceRecipeRegistrationError(unsafeArgument)).toMatch( + /bounded safe text/u, + ); + }); + it("rejects schema-valid values that the registered adapter cannot execute", () => { const recipe = shippedRecipe(); const unsupportedCache = { diff --git a/src/lib/inference/serving/adapter-registry.ts b/src/lib/inference/serving/adapter-registry.ts index a10be5c0159..00ae863ca73 100644 --- a/src/lib/inference/serving/adapter-registry.ts +++ b/src/lib/inference/serving/adapter-registry.ts @@ -81,6 +81,9 @@ const STABLE_ID = /^[A-Za-z0-9][A-Za-z0-9._/-]{0,159}$/u; const PINNED_IMAGE = /^(?:[a-z0-9]+(?:[._-][a-z0-9]+)*(?::[0-9]+)?\/)?(?:[a-z0-9]+(?:[._-][a-z0-9]+)*\/)*[a-z0-9]+(?:[._-][a-z0-9]+)*@sha256:[0-9a-f]{64}$/u; const SAFE_ENVIRONMENT_NAME = /^[A-Z][A-Z0-9_]{0,127}$/u; +// Structured vLLM flags are passed as individual argv values after shell +// quoting. Keep the allowlist narrow while admitting JSON objects and arrays. +const HOST_LOCAL_SAFE_VALUE = /^[A-Za-z0-9_@%+=:,./{}[\]"-]+$/u; const SHA256_DIGEST = /^sha256:[0-9a-f]{64}$/u; const MANAGED_CLUSTER_MATERIALIZER_OWNED_ENVIRONMENT = new Set([ "GLOO_SOCKET_IFNAME", @@ -410,13 +413,13 @@ function validateHostLocalVllmMaterializerRecipe( typeof value === "string" && (Buffer.byteLength(value, "utf8") > 16_384 || value.includes("\0") || - !/^[A-Za-z0-9_@%+=:,./-]+$/u.test(value)), + !HOST_LOCAL_SAFE_VALUE.test(value)), ) || Object.values(recipe.spec.runtime.environment).some( (value) => Buffer.byteLength(value, "utf8") > 4_096 || value.includes("\0") || - !/^[A-Za-z0-9_@%+=:,./-]+$/u.test(value), + !HOST_LOCAL_SAFE_VALUE.test(value), ) ) { return "host-local vLLM serving values must be bounded safe text"; diff --git a/src/lib/inference/serving/catalog-loader.test.ts b/src/lib/inference/serving/catalog-loader.test.ts index 88e75d13e2f..4b34cbe7d49 100644 --- a/src/lib/inference/serving/catalog-loader.test.ts +++ b/src/lib/inference/serving/catalog-loader.test.ts @@ -32,6 +32,7 @@ const EMPTY_CATALOG: CompiledServingCatalog = { const EXPECTED_MANAGED_RECIPE_IDS = [ "llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1", "vllm.deepseek-v4-flash-0731.spark-dual.v1", + "vllm.nemotron-3.5-lightning-30b-a3b-nvfp4.spark-single.v1", "vllm.qwen3-6-35b-a3b-nvfp4.spark-single.v1", ]; const EXPECTED_MANAGED_PRESET_IDS = [ @@ -39,14 +40,17 @@ const EXPECTED_MANAGED_PRESET_IDS = [ "llama-cpp.linux-amd64-nvidia.single.nemotron-3-nano-30b-a3b", "local-model-profile.vllm.spark.v1", "vllm.dgx-spark-gb10.dual.deepseek-v4-flash-0731", + "vllm.dgx-spark-gb10.single.nemotron-3.5-lightning-30b-a3b-nvfp4", ]; const EXPECTED_MANAGED_SOURCE_IDS = [ "llama-cpp.dgx-spark-gb10.single.nemotron-3-nano-30b-a3b", "llama-cpp.linux-amd64-nvidia.single.nemotron-3-nano-30b-a3b", "local-model-profile.vllm.spark.v1", "vllm.dgx-spark-gb10.dual.deepseek-v4-flash-0731", + "vllm.dgx-spark-gb10.single.nemotron-3.5-lightning-30b-a3b-nvfp4", "llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1", "vllm.deepseek-v4-flash-0731.spark-dual.v1", + "vllm.nemotron-3.5-lightning-30b-a3b-nvfp4.spark-single.v1", "vllm.qwen3-6-35b-a3b-nvfp4.spark-single.v1", ]; diff --git a/test/managed-inference-catalog-compiler.test.ts b/test/managed-inference-catalog-compiler.test.ts index 49c8fbd2a35..11ea4ca683a 100644 --- a/test/managed-inference-catalog-compiler.test.ts +++ b/test/managed-inference-catalog-compiler.test.ts @@ -21,6 +21,8 @@ const LLAMA_CPP_IMAGE = "ghcr.io/ggml-org/llama.cpp@sha256:866ad568474de9e835e487ae841ad6ace1a494b5eab4f292cbd45adb6180f711"; const LLAMA_CPP_MODEL_DIGEST = "sha256:627f5b04aedc97f967332f331bd75b7a4ed2f33ca83e6ee74b44235cc1887890"; +const LIGHTNING_PROFILE_ID = "vllm.dgx-spark-gb10.single.nemotron-3.5-lightning-30b-a3b-nvfp4"; +const LIGHTNING_RECIPE_ID = "vllm.nemotron-3.5-lightning-30b-a3b-nvfp4.spark-single.v1"; function catalogSources(): ServingCatalogSource[] { return (["presets", "recipes"] as const).flatMap((kind) => { @@ -140,6 +142,65 @@ describe("managed inference YAML profile contract", () => { }); }); + it("compiles the explicit DGX Spark Lightning 3.5 vLLM profile from YAML (#8385)", () => { + const catalog = compile(catalogSources()); + const preset = catalog.presets.find(({ metadata }) => metadata.id === LIGHTNING_PROFILE_ID); + const recipe = catalog.recipes.find(({ metadata }) => metadata.id === LIGHTNING_RECIPE_ID); + + expect(preset?.metadata.supportState).toBe("experimental"); + expect(preset?.spec).toMatchObject({ + selection: "explicit-only", + plan: { backend: "vllm", recipeRef: LIGHTNING_RECIPE_ID }, + }); + expect(recipe?.spec).toMatchObject({ + backend: "vllm", + model: { + id: "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", + revision: "0dcd680e5585c791728c83342b311d0a0026dbeb", + servedName: "nvidia-nemotron-3.5-lightning-30b-a3b-nvfp4", + }, + runtime: { + architecture: "arm64", + image: + "vllm/vllm-openai@sha256:3af90144a0926e5c5fe46ee16e5201e763dd854538b9d7ce433755f11dadaf78", + environment: { VLLM_USE_RUST_FRONTEND: "0" }, + }, + execution: { + materializerRef: "vllm.host-local/v1", + lifecycleRef: "vllm.host-local.lifecycle/v1", + }, + serve: { + arguments: expect.arrayContaining([ + { name: "--tool-call-parser", value: "step3p5" }, + { name: "--reasoning-parser", value: "step3p5" }, + { + name: "--speculative-config", + value: '{"method":"mtp","num_speculative_tokens":1,"moe_backend":"flashinfer_cutlass"}', + }, + ]), + }, + }); + }); + + it("documents the Experimental Lightning support boundary (#8385)", () => { + const setupGuide = readFileSync( + path.join(REPOSITORY_ROOT, "docs", "inference", "set-up-vllm.mdx"), + "utf8", + ); + const warning = setupGuide.match( + /([\s\S]*?)<\/Warning>/u, + )?.[1]; + + expect(warning).toBeDefined(); + expect(warning).toContain( + "This profile is an explicit opt-in for one DGX Spark and remains Experimental.", + ); + expect(warning).toContain( + "Promotion requires broader validation of the pinned Python frontend and the `step3p5` reasoning and tool-call parsers.", + ); + expect(warning).not.toContain("qualified for broader support"); + }); + it("keeps vLLM automatic while llama.cpp remains explicit-only (#8173)", () => { const catalog = compile(catalogSources()); const vllmPreset = catalog.presets.find(({ metadata }) => metadata.id === PROFILE_ID); @@ -177,5 +238,7 @@ describe("managed inference YAML profile contract", () => { expect(productionSources).not.toContain(RECIPE_ID); expect(productionSources).not.toContain(LLAMA_CPP_PROFILE_ID); expect(productionSources).not.toContain(LLAMA_CPP_RECIPE_ID); + expect(productionSources).not.toContain(LIGHTNING_PROFILE_ID); + expect(productionSources).not.toContain(LIGHTNING_RECIPE_ID); }); });