From 77ba46da1d04c3e4373b5d705583b213e99921ac Mon Sep 17 00:00:00 2001 From: Fadi Saady Date: Wed, 12 Aug 2026 23:12:01 +0200 Subject: [PATCH 1/2] chore(recipes): minor fixes to Nemotron 3.5 Lightning recipes. - Added more perf data. - Corrected context size in docs. - Added better instructions for EFA setup. - Removing redundancies. Signed-off-by: Fadi Saady --- .../pages/recipes/model-recipes/overview.mdx | 2 +- recipes/nemotron-3.5-lightning/perf/README.md | 8 +- recipes/nemotron-3.5-lightning/vllm/README.md | 21 ++-- .../vllm/agg-b200-dflash/deploy.yaml | 2 - .../agg-b200-dspark-kv-router/deploy.yaml | 2 - .../vllm/agg-b200-dspark/deploy.yaml | 2 - .../vllm/agg-b200-mtp/deploy.yaml | 2 - .../vllm/agg-gb200-dflash/deploy.yaml | 2 - .../vllm/agg-gb200-dspark/deploy.yaml | 2 - .../vllm/agg-gb200-mtp/deploy.yaml | 2 - .../vllm/agg-h100-dflash/deploy.yaml | 2 - .../agg-h100-dspark-kv-router/deploy.yaml | 2 - .../vllm/agg-h100-dspark/deploy.yaml | 2 - .../vllm/agg-h100-mtp/deploy.yaml | 2 - .../vllm/agg-h200-dflash/deploy.yaml | 2 - .../agg-h200-dspark-kv-router/deploy.yaml | 2 - .../vllm/agg-h200-dspark/deploy.yaml | 2 - .../vllm/agg-h200-mtp/deploy.yaml | 2 - .../vllm/disagg-b200-dspark/deploy.yaml | 10 -- .../vllm/disagg-gb200-dflash/deploy.yaml | 4 - .../vllm/disagg-gb200-dspark/deploy.yaml | 4 - .../vllm/disagg-h100-dflash/deploy.yaml | 4 - .../vllm/disagg-h100-dspark/deploy.yaml | 4 - .../vllm/disagg-h100-mtp/deploy.yaml | 4 - .../vllm/disagg-h200-dflash/deploy.yaml | 4 - .../vllm/disagg-h200-dspark/deploy.yaml | 4 - .../vllm/disagg-h200-mtp/deploy.yaml | 4 - recipes/nemotron-3.5-lightning/vllm/efa.md | 101 ++++++++++++++++++ 28 files changed, 114 insertions(+), 90 deletions(-) create mode 100644 recipes/nemotron-3.5-lightning/vllm/efa.md diff --git a/docs/fern/pages/recipes/model-recipes/overview.mdx b/docs/fern/pages/recipes/model-recipes/overview.mdx index f61a3c379f5c..8c07495fd3f4 100644 --- a/docs/fern/pages/recipes/model-recipes/overview.mdx +++ b/docs/fern/pages/recipes/model-recipes/overview.mdx @@ -111,7 +111,7 @@ Start with the model, runtime, and hardware you need to run. Model cards are sor

Nemotron-3.5-Lightning

NVIDIA

30B hybrid Mamba/Attention/MoE recipe matrix for H100, H200, B200, and GB200, with vLLM MTP/DFlash/DSpark targets, DSpark KV-routing aggregate variants on H100/H200/B200, and experimental TensorRT-LLM MTP/no-spec aggregates.

-
vLLM + TRT-LLM1-4x GPUAgentic · 262K context
+
vLLM + TRT-LLM1-4x GPUAgentic · 1M context
Open recipe
diff --git a/recipes/nemotron-3.5-lightning/perf/README.md b/recipes/nemotron-3.5-lightning/perf/README.md index c85acfaa8129..19ca890bf21d 100644 --- a/recipes/nemotron-3.5-lightning/perf/README.md +++ b/recipes/nemotron-3.5-lightning/perf/README.md @@ -50,7 +50,7 @@ aiperf profile --custom-dataset-type mooncake_trace --num-requests \ | `vllm/agg-h200-dspark/deploy.yaml` | vLLM | agg | H200 | Single worker | DSpark | 7 | 3.69 | 20 | 3541 | 3411 | 130 | 243.97 | 2193.59 | 109.68 | | `vllm/agg-h200-dspark-kv-router/deploy.yaml` | vLLM | agg | H200 | KV router | DSpark | 7 | 3.69 | 80 | 3541 | 3411 | 130 | 286.67 | 2171.00 | 108.55 | | `vllm/agg-h200-mtp/deploy.yaml` | vLLM | agg | H200 | Single worker | MTP | 7 | 3.687 | 12 | 3541 | 3411 | 130 | 250.66 | 1478.67 | 123.22 | -| `vllm/disagg-b200-dspark/deploy.yaml` | vLLM | disagg | B200 | 1P1D | DSpark | 1 | 1.83 | 8 | 500 | 489 | 11 | 236.43 | 904.03 | 226.01 | +| `vllm/disagg-b200-dspark/deploy.yaml` | vLLM | disagg | B200 | 1P1D | DSpark | 1 | 1.83 | 8 | 3541 | 3410 | 131 | 235.95 | 984.09 | 246.02 | | `vllm/disagg-gb200-dflash/deploy.yaml` | vLLM | disagg | GB200 | 1P1D | DFlash | 7 | 3.41 | 4 | 3541 | 3411 | 130 | 491.40 | 584.85 | 292.43 | | `vllm/disagg-gb200-dspark/deploy.yaml` | vLLM | disagg | GB200 | 1P1D | DSpark | 1 | 1.83 | 8 | 3541 | 3411 | 130 | 599.17 | 939.28 | 234.82 | | `vllm/disagg-h100-dflash/deploy.yaml` | vLLM | disagg | H100 | 1P1D | DFlash | 3 | 2.73 | 12 | 3541 | 3404 | 137 | 542.71 | 1117.29 | 186.21 | @@ -59,9 +59,9 @@ aiperf profile --custom-dataset-type mooncake_trace --num-requests \ | `vllm/disagg-h200-dflash/deploy.yaml` | vLLM | disagg | H200 | 1P1D | DFlash | 3 | 2.73 | 12 | 3541 | 3401 | 140 | 408.30 | 1277.86 | 212.98 | | `vllm/disagg-h200-dspark/deploy.yaml` | vLLM | disagg | H200 | 1P1D | DSpark | 1 | 1.83 | 24 | 3541 | 3411 | 130 | 834.80 | 1656.90 | 138.08 | | `vllm/disagg-h200-mtp/deploy.yaml` | vLLM | disagg | H200 | 1P1D | MTP | 5 | 3.421 | 12 | 3541 | 3342 | 199 | 424.66 | 1203.97 | 200.66 | -| `trtllm/agg-b200/deploy.yaml` | TensorRT-LLM | agg | B200 | Single worker | None | 0 | 0.0 | 18 | 500 | 489 | 11 | 377.68 | 386.998 | 21.50 | -| `trtllm/agg-b200-mtp/deploy.yaml` | TensorRT-LLM | agg | B200 | Single worker | MTP | 3 | 2.874 | 18 | 500 | 489 | 11 | 446.88 | 427.82 | 23.77 | +| `trtllm/agg-b200/deploy.yaml` | TensorRT-LLM | agg | B200 | Single worker | None | 0 | 0.0 | 18 | 3541 | 3411 | 130 | 378.04 | 378.27 | 21.01 | +| `trtllm/agg-b200-mtp/deploy.yaml` | TensorRT-LLM | agg | B200 | Single worker | MTP | 3 | 2.874 | 18 | 3541 | 3411 | 130 | 448.71 | 490.28 | 27.24 | | `trtllm/agg-gb200/deploy.yaml` | TensorRT-LLM | agg | GB200 | Single worker | None | 0 | 0.0 | 18 | 500 | 499 | 1 | 318.88 | 280.10 | 15.56 | | `trtllm/agg-gb200-mtp/deploy.yaml` | TensorRT-LLM | agg | GB200 | Single worker | MTP | 3 | 2.874 | 18 | 500 | 499 | 1 | 400.88 | 580.09 | 32.23 | | `trtllm/agg-h100-mtp/deploy.yaml` | TensorRT-LLM | agg | H100 | Single worker | MTP | 7 | 3.687 | 12 | 500 | 499 | 1 | 1757.00 | 264.87 | 22.07 | -| `trtllm/agg-h200-mtp/deploy.yaml` | TensorRT-LLM | agg | H200 | Single worker | MTP | 7 | 3.687 | 12 | 500 | 499 | 1 | 1630.96 | 269.26 | 22.44 | +| `trtllm/agg-h200-mtp/deploy.yaml` | TensorRT-LLM | agg | H200 | Single worker | MTP | 7 | 3.687 | 12 | 3541 | 3527 | 14 | 1692.98 | 250.05 | 20.84 | diff --git a/recipes/nemotron-3.5-lightning/vllm/README.md b/recipes/nemotron-3.5-lightning/vllm/README.md index 3f6a3177b5bd..e20ee5e0303b 100644 --- a/recipes/nemotron-3.5-lightning/vllm/README.md +++ b/recipes/nemotron-3.5-lightning/vllm/README.md @@ -58,18 +58,12 @@ more GPUs. populated with `../model-cache/model-download.yaml`. 3. A Hugging Face secret named `hf-token-secret` and an image-pull secret when required by the cluster. -4. For H100 disaggregated recipes, AWS EFA userspace must already be available - in the runtime image or installed through your cluster's approved bootstrap - flow. These manifests request `vpc.amazonaws.com/efa` and select the NIXL - LIBFABRIC backend, but they do not install EFA packages during pod startup. - Use `check-efa-userspace.sh` to validate a prepared image or running pod. If - you use its install mode, set `EFA_INSTALLER_VERSION` and - `EFA_INSTALLER_SHA256` to pin and verify the installer archive. - -```bash -kubectl exec -i -n "${NAMESPACE}" -- \ - bash -s -- check < check-efa-userspace.sh -``` +4. For disaggregated recipes that use AWS EFA, EFA userspace must be prepared by + the user before deployment. These manifests request `vpc.amazonaws.com/efa` + and select the NIXL LIBFABRIC backend, but they do not install AWS EFA + packages during pod startup because non-AWS clusters should not run + AWS-specific setup. See [EFA setup](efa.md) for validation and + deployment-time setup options. ## Quick start @@ -125,6 +119,7 @@ ConfigMap key. - B200 DFlash recipes disable the FlashInfer FP8 ScaledMM linear kernel. - Disaggregated recipes require matching speculative-decoding settings on the prefill and decode workers for compatible NIXL cache metadata. -- H100 disaggregated recipes require the EFA userspace stack before deployment. +- AWS EFA disaggregated recipes require the EFA userspace stack before + deployment. - Replace `your-image-pull-secret` and any cluster-specific networking resource names with values available in the target cluster. diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-b200-dflash/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-b200-dflash/deploy.yaml index ffd80c6ba5e5..f67441b8334c 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-b200-dflash/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-b200-dflash/deploy.yaml @@ -134,8 +134,6 @@ spec: - --enable-prefix-caching - --async-scheduling - --no-enable-flashinfer-autotune - - --quantization - - modelopt_fp4 - --mamba-backend - flashinfer - --mamba-ssm-cache-dtype diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-kv-router/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-kv-router/deploy.yaml index 8c515ca2fb63..9770abfa62e0 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-kv-router/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-kv-router/deploy.yaml @@ -141,8 +141,6 @@ spec: - "64" - --async-scheduling - --no-enable-flashinfer-autotune - - --quantization - - modelopt_fp4 - --mamba-backend - flashinfer - --mamba-ssm-cache-dtype diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark/deploy.yaml index 5ecb60cb6fc1..85258b374f96 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark/deploy.yaml @@ -134,8 +134,6 @@ spec: - --enable-prefix-caching - --async-scheduling - --no-enable-flashinfer-autotune - - --quantization - - modelopt_fp4 - --mamba-backend - flashinfer - --mamba-ssm-cache-dtype diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-b200-mtp/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-b200-mtp/deploy.yaml index d2a077502cec..7a33d86816b7 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-b200-mtp/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-b200-mtp/deploy.yaml @@ -130,8 +130,6 @@ spec: - --enable-prefix-caching - --async-scheduling - --no-enable-flashinfer-autotune - - --quantization - - modelopt_fp4 - --mamba-backend - flashinfer - --mamba-ssm-cache-dtype diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-gb200-dflash/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-gb200-dflash/deploy.yaml index cbe57d09a02f..bd8a4cac8e1c 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-gb200-dflash/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-gb200-dflash/deploy.yaml @@ -159,8 +159,6 @@ spec: - "10240" - --enable-prefix-caching - --async-scheduling - - --quantization - - modelopt_fp4 - --mamba-backend - flashinfer - --mamba-ssm-cache-dtype diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-gb200-dspark/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-gb200-dspark/deploy.yaml index fba41bd748c4..4d399a25dba3 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-gb200-dspark/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-gb200-dspark/deploy.yaml @@ -159,8 +159,6 @@ spec: - "10240" - --enable-prefix-caching - --async-scheduling - - --quantization - - modelopt_fp4 - --mamba-backend - flashinfer - --mamba-ssm-cache-dtype diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-gb200-mtp/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-gb200-mtp/deploy.yaml index 4626b54763a9..adfeee814650 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-gb200-mtp/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-gb200-mtp/deploy.yaml @@ -155,8 +155,6 @@ spec: - "10240" - --enable-prefix-caching - --async-scheduling - - --quantization - - modelopt_fp4 - --mamba-backend - flashinfer - --mamba-ssm-cache-dtype diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-h100-dflash/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-h100-dflash/deploy.yaml index e9f4e38679ea..479dd630980b 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-h100-dflash/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-h100-dflash/deploy.yaml @@ -132,8 +132,6 @@ spec: - "32768" - --enable-prefix-caching - --async-scheduling - - --quantization - - modelopt_fp4 - --dyn-tool-call-parser - nemotron_nano - --dyn-reasoning-parser diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-h100-dspark-kv-router/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-h100-dspark-kv-router/deploy.yaml index 589568e789ee..24f8a7c50009 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-h100-dspark-kv-router/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-h100-dspark-kv-router/deploy.yaml @@ -140,8 +140,6 @@ spec: - --block-size - "64" - --async-scheduling - - --quantization - - modelopt_fp4 - --mamba-backend - flashinfer - --mamba-ssm-cache-dtype diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-h100-dspark/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-h100-dspark/deploy.yaml index c342926e2dcf..78142ffe39cb 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-h100-dspark/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-h100-dspark/deploy.yaml @@ -133,8 +133,6 @@ spec: - "32768" - --enable-prefix-caching - --async-scheduling - - --quantization - - modelopt_fp4 - --mamba-backend - flashinfer - --mamba-ssm-cache-dtype diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-h100-mtp/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-h100-mtp/deploy.yaml index 55d14c044520..d30c2c7c4938 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-h100-mtp/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-h100-mtp/deploy.yaml @@ -130,8 +130,6 @@ spec: - "32768" - --enable-prefix-caching - --async-scheduling - - --quantization - - modelopt_fp4 - --dyn-tool-call-parser - nemotron_nano - --dyn-reasoning-parser diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-h200-dflash/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-h200-dflash/deploy.yaml index 1140cec84799..aa5a359b1c26 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-h200-dflash/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-h200-dflash/deploy.yaml @@ -132,8 +132,6 @@ spec: - "32768" - --enable-prefix-caching - --async-scheduling - - --quantization - - modelopt_fp4 - --dyn-tool-call-parser - nemotron_nano - --dyn-reasoning-parser diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-h200-dspark-kv-router/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-h200-dspark-kv-router/deploy.yaml index ebe636121a0f..075a6377a01c 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-h200-dspark-kv-router/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-h200-dspark-kv-router/deploy.yaml @@ -140,8 +140,6 @@ spec: - --block-size - "64" - --async-scheduling - - --quantization - - modelopt_fp4 - --mamba-backend - flashinfer - --mamba-ssm-cache-dtype diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-h200-dspark/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-h200-dspark/deploy.yaml index 3bb031bcd509..3f704ca02e1a 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-h200-dspark/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-h200-dspark/deploy.yaml @@ -133,8 +133,6 @@ spec: - "32768" - --enable-prefix-caching - --async-scheduling - - --quantization - - modelopt_fp4 - --mamba-backend - flashinfer - --mamba-ssm-cache-dtype diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-h200-mtp/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-h200-mtp/deploy.yaml index 8ecefdc2cb9e..277188128ce7 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-h200-mtp/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-h200-mtp/deploy.yaml @@ -130,8 +130,6 @@ spec: - "32768" - --enable-prefix-caching - --async-scheduling - - --quantization - - modelopt_fp4 - --dyn-tool-call-parser - nemotron_nano - --dyn-reasoning-parser diff --git a/recipes/nemotron-3.5-lightning/vllm/disagg-b200-dspark/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/disagg-b200-dspark/deploy.yaml index 9d90e53a7213..7f5b1f5cd2fe 100644 --- a/recipes/nemotron-3.5-lightning/vllm/disagg-b200-dspark/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/disagg-b200-dspark/deploy.yaml @@ -129,8 +129,6 @@ spec: - '32' - --async-scheduling - --no-enable-flashinfer-autotune - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align @@ -148,7 +146,6 @@ spec: - nemotron_nano - --reasoning-parser - nemotron_v3 - - --speculative-config=$(SPECULATIVE_CONFIG) - --kv-transfer-config - '{"kv_connector":"NixlConnector","kv_role":"kv_producer"}' - --disaggregation-mode @@ -170,11 +167,6 @@ spec: value: /tmp/vllm-config - name: VLLM_CACHE_ROOT value: /tmp/vllm-cache - - name: SPECULATIVE_CONFIG - valueFrom: - configMapKeyRef: - name: vllm-disagg-b200-dspark-config - key: speculative-config # speculative-config-synthetic for benchmarking - name: VLLM_DISABLED_KERNELS value: FlashInferFP8ScaledMMLinearKernel - name: NCCL_IB_DISABLE @@ -268,8 +260,6 @@ spec: - '32' - --async-scheduling - --no-enable-flashinfer-autotune - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align diff --git a/recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dflash/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dflash/deploy.yaml index 5e26356f1cb4..cba84430501b 100644 --- a/recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dflash/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dflash/deploy.yaml @@ -146,8 +146,6 @@ spec: - --max-num-seqs - '32' - --async-scheduling - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align @@ -296,8 +294,6 @@ spec: - --max-num-seqs - '32' - --async-scheduling - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align diff --git a/recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dspark/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dspark/deploy.yaml index 609773e8b3d3..7e1c8a78376f 100644 --- a/recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dspark/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dspark/deploy.yaml @@ -148,8 +148,6 @@ spec: - --max-num-seqs - '32' - --async-scheduling - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align @@ -298,8 +296,6 @@ spec: - --max-num-seqs - '32' - --async-scheduling - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align diff --git a/recipes/nemotron-3.5-lightning/vllm/disagg-h100-dflash/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/disagg-h100-dflash/deploy.yaml index 17979658d6be..3cb2649fb7f9 100644 --- a/recipes/nemotron-3.5-lightning/vllm/disagg-h100-dflash/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/disagg-h100-dflash/deploy.yaml @@ -128,8 +128,6 @@ spec: - --max-num-batched-tokens - '32768' - --async-scheduling - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align @@ -279,8 +277,6 @@ spec: - --max-num-batched-tokens - '32768' - --async-scheduling - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align diff --git a/recipes/nemotron-3.5-lightning/vllm/disagg-h100-dspark/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/disagg-h100-dspark/deploy.yaml index e2fc09cc19ef..ca1225500f1a 100644 --- a/recipes/nemotron-3.5-lightning/vllm/disagg-h100-dspark/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/disagg-h100-dspark/deploy.yaml @@ -130,8 +130,6 @@ spec: - --max-num-batched-tokens - '32768' - --async-scheduling - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align @@ -281,8 +279,6 @@ spec: - --max-num-batched-tokens - '32768' - --async-scheduling - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align diff --git a/recipes/nemotron-3.5-lightning/vllm/disagg-h100-mtp/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/disagg-h100-mtp/deploy.yaml index 1b5e32a75dcf..b625583c214c 100644 --- a/recipes/nemotron-3.5-lightning/vllm/disagg-h100-mtp/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/disagg-h100-mtp/deploy.yaml @@ -126,8 +126,6 @@ spec: - --max-num-batched-tokens - '32768' - --async-scheduling - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align @@ -277,8 +275,6 @@ spec: - --max-num-batched-tokens - '32768' - --async-scheduling - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align diff --git a/recipes/nemotron-3.5-lightning/vllm/disagg-h200-dflash/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/disagg-h200-dflash/deploy.yaml index 0c4262dc8159..bda762e75666 100644 --- a/recipes/nemotron-3.5-lightning/vllm/disagg-h200-dflash/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/disagg-h200-dflash/deploy.yaml @@ -127,8 +127,6 @@ spec: - '32' - --async-scheduling - --no-enable-flashinfer-autotune - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align @@ -264,8 +262,6 @@ spec: - '32' - --async-scheduling - --no-enable-flashinfer-autotune - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align diff --git a/recipes/nemotron-3.5-lightning/vllm/disagg-h200-dspark/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/disagg-h200-dspark/deploy.yaml index 1d96b29b6bcc..936c26d046eb 100644 --- a/recipes/nemotron-3.5-lightning/vllm/disagg-h200-dspark/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/disagg-h200-dspark/deploy.yaml @@ -129,8 +129,6 @@ spec: - '32' - --async-scheduling - --no-enable-flashinfer-autotune - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align @@ -266,8 +264,6 @@ spec: - '32' - --async-scheduling - --no-enable-flashinfer-autotune - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align diff --git a/recipes/nemotron-3.5-lightning/vllm/disagg-h200-mtp/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/disagg-h200-mtp/deploy.yaml index c71a8ff70cce..9868aa39603c 100644 --- a/recipes/nemotron-3.5-lightning/vllm/disagg-h200-mtp/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/disagg-h200-mtp/deploy.yaml @@ -125,8 +125,6 @@ spec: - '32' - --async-scheduling - --no-enable-flashinfer-autotune - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align @@ -262,8 +260,6 @@ spec: - '32' - --async-scheduling - --no-enable-flashinfer-autotune - - --quantization - - modelopt_fp4 - --enable-prefix-caching - --mamba-cache-mode - align diff --git a/recipes/nemotron-3.5-lightning/vllm/efa.md b/recipes/nemotron-3.5-lightning/vllm/efa.md new file mode 100644 index 000000000000..2c434978f9c9 --- /dev/null +++ b/recipes/nemotron-3.5-lightning/vllm/efa.md @@ -0,0 +1,101 @@ + + +# EFA setup + +AWS EFA-backed disaggregated vLLM recipes use NIXL LIBFABRIC for prefill/decode +transport. The manifests request `vpc.amazonaws.com/efa`, but they do not +install AWS EFA packages during pod startup because non-AWS clusters should not +run AWS-specific setup. + +Prepare the complete matching EFA userspace stack before deployment, either in +the runtime image, through cluster bootstrap, or through a local deployment +overlay. Validate the final runtime from a pod that requests EFA resources: + +```bash +kubectl exec -i -n "${NAMESPACE}" -- \ + bash -s -- check < check-efa-userspace.sh +``` + +## Install helper + +On AWS, `check-efa-userspace.sh install` can be used while preparing a derived +runtime image, cluster bootstrap layer, or local deployment overlay. Pin the +installer version and checksum: + +```bash +EFA_INSTALLER_VERSION=1.49.0 \ +EFA_INSTALLER_SHA256=cf2e9281a2328a243c76f911a490faed43ca0fecfe4733c25e34b2e92a32c309 \ + bash check-efa-userspace.sh install +``` + +If EFA is installed under `/opt/amazon/efa` but is not registered with +`ldconfig`, add these environment variables to the disaggregated worker +components before applying the manifest: + +```yaml +- name: PATH + value: /opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +- name: LD_LIBRARY_PATH + value: /opt/amazon/efa/lib:/opt/amazon/efa/lib64:/usr/lib/x86_64-linux-gnu +``` + +## Deployment-time wrapper + +For deployment-time setup, use a wrapper entrypoint on the worker containers. +Run the install in the actual worker container before +`python3 -m dynamo.vllm`, so the installed `/opt/amazon/efa` and system +libraries are visible to vLLM: + +```bash +#!/usr/bin/env bash +set -euo pipefail + +export DEBIAN_FRONTEND=noninteractive +if [ ! -x /opt/amazon/efa/bin/fi_info ]; then + apt-get update + apt-get install -y --no-install-recommends ca-certificates curl tar gzip + EFA_INSTALLER_VERSION=1.49.0 \ + EFA_INSTALLER_SHA256=cf2e9281a2328a243c76f911a490faed43ca0fecfe4733c25e34b2e92a32c309 \ + bash /path/to/check-efa-userspace.sh install +fi + +export PATH="/opt/amazon/efa/bin:${PATH}" +export LD_LIBRARY_PATH="/opt/amazon/efa/lib:/opt/amazon/efa/lib64:/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" +fi_info -p efa | head -80 + +exec "$@" +``` + +Mount the wrapper into both prefill and decode workers and move the original +`python3 -m dynamo.vllm ...` command into `args`: + +```yaml +command: +- /opt/dynamo/efa/efa-entrypoint.sh +args: +- python3 +- -m +- dynamo.vllm +# Keep the existing recipe arguments here. +volumeMounts: +- name: efa-entrypoint + mountPath: /opt/dynamo/efa + readOnly: true +volumes: +- name: efa-entrypoint + configMap: + name: + defaultMode: 0555 + items: + - key: efa-entrypoint.sh + path: efa-entrypoint.sh +``` + +Do not rely on a one-off init container that installs EFA into an `emptyDir` +mounted only at `/opt/amazon/efa`: the installer also installs ABI-matched +libraries and provider configuration under system paths such as +`/usr/lib/x86_64-linux-gnu`, and the final `fi_info -p efa` validation must run +inside a container that has been allocated EFA devices. From 5cdd2fb4231280372e2f18a532e057df01c43440 Mon Sep 17 00:00:00 2001 From: Fadi Saady Date: Mon, 17 Aug 2026 17:08:40 +0200 Subject: [PATCH 2/2] feat(recipes): add Nemotron Lightning BF16 recipes. - For B200 and GB200, BF16 gets better perf on the tested agentic workload compared to nvfp4 (w4a16). - Moving these recipes to use BF16 for now. - Updated fern docs. Signed-off-by: Fadi Saady --- docs/fern/index.yml | 2 +- .../recipes/nemotron-3-5-lightning.yaml | 435 ++++++++---------- .../model-recipes/nemotron-3-5-lightning.mdx | 194 +++++--- .../pages/recipes/model-recipes/overview.mdx | 6 +- recipes/nemotron-3.5-lightning/README.md | 51 +- .../model-cache/model-cache.yaml | 2 +- .../model-cache/model-download-bf16.yaml | 64 +++ recipes/nemotron-3.5-lightning/perf/README.md | 79 ++-- .../nemotron-3.5-lightning/trtllm/README.md | 27 +- .../{agg-b200 => agg-b200-bf16}/deploy.yaml | 14 +- .../deploy.yaml | 16 +- .../{agg-gb200 => agg-gb200-bf16}/deploy.yaml | 14 +- .../deploy.yaml | 16 +- recipes/nemotron-3.5-lightning/vllm/README.md | 85 +--- .../vllm/agg-b200-dflash/deploy.yaml | 204 -------- .../deploy.yaml | 18 +- .../deploy.yaml | 18 +- .../deploy.yaml | 14 +- .../vllm/agg-gb200-dflash/deploy.yaml | 239 ---------- .../deploy.yaml | 18 +- .../deploy.yaml | 14 +- .../deploy.yaml | 30 +- .../deploy.yaml | 22 +- .../deploy.yaml | 22 +- .../vllm/disagg-h100-mtp/deploy.yaml | 374 --------------- .../vllm/disagg-h200-mtp/deploy.yaml | 346 -------------- 26 files changed, 587 insertions(+), 1737 deletions(-) create mode 100644 recipes/nemotron-3.5-lightning/model-cache/model-download-bf16.yaml rename recipes/nemotron-3.5-lightning/trtllm/{agg-b200 => agg-b200-bf16}/deploy.yaml (94%) rename recipes/nemotron-3.5-lightning/trtllm/{agg-b200-mtp => agg-b200-mtp-bf16}/deploy.yaml (94%) rename recipes/nemotron-3.5-lightning/trtllm/{agg-gb200 => agg-gb200-bf16}/deploy.yaml (94%) rename recipes/nemotron-3.5-lightning/trtllm/{agg-gb200-mtp => agg-gb200-mtp-bf16}/deploy.yaml (94%) delete mode 100644 recipes/nemotron-3.5-lightning/vllm/agg-b200-dflash/deploy.yaml rename recipes/nemotron-3.5-lightning/vllm/{agg-b200-dspark => agg-b200-dspark-bf16}/deploy.yaml (91%) rename recipes/nemotron-3.5-lightning/vllm/{agg-b200-dspark-kv-router => agg-b200-dspark-kv-router-bf16}/deploy.yaml (91%) rename recipes/nemotron-3.5-lightning/vllm/{agg-b200-mtp => agg-b200-mtp-bf16}/deploy.yaml (93%) delete mode 100644 recipes/nemotron-3.5-lightning/vllm/agg-gb200-dflash/deploy.yaml rename recipes/nemotron-3.5-lightning/vllm/{agg-gb200-dspark => agg-gb200-dspark-bf16}/deploy.yaml (92%) rename recipes/nemotron-3.5-lightning/vllm/{agg-gb200-mtp => agg-gb200-mtp-bf16}/deploy.yaml (94%) rename recipes/nemotron-3.5-lightning/vllm/{disagg-b200-dspark => disagg-b200-dspark-bf16}/deploy.yaml (91%) rename recipes/nemotron-3.5-lightning/vllm/{disagg-gb200-dflash => disagg-gb200-dflash-bf16}/deploy.yaml (94%) rename recipes/nemotron-3.5-lightning/vllm/{disagg-gb200-dspark => disagg-gb200-dspark-bf16}/deploy.yaml (94%) delete mode 100644 recipes/nemotron-3.5-lightning/vllm/disagg-h100-mtp/deploy.yaml delete mode 100644 recipes/nemotron-3.5-lightning/vllm/disagg-h200-mtp/deploy.yaml diff --git a/docs/fern/index.yml b/docs/fern/index.yml index ce5d9e9ec71d..6dc5760a9ef7 100644 --- a/docs/fern/index.yml +++ b/docs/fern/index.yml @@ -462,7 +462,7 @@ navigation: - page: Kimi-K2.6 path: pages/recipes/model-recipes/kimi-k2-6.mdx slug: kimi-k2-6 - - page: Nemotron-3.5-Lightning + - page: Nemotron 3.5 Lightning path: pages/recipes/model-recipes/nemotron-3-5-lightning.mdx slug: nemotron-3-5-lightning - page: Nemotron 3 Ultra diff --git a/docs/fern/pages/recipes/_catalog/recipes/nemotron-3-5-lightning.yaml b/docs/fern/pages/recipes/_catalog/recipes/nemotron-3-5-lightning.yaml index 75baf5b57e0c..c900e92058fe 100644 --- a/docs/fern/pages/recipes/_catalog/recipes/nemotron-3-5-lightning.yaml +++ b/docs/fern/pages/recipes/_catalog/recipes/nemotron-3-5-lightning.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # id: nemotron-3-5-lightning -title: Nemotron-3.5-Lightning +title: Nemotron 3.5 Lightning page: pages/recipes/model-recipes/nemotron-3-5-lightning.mdx provider: nvidia model: @@ -11,10 +11,10 @@ model: precision: NVFP4 status: experimental targets: -- id: vllm-agg-b200-dspark +- id: vllm-agg-h100-dspark recommended: true hardware: - gpu: B200 + gpu: H100 count: 1 runtime: framework: vLLM @@ -23,20 +23,19 @@ targets: - speculative-decoding - dspark - prefix-caching - - kv-aware-routing workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/agg-h100-dspark/deploy.yaml expected_performance: available: true - summary: 2471.12 output tok/s/GPU, 102.96 tok/s/user, TTFT p50 207.44ms @ concurrency 24 -- id: vllm-agg-b200-dspark-kv-router + summary: 1878.11 output tok/s/GPU, 93.91 tok/s/user, TTFT p50 238.64ms @ concurrency 20 +- id: vllm-agg-h100-dspark-kv-router recommended: true hardware: - gpu: B200 + gpu: H100 count: 4 runtime: framework: vLLM @@ -51,14 +50,14 @@ targets: traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-kv-router/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/agg-h100-dspark-kv-router/deploy.yaml expected_performance: available: true - summary: 10030.79 total output tok/s, 2507.70 output tok/s/GPU, 104.49 tok/s/user, TTFT p50 295.03ms @ concurrency 96 -- id: vllm-agg-b200-dflash + summary: 8149.70 total output tok/s, 2037.43 output tok/s/GPU, 101.87 tok/s/user, TTFT p50 303.84ms @ concurrency 80 +- id: vllm-agg-h100-dflash recommended: true hardware: - gpu: B200 + gpu: H100 count: 1 runtime: framework: vLLM @@ -67,20 +66,19 @@ targets: - speculative-decoding - dflash - prefix-caching - - kv-aware-routing workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/agg-b200-dflash/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/agg-h100-dflash/deploy.yaml expected_performance: available: true - summary: 2004.06 output tok/s/GPU, 100.20 tok/s/user, TTFT p50 212.57ms @ concurrency 20 -- id: vllm-agg-b200-mtp + summary: 1757.75 output tok/s/GPU, 109.86 tok/s/user, TTFT p50 216.44ms @ concurrency 16 +- id: vllm-agg-h100-mtp recommended: true hardware: - gpu: B200 + gpu: H100 count: 1 runtime: framework: vLLM @@ -89,20 +87,19 @@ targets: - speculative-decoding - mtp - prefix-caching - - kv-aware-routing workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/agg-b200-mtp/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/agg-h100-mtp/deploy.yaml expected_performance: available: true - summary: 1829.94 output tok/s/GPU, 101.66 tok/s/user, TTFT p50 217.12ms @ concurrency 18 -- id: vllm-agg-gb200-dspark + summary: 1275.37 output tok/s/GPU, 106.28 tok/s/user, TTFT p50 241.21ms @ concurrency 12 +- id: vllm-agg-h200-dspark recommended: true hardware: - gpu: GB200 + gpu: H200 count: 1 runtime: framework: vLLM @@ -111,27 +108,26 @@ targets: - speculative-decoding - dspark - prefix-caching - - kv-aware-routing workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/agg-gb200-dspark/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/agg-h200-dspark/deploy.yaml expected_performance: available: true - summary: 2860.20 output tok/s/GPU, 119.18 tok/s/user, TTFT p50 296.82ms @ concurrency 24 -- id: vllm-agg-gb200-dflash - recommended: false + summary: 2193.59 output tok/s/GPU, 109.68 tok/s/user, TTFT p50 243.97ms @ concurrency 20 +- id: vllm-agg-h200-dspark-kv-router + recommended: true hardware: - gpu: GB200 - count: 1 + gpu: H200 + count: 4 runtime: framework: vLLM topology: aggregated techniques: - speculative-decoding - - dflash + - dspark - prefix-caching - kv-aware-routing workload: @@ -139,408 +135,372 @@ targets: traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/agg-gb200-dflash/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/agg-h200-dspark-kv-router/deploy.yaml expected_performance: available: true - summary: 842.42 output tok/s/GPU, 42.12 tok/s/user, TTFT p50 429.97ms @ concurrency 20; failed tok/s/user SLA -- id: vllm-agg-gb200-mtp + summary: 8683.99 total output tok/s, 2171.00 output tok/s/GPU, 108.55 tok/s/user, TTFT p50 286.67ms @ concurrency 80 +- id: vllm-agg-h200-dflash recommended: true hardware: - gpu: GB200 + gpu: H200 count: 1 runtime: framework: vLLM topology: aggregated techniques: - speculative-decoding - - mtp + - dflash - prefix-caching - - kv-aware-routing workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/agg-gb200-mtp/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/agg-h200-dflash/deploy.yaml expected_performance: available: true - summary: 1758.22 output tok/s/GPU, 97.68 tok/s/user, TTFT p50 402.99ms @ concurrency 18 -- id: vllm-agg-h100-dspark + summary: 1744.04 output tok/s/GPU, 109.00 tok/s/user, TTFT p50 243.30ms @ concurrency 16 +- id: vllm-agg-h200-mtp recommended: true hardware: - gpu: H100 + gpu: H200 count: 1 runtime: framework: vLLM topology: aggregated techniques: - speculative-decoding - - dspark + - mtp - prefix-caching - - kv-aware-routing workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/agg-h100-dspark/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/agg-h200-mtp/deploy.yaml expected_performance: available: true - summary: 1878.11 output tok/s/GPU, 93.91 tok/s/user, TTFT p50 238.64ms @ concurrency 20 -- id: vllm-agg-h100-dspark-kv-router + summary: 1478.67 output tok/s/GPU, 123.22 tok/s/user, TTFT p50 250.66ms @ concurrency 12 +- id: vllm-disagg-h100-dflash recommended: true hardware: gpu: H100 - count: 4 + count: 2 runtime: framework: vLLM - topology: aggregated + topology: disaggregated techniques: - speculative-decoding - - dspark + - dflash - prefix-caching - - kv-aware-routing + - nixl workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/agg-h100-dspark-kv-router/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/disagg-h100-dflash/deploy.yaml expected_performance: available: true - summary: 8149.70 total output tok/s, 2037.43 output tok/s/GPU, 101.87 tok/s/user, TTFT p50 303.84ms @ concurrency 80 -- id: vllm-agg-h100-dflash + summary: 1117.29 output tok/s/GPU, 186.21 tok/s/user, TTFT p50 542.71ms @ concurrency 12 +- id: vllm-disagg-h100-dspark recommended: true hardware: gpu: H100 - count: 1 + count: 2 runtime: framework: vLLM - topology: aggregated + topology: disaggregated techniques: - speculative-decoding - - dflash + - dspark - prefix-caching - - kv-aware-routing + - nixl workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/agg-h100-dflash/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/disagg-h100-dspark/deploy.yaml expected_performance: available: true - summary: 1757.75 output tok/s/GPU, 109.86 tok/s/user, TTFT p50 216.44ms @ concurrency 16 -- id: vllm-agg-h100-mtp + summary: 1445.29 output tok/s/GPU, 120.44 tok/s/user, TTFT p50 2028.23ms @ concurrency 24 +- id: vllm-disagg-h200-dflash recommended: true hardware: - gpu: H100 - count: 1 + gpu: H200 + count: 2 runtime: framework: vLLM - topology: aggregated + topology: disaggregated techniques: - speculative-decoding - - mtp + - dflash - prefix-caching - - kv-aware-routing + - nixl workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/agg-h100-mtp/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/disagg-h200-dflash/deploy.yaml expected_performance: available: true - summary: 1275.37 output tok/s/GPU, 106.28 tok/s/user, TTFT p50 241.21ms @ concurrency 12 -- id: vllm-agg-h200-dspark + summary: 1277.86 output tok/s/GPU, 212.98 tok/s/user, TTFT p50 408.30ms @ concurrency 12 +- id: vllm-disagg-h200-dspark recommended: true hardware: gpu: H200 - count: 1 + count: 2 runtime: framework: vLLM - topology: aggregated + topology: disaggregated techniques: - speculative-decoding - dspark - prefix-caching - - kv-aware-routing + - nixl workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/agg-h200-dspark/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/disagg-h200-dspark/deploy.yaml expected_performance: available: true - summary: 2193.59 output tok/s/GPU, 109.68 tok/s/user, TTFT p50 243.97ms @ concurrency 20 -- id: vllm-agg-h200-dspark-kv-router - recommended: true + summary: 1656.90 output tok/s/GPU, 138.08 tok/s/user, TTFT p50 834.80ms @ concurrency 24 +- id: trtllm-agg-h100-mtp + recommended: false hardware: - gpu: H200 - count: 4 + gpu: H100 + count: 1 runtime: - framework: vLLM + framework: TensorRT-LLM topology: aggregated techniques: + - tensorrt-llm - speculative-decoding - - dspark - - prefix-caching - - kv-aware-routing + - mtp workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/agg-h200-dspark-kv-router/deploy.yaml + asset: recipes/nemotron-3.5-lightning/trtllm/agg-h100-mtp/deploy.yaml expected_performance: available: true - summary: 8683.99 total output tok/s, 2171.00 output tok/s/GPU, 108.55 tok/s/user, TTFT p50 286.67ms @ concurrency 80 -- id: vllm-agg-h200-dflash - recommended: true + summary: 264.87 output tok/s/GPU, 22.07 tok/s/user, TTFT p50 1757.00ms @ concurrency 12; experimental smoke result +- id: trtllm-agg-h200-mtp + recommended: false hardware: gpu: H200 count: 1 runtime: - framework: vLLM + framework: TensorRT-LLM topology: aggregated techniques: + - tensorrt-llm - speculative-decoding - - dflash - - prefix-caching - - kv-aware-routing + - mtp workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/agg-h200-dflash/deploy.yaml + asset: recipes/nemotron-3.5-lightning/trtllm/agg-h200-mtp/deploy.yaml expected_performance: available: true - summary: 1744.04 output tok/s/GPU, 109.00 tok/s/user, TTFT p50 243.30ms @ concurrency 16 -- id: vllm-agg-h200-mtp + summary: 250.05 output tok/s/GPU, 20.84 tok/s/user, TTFT p50 1692.98ms @ concurrency 12 + +- id: vllm-agg-b200-dspark-bf16 recommended: true + precision: BF16 hardware: - gpu: H200 + gpu: B200 count: 1 runtime: framework: vLLM topology: aggregated techniques: - speculative-decoding - - mtp - - prefix-caching - - kv-aware-routing + - dspark + - bf16 workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/agg-h200-mtp/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-bf16/deploy.yaml expected_performance: available: true - summary: 1478.67 output tok/s/GPU, 123.22 tok/s/user, TTFT p50 250.66ms @ concurrency 12 -- id: vllm-disagg-b200-dspark + summary: 4005.48 output tok/s/GPU, 80.11 tok/s/user, TTFT p50 221.49ms @ concurrency 50 + +- id: vllm-agg-b200-dspark-kv-router-bf16 recommended: true + precision: BF16 hardware: gpu: B200 - count: 2 + count: 4 runtime: framework: vLLM - topology: disaggregated + topology: aggregated techniques: - speculative-decoding - dspark - - prefix-caching - - nixl + - kv-aware-routing + - bf16 workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/disagg-b200-dspark/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-kv-router-bf16/deploy.yaml expected_performance: available: true - summary: 904.03 output tok/s/GPU, 226.01 tok/s/user, TTFT p50 236.43ms @ concurrency 8 -- id: vllm-disagg-gb200-dflash + summary: 16648.35 total output tok/s, 4162.09 output tok/s/GPU, 86.71 tok/s/user, TTFT p50 254.26ms @ concurrency 192 + +- id: vllm-agg-b200-mtp-bf16 recommended: true + precision: BF16 hardware: - gpu: GB200 - count: 2 + gpu: B200 + count: 1 runtime: framework: vLLM - topology: disaggregated + topology: aggregated techniques: - speculative-decoding - - dflash - - prefix-caching - - nixl + - mtp + - bf16 workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dflash/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/agg-b200-mtp-bf16/deploy.yaml expected_performance: available: true - summary: 584.85 output tok/s/GPU, 292.43 tok/s/user, TTFT p50 491.40ms @ concurrency 4 -- id: vllm-disagg-gb200-dspark + summary: 2454.72 output tok/s/GPU, 136.37 tok/s/user, TTFT p50 165.43ms @ concurrency 18 + + +- id: vllm-agg-gb200-dspark-bf16 recommended: true + precision: BF16 hardware: gpu: GB200 - count: 2 + count: 1 runtime: framework: vLLM - topology: disaggregated + topology: aggregated techniques: - speculative-decoding - dspark - - prefix-caching - - nixl + - bf16 workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dspark/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/agg-gb200-dspark-bf16/deploy.yaml expected_performance: available: true - summary: 939.28 output tok/s/GPU, 234.82 tok/s/user, TTFT p50 599.17ms @ concurrency 8 -- id: vllm-disagg-h100-dflash + summary: 4327.90 output tok/s/GPU, 74.62 tok/s/user, TTFT p50 307.73ms @ concurrency 58 + +- id: vllm-agg-gb200-mtp-bf16 recommended: true + precision: BF16 hardware: - gpu: H100 - count: 2 + gpu: GB200 + count: 1 runtime: framework: vLLM - topology: disaggregated + topology: aggregated techniques: - speculative-decoding - - dflash - - prefix-caching - - nixl + - mtp + - bf16 workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/disagg-h100-dflash/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/agg-gb200-mtp-bf16/deploy.yaml expected_performance: available: true - summary: 1117.29 output tok/s/GPU, 186.21 tok/s/user, TTFT p50 542.71ms @ concurrency 12 -- id: vllm-disagg-h100-dspark + summary: 2289.75 output tok/s/GPU, 127.21 tok/s/user, TTFT p50 228.77ms @ concurrency 18 + +- id: vllm-disagg-b200-dspark-bf16 recommended: true + precision: BF16 hardware: - gpu: H100 + gpu: B200 count: 2 runtime: framework: vLLM topology: disaggregated techniques: - - speculative-decoding + - disaggregated-prefill-decode - dspark - - prefix-caching - - nixl - workload: - type: agentic - traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse - sla: TTFT p50 < 5000ms and >= 50 tok/s/user - deploy: - asset: recipes/nemotron-3.5-lightning/vllm/disagg-h100-dspark/deploy.yaml - expected_performance: - available: true - summary: 1445.29 output tok/s/GPU, 120.44 tok/s/user, TTFT p50 2028.23ms @ concurrency 24 -- id: vllm-disagg-h100-mtp - recommended: true - hardware: - gpu: H100 - count: 2 - runtime: - framework: vLLM - topology: disaggregated - techniques: - - speculative-decoding - - mtp - - prefix-caching - - nixl + - bf16 workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/disagg-h100-mtp/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/disagg-b200-dspark-bf16/deploy.yaml expected_performance: available: true - summary: 1055.68 output tok/s/GPU, 175.95 tok/s/user, TTFT p50 493.52ms @ concurrency 12 -- id: vllm-disagg-h200-dflash - recommended: true + summary: 3711.37 output tok/s/GPU, 92.78 tok/s/user, TTFT p50 1059.12ms @ concurrency 80 + +- id: vllm-disagg-gb200-dflash-bf16 + recommended: false + precision: BF16 hardware: - gpu: H200 + gpu: GB200 count: 2 runtime: framework: vLLM topology: disaggregated techniques: - - speculative-decoding + - disaggregated-prefill-decode - dflash - - prefix-caching - - nixl + - bf16 workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/disagg-h200-dflash/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dflash-bf16/deploy.yaml expected_performance: available: true - summary: 1277.86 output tok/s/GPU, 212.98 tok/s/user, TTFT p50 408.30ms @ concurrency 12 -- id: vllm-disagg-h200-dspark + summary: 1148.33 output tok/s/GPU, 95.69 tok/s/user, TTFT p50 10641.10ms @ concurrency 24; failed TTFT SLA + +- id: vllm-disagg-gb200-dspark-bf16 recommended: true + precision: BF16 hardware: - gpu: H200 + gpu: GB200 count: 2 runtime: framework: vLLM topology: disaggregated techniques: - - speculative-decoding + - disaggregated-prefill-decode - dspark - - prefix-caching - - nixl + - bf16 workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/vllm/disagg-h200-dspark/deploy.yaml + asset: recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dspark-bf16/deploy.yaml expected_performance: available: true - summary: 1656.90 output tok/s/GPU, 138.08 tok/s/user, TTFT p50 834.80ms @ concurrency 24 -- id: vllm-disagg-h200-mtp + summary: 802.30 output tok/s/GPU, 200.58 tok/s/user, TTFT p50 569.70ms @ concurrency 8 + +- id: trtllm-agg-b200-bf16 recommended: true - hardware: - gpu: H200 - count: 2 - runtime: - framework: vLLM - topology: disaggregated - techniques: - - speculative-decoding - - mtp - - prefix-caching - - nixl - workload: - type: agentic - traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse - sla: TTFT p50 < 5000ms and >= 50 tok/s/user - deploy: - asset: recipes/nemotron-3.5-lightning/vllm/disagg-h200-mtp/deploy.yaml - expected_performance: - available: true - summary: 1203.97 output tok/s/GPU, 200.66 tok/s/user, TTFT p50 424.66ms @ concurrency 12 -- id: trtllm-agg-b200 - recommended: false + precision: BF16 hardware: gpu: B200 count: 1 @@ -549,17 +509,20 @@ targets: topology: aggregated techniques: - tensorrt-llm + - bf16 workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/trtllm/agg-b200/deploy.yaml + asset: recipes/nemotron-3.5-lightning/trtllm/agg-b200-bf16/deploy.yaml expected_performance: available: true - summary: 386.998 output tok/s/GPU, 21.50 tok/s/user, TTFT p50 377.68ms @ concurrency 18; experimental smoke result -- id: trtllm-agg-b200-mtp - recommended: false + summary: 1869.54 output tok/s/GPU, 103.86 tok/s/user, TTFT p50 219.18ms @ concurrency 18; experimental trace result + +- id: trtllm-agg-b200-mtp-bf16 + recommended: true + precision: BF16 hardware: gpu: B200 count: 1 @@ -570,17 +533,20 @@ targets: - tensorrt-llm - speculative-decoding - mtp + - bf16 workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/trtllm/agg-b200-mtp/deploy.yaml + asset: recipes/nemotron-3.5-lightning/trtllm/agg-b200-mtp-bf16/deploy.yaml expected_performance: available: true - summary: 427.82 output tok/s/GPU, 23.77 tok/s/user, TTFT p50 446.88ms @ concurrency 18; experimental smoke result -- id: trtllm-agg-gb200 - recommended: false + summary: 3295.50 output tok/s/GPU, 183.08 tok/s/user, TTFT p50 244.95ms @ concurrency 18; experimental trace result + +- id: trtllm-agg-gb200-bf16 + recommended: true + precision: BF16 hardware: gpu: GB200 count: 1 @@ -589,17 +555,20 @@ targets: topology: aggregated techniques: - tensorrt-llm + - bf16 workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/trtllm/agg-gb200/deploy.yaml + asset: recipes/nemotron-3.5-lightning/trtllm/agg-gb200-bf16/deploy.yaml expected_performance: available: true - summary: 280.10 output tok/s/GPU, 15.56 tok/s/user, TTFT p50 318.88ms @ concurrency 18; experimental smoke result -- id: trtllm-agg-gb200-mtp - recommended: false + summary: 1985.84 output tok/s/GPU, 110.32 tok/s/user, TTFT p50 216.77ms @ concurrency 18; experimental trace result + +- id: trtllm-agg-gb200-mtp-bf16 + recommended: true + precision: BF16 hardware: gpu: GB200 count: 1 @@ -610,61 +579,21 @@ targets: - tensorrt-llm - speculative-decoding - mtp + - bf16 workload: type: agentic traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse sla: TTFT p50 < 5000ms and >= 50 tok/s/user deploy: - asset: recipes/nemotron-3.5-lightning/trtllm/agg-gb200-mtp/deploy.yaml - expected_performance: - available: true - summary: 580.09 output tok/s/GPU, 32.23 tok/s/user, TTFT p50 400.88ms @ concurrency 18; experimental smoke result -- id: trtllm-agg-h100-mtp - recommended: false - hardware: - gpu: H100 - count: 1 - runtime: - framework: TensorRT-LLM - topology: aggregated - techniques: - - tensorrt-llm - - speculative-decoding - - mtp - workload: - type: agentic - traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse - sla: TTFT p50 < 5000ms and >= 50 tok/s/user - deploy: - asset: recipes/nemotron-3.5-lightning/trtllm/agg-h100-mtp/deploy.yaml - expected_performance: - available: true - summary: 264.87 output tok/s/GPU, 22.07 tok/s/user, TTFT p50 1757.00ms @ concurrency 12; experimental smoke result -- id: trtllm-agg-h200-mtp - recommended: false - hardware: - gpu: H200 - count: 1 - runtime: - framework: TensorRT-LLM - topology: aggregated - techniques: - - tensorrt-llm - - speculative-decoding - - mtp - workload: - type: agentic - traffic: 64K/400 Mooncake-format agentic trace, 90% KV reuse - sla: TTFT p50 < 5000ms and >= 50 tok/s/user - deploy: - asset: recipes/nemotron-3.5-lightning/trtllm/agg-h200-mtp/deploy.yaml + asset: recipes/nemotron-3.5-lightning/trtllm/agg-gb200-mtp-bf16/deploy.yaml expected_performance: available: true - summary: 269.26 output tok/s/GPU, 22.44 tok/s/user, TTFT p50 1630.96ms @ concurrency 12; experimental smoke result + summary: 3556.40 output tok/s/GPU, 197.58 tok/s/user, TTFT p50 231.34ms @ concurrency 18; experimental trace result + related_benchmarks: [] maintainer: null gaps: - no related Feature Benchmark page exists for this model - no checked-in benchmark Job YAML exists; results are documented in recipes/nemotron-3.5-lightning/perf/README.md -- TensorRT-LLM and GB200 paths are experimental or review-stage in the source README +- TensorRT-LLM paths are experimental or review-stage in the source README - maintainer unknown diff --git a/docs/fern/pages/recipes/model-recipes/nemotron-3-5-lightning.mdx b/docs/fern/pages/recipes/model-recipes/nemotron-3-5-lightning.mdx index 56dfe5605172..70ba1e7caae7 100644 --- a/docs/fern/pages/recipes/model-recipes/nemotron-3-5-lightning.mdx +++ b/docs/fern/pages/recipes/model-recipes/nemotron-3-5-lightning.mdx @@ -1,15 +1,15 @@ --- # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -title: "Nemotron-3.5-Lightning" -subtitle: "Serve NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 with Dynamo, vLLM, and experimental TensorRT-LLM targets." +title: "Nemotron 3.5 Lightning" +subtitle: "Serve NVIDIA-Nemotron-3.5-Lightning-30B-A3B in NVFP4 or BF16 with Dynamo, vLLM, and experimental TensorRT-LLM targets." --- import { RecipeStyles } from "@/components/RecipeStyles"; -This recipe deploys `nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4`, a 30B hybrid Mamba/Attention/MoE model with about 3B active parameters per token. The vLLM targets cover H100, H200, B200, and GB200 with MTP, DFlash, or DSpark speculative decoding. The benchmark tables use a Mooncake-format agentic trace with 64K input tokens, 400 output tokens, and 90% KV reuse. +This recipe deploys the NVFP4 or BF16 variant of `NVIDIA-Nemotron-3.5-Lightning-30B-A3B`, a 30B hybrid Mamba/Attention/MoE model with about 3B active parameters per token. The vLLM targets cover H100, H200, B200, and GB200 with MTP, DFlash, or DSpark speculative decoding. The benchmark tables use a Mooncake-format agentic trace with 64K input tokens, 400 output tokens, and 90% KV reuse.

Choose your deployment target

@@ -34,31 +34,36 @@ This recipe deploys `nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4`, a 30B
-Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 +Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 +Precision BF16 GPUs 1x B200, or 4x B200 with KV routing Runtime vLLM aggregate worker -Spec decode DSpark, DFlash, or MTP +Spec decode DSpark or MTP
Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 +Precision NVFP4 (W4A16) GPUs 1x H100, or 4x H100 with KV routing Runtime vLLM aggregate worker Spec decode DSpark, DFlash, or MTP
Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 +Precision NVFP4 (W4A16) GPUs 1x H200, or 4x H200 with KV routing Runtime vLLM aggregate worker Spec decode DSpark, DFlash, or MTP
-Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 +Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 +Precision BF16 GPUs 1x GB200 Runtime vLLM, aggregate worker -Spec decode DSpark, DFlash, or MTP +Spec decode DSpark or MTP
-Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 +Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 +Precision BF16 GPUs 2x B200 Runtime vLLM, 1 prefill + 1 decode Transport IB/RDMA with UCX @@ -66,27 +71,31 @@ This recipe deploys `nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4`, a 30B
Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 +Precision NVFP4 (W4A16) GPUs 2x H100 Runtime vLLM, 1 prefill + 1 decode Transport AWS EFA/LIBFABRIC -Spec decode MTP, DFlash, or DSpark +Spec decode DFlash or DSpark
Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 +Precision NVFP4 (W4A16) GPUs 2x H200 Runtime vLLM, 1 prefill + 1 decode Transport IB/RDMA with UCX -Spec decode MTP, DFlash, or DSpark +Spec decode DFlash or DSpark
-Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 +Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 +Precision BF16 GPUs 2x GB200 Runtime vLLM, 1 prefill + 1 decode Transport UCX/NIXL with cluster-specific fabric resources Spec decode DFlash or DSpark
-Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 +Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 +Precision BF16 GPUs 1x B200 Runtime TensorRT-LLM, aggregate worker Spec decode MTP or no-spec fallback @@ -94,6 +103,7 @@ This recipe deploys `nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4`, a 30B
Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 +Precision NVFP4 (W4A16) GPUs 1x H100 Runtime TensorRT-LLM, aggregate worker Spec decode MTP @@ -101,13 +111,15 @@ This recipe deploys `nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4`, a 30B
Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 +Precision NVFP4 GPUs 1x H200 Runtime TensorRT-LLM, aggregate worker Spec decode MTP Status Experimental
-Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 +Checkpoint nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 +Precision BF16 GPUs 1x GB200 Runtime TensorRT-LLM, aggregate worker Spec decode MTP or no-spec fallback @@ -157,7 +169,9 @@ Edit storage class, image pull secret names, node selectors, fabric resources, a ## Deploy -Create the shared model cache and download the base and draft checkpoints. This step is common to every target: +Create the shared model cache, then download the checkpoints for the selected GPU family: + +
```bash # Edit storageClassName in model-cache/model-cache.yaml first (kubectl get storageclass). @@ -166,16 +180,29 @@ kubectl apply -f recipes/nemotron-3.5-lightning/model-cache/model-download.yaml kubectl wait --for=condition=Complete job/model-download -n ${NAMESPACE} --timeout=3600s ``` +
+ +
+ +```bash +# Edit storageClassName in model-cache/model-cache.yaml first (kubectl get storageclass). +kubectl apply -f recipes/nemotron-3.5-lightning/model-cache/model-cache.yaml -n ${NAMESPACE} +kubectl apply -f recipes/nemotron-3.5-lightning/model-cache/model-download-bf16.yaml -n ${NAMESPACE} +kubectl wait --for=condition=Complete job/model-download-bf16 -n ${NAMESPACE} --timeout=3600s +``` + +
+ Then deploy the selected target.
```bash -kubectl apply -f recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark/deploy.yaml -n ${NAMESPACE} -kubectl get dgd vllm-agg-b200-dspark -n ${NAMESPACE} -w +kubectl apply -f recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-bf16/deploy.yaml -n ${NAMESPACE} +kubectl get dgd vllm-agg-b200-dspark-bf16 -n ${NAMESPACE} -w ``` -DSpark is the promoted single-worker B200 aggregate path from the checked-in validation table. Use `agg-b200-dflash`, `agg-b200-mtp`, or `agg-b200-dspark-kv-router` to compare the other aggregate options. For this validation trace, KV-aware routing did not materially change the result because KV-cache pressure was low; it can help workloads with enough reusable KV state to pressure a single worker. +DSpark is the promoted single-worker B200 aggregate path from the checked-in validation table. Use `agg-b200-mtp-bf16` or `agg-b200-dspark-kv-router-bf16` to compare the other aggregate options. For this validation trace, KV-aware routing did not materially change the result because KV-cache pressure was low; it can help workloads with enough reusable KV state to pressure a single worker.
@@ -204,19 +231,19 @@ DSpark is the highest-throughput H200 single-worker aggregate row in the checked
```bash -kubectl apply -f recipes/nemotron-3.5-lightning/vllm/agg-gb200-dspark/deploy.yaml -n ${NAMESPACE} -kubectl get dgd vllm-agg-gb200-dspark -n ${NAMESPACE} -w +kubectl apply -f recipes/nemotron-3.5-lightning/vllm/agg-gb200-dspark-bf16/deploy.yaml -n ${NAMESPACE} +kubectl get dgd vllm-agg-gb200-dspark-bf16 -n ${NAMESPACE} -w ``` -DSpark is the highest-throughput GB200 aggregate row in the checked-in validation table. Use `agg-gb200-mtp` for the other passing aggregate row; `agg-gb200-dflash` missed the tok/s/user objective in the source results. +DSpark is the highest-throughput GB200 aggregate row in the checked-in validation table. Use `agg-gb200-mtp-bf16` for the other passing aggregate row.
```bash -kubectl apply -f recipes/nemotron-3.5-lightning/vllm/disagg-b200-dspark/deploy.yaml -n ${NAMESPACE} -kubectl get dgd vllm-disagg-b200-dspark -n ${NAMESPACE} -w +kubectl apply -f recipes/nemotron-3.5-lightning/vllm/disagg-b200-dspark-bf16/deploy.yaml -n ${NAMESPACE} +kubectl get dgd vllm-disagg-b200-dspark-bf16 -n ${NAMESPACE} -w ``` Use this target for B200 disaggregated vLLM validation with DSpark speculative decoding. @@ -230,7 +257,7 @@ kubectl apply -f recipes/nemotron-3.5-lightning/vllm/disagg-h100-dspark/deploy.y kubectl get dgd vllm-disagg-h100-dspark -n ${NAMESPACE} -w ``` -DSpark is the highest-output-throughput H100 disaggregated row in the source inventory. Use `disagg-h100-dflash` or `disagg-h100-mtp` to compare the other speculative decoding modes. +DSpark is the highest-output-throughput H100 disaggregated row in the source inventory. Use `disagg-h100-dflash` to compare the other speculative decoding mode.
@@ -241,29 +268,29 @@ kubectl apply -f recipes/nemotron-3.5-lightning/vllm/disagg-h200-dspark/deploy.y kubectl get dgd vllm-disagg-h200-dspark -n ${NAMESPACE} -w ``` -DSpark is the highest-throughput H200 disaggregated row in the source inventory. Use `disagg-h200-dflash` or `disagg-h200-mtp` to compare the other speculative decoding modes. +DSpark is the highest-throughput H200 disaggregated row in the source inventory. Use `disagg-h200-dflash` to compare the other speculative decoding mode.
```bash -kubectl apply -f recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dspark/deploy.yaml -n ${NAMESPACE} -kubectl get dgd vllm-disagg-gb200-dspark -n ${NAMESPACE} -w +kubectl apply -f recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dspark-bf16/deploy.yaml -n ${NAMESPACE} +kubectl get dgd vllm-disagg-gb200-dspark-bf16 -n ${NAMESPACE} -w ``` -DSpark is the highest-throughput GB200 disaggregated row in the source inventory. Use `disagg-gb200-dflash` for the lower-throughput passing row. +The delivered GB200 disaggregated recipes are DSpark and DFlash. The command above deploys DSpark; use `disagg-gb200-dflash-bf16` for the DFlash recipe.
```bash -kubectl apply -f recipes/nemotron-3.5-lightning/trtllm/agg-b200-mtp/deploy.yaml -n ${NAMESPACE} -kubectl get dgd trtllm-agg-b200-mtp -n ${NAMESPACE} -w +kubectl apply -f recipes/nemotron-3.5-lightning/trtllm/agg-b200-mtp-bf16/deploy.yaml -n ${NAMESPACE} +kubectl get dgd trtllm-agg-b200-mtp-bf16 -n ${NAMESPACE} -w ``` -Use `trtllm/agg-b200/deploy.yaml` for the no-spec fallback. +Use `trtllm/agg-b200-bf16/deploy.yaml` for the no-spec fallback.
@@ -292,11 +319,11 @@ The H200 TensorRT-LLM target uses the MTP manifest.
```bash -kubectl apply -f recipes/nemotron-3.5-lightning/trtllm/agg-gb200-mtp/deploy.yaml -n ${NAMESPACE} -kubectl get dgd trtllm-agg-gb200-mtp -n ${NAMESPACE} -w +kubectl apply -f recipes/nemotron-3.5-lightning/trtllm/agg-gb200-mtp-bf16/deploy.yaml -n ${NAMESPACE} +kubectl get dgd trtllm-agg-gb200-mtp-bf16 -n ${NAMESPACE} -w ``` -Use `trtllm/agg-gb200/deploy.yaml` for the no-spec fallback. +Use `trtllm/agg-gb200-bf16/deploy.yaml` for the no-spec fallback.
@@ -307,7 +334,7 @@ Port-forward the frontend service for the deployment you applied:
```bash -kubectl port-forward svc/vllm-agg-b200-dspark-frontend 8000:8000 -n ${NAMESPACE} +kubectl port-forward svc/vllm-agg-b200-dspark-bf16-frontend 8000:8000 -n ${NAMESPACE} ```
@@ -331,7 +358,7 @@ kubectl port-forward svc/vllm-agg-h200-dspark-frontend 8000:8000 -n ${NAMESPACE}
```bash -kubectl port-forward svc/vllm-agg-gb200-dspark-frontend 8000:8000 -n ${NAMESPACE} +kubectl port-forward svc/vllm-agg-gb200-dspark-bf16-frontend 8000:8000 -n ${NAMESPACE} ```
@@ -339,7 +366,7 @@ kubectl port-forward svc/vllm-agg-gb200-dspark-frontend 8000:8000 -n ${NAMESPACE
```bash -kubectl port-forward svc/vllm-disagg-b200-dspark-frontend 8000:8000 -n ${NAMESPACE} +kubectl port-forward svc/vllm-disagg-b200-dspark-bf16-frontend 8000:8000 -n ${NAMESPACE} ```
@@ -363,7 +390,7 @@ kubectl port-forward svc/vllm-disagg-h200-dspark-frontend 8000:8000 -n ${NAMESPA
```bash -kubectl port-forward svc/vllm-disagg-gb200-dspark-frontend 8000:8000 -n ${NAMESPACE} +kubectl port-forward svc/vllm-disagg-gb200-dspark-bf16-frontend 8000:8000 -n ${NAMESPACE} ```
@@ -371,7 +398,7 @@ kubectl port-forward svc/vllm-disagg-gb200-dspark-frontend 8000:8000 -n ${NAMESP
```bash -kubectl port-forward svc/trtllm-agg-b200-mtp-frontend 8000:8000 -n ${NAMESPACE} +kubectl port-forward svc/trtllm-agg-b200-mtp-bf16-frontend 8000:8000 -n ${NAMESPACE} ```
@@ -395,13 +422,31 @@ kubectl port-forward svc/trtllm-agg-h200-mtp-frontend 8000:8000 -n ${NAMESPACE}
```bash -kubectl port-forward svc/trtllm-agg-gb200-mtp-frontend 8000:8000 -n ${NAMESPACE} +kubectl port-forward svc/trtllm-agg-gb200-mtp-bf16-frontend 8000:8000 -n ${NAMESPACE} ```
Then send a chat completion request: +
+ +```bash +curl http://localhost:8000/v1/models +curl http://localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 32, + "temperature": 0 + }' +``` + +
+ +
+ ```bash curl http://localhost:8000/v1/models curl http://localhost:8000/v1/chat/completions \ @@ -414,6 +459,8 @@ curl http://localhost:8000/v1/chat/completions \ }' ``` +
+ ## Benchmark The checked-in recipe includes benchmark results in [`recipes/nemotron-3.5-lightning/perf/README.md`](https://github.com/ai-dynamo/dynamo/blob/main/recipes/nemotron-3.5-lightning/perf/README.md). The pass criteria are `tok/s/user >= 50` and TTFT p50 under 5 seconds. @@ -424,7 +471,7 @@ To reproduce the shape, run AIPerf against the selected frontend with a Mooncake ```bash aiperf profile \ - -m nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 \ + -m nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 \ --custom-dataset-type mooncake_trace \ --num-requests 3541 \ --endpoint-type chat \ @@ -432,8 +479,8 @@ aiperf profile \ --use-server-token-count \ --extra-inputs ignore_eos:true \ --request-timeout-seconds 1200 \ - --url http://vllm-agg-b200-dspark-frontend:8000 \ - --concurrency 24 + --url http://vllm-agg-b200-dspark-bf16-frontend:8000 \ + --concurrency 50 ```
@@ -478,7 +525,7 @@ aiperf profile \ ```bash aiperf profile \ - -m nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 \ + -m nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 \ --custom-dataset-type mooncake_trace \ --num-requests 3541 \ --endpoint-type chat \ @@ -486,8 +533,8 @@ aiperf profile \ --use-server-token-count \ --extra-inputs ignore_eos:true \ --request-timeout-seconds 1200 \ - --url http://vllm-agg-gb200-dspark-frontend:8000 \ - --concurrency 24 + --url http://vllm-agg-gb200-dspark-bf16-frontend:8000 \ + --concurrency 58 ``` @@ -496,7 +543,7 @@ aiperf profile \ ```bash aiperf profile \ - -m nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 \ + -m nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 \ --custom-dataset-type mooncake_trace \ --num-requests 3541 \ --endpoint-type chat \ @@ -504,8 +551,8 @@ aiperf profile \ --use-server-token-count \ --extra-inputs ignore_eos:true \ --request-timeout-seconds 1200 \ - --url http://vllm-disagg-b200-dspark-frontend:8000 \ - --concurrency 8 + --url http://vllm-disagg-b200-dspark-bf16-frontend:8000 \ + --concurrency 80 ``` @@ -550,7 +597,7 @@ aiperf profile \ ```bash aiperf profile \ - -m nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 \ + -m nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 \ --custom-dataset-type mooncake_trace \ --num-requests 3541 \ --endpoint-type chat \ @@ -558,7 +605,7 @@ aiperf profile \ --use-server-token-count \ --extra-inputs ignore_eos:true \ --request-timeout-seconds 1200 \ - --url http://vllm-disagg-gb200-dspark-frontend:8000 \ + --url http://vllm-disagg-gb200-dspark-bf16-frontend:8000 \ --concurrency 8 ``` @@ -578,10 +625,9 @@ TensorRT-LLM rows are experimental smoke or inventory results in the source READ RecipeSpec decodeRoutingConcurrencyTTFT p50 (ms)Output tok/s/GPUTok/s/user -vllm/agg-b200-dspark-kv-router/deploy.yamlDSparkKV router96295.032507.70104.49 -vllm/agg-b200-dspark/deploy.yamlDSparkSingle worker24207.442471.12102.96 -vllm/agg-b200-dflash/deploy.yamlDFlashSingle worker20212.572004.06100.20 -vllm/agg-b200-mtp/deploy.yamlMTPSingle worker18217.121829.94101.66 +vllm/agg-b200-dspark-kv-router-bf16/deploy.yamlDSparkKV router192254.264162.0986.71 +vllm/agg-b200-dspark-bf16/deploy.yamlDSparkSingle worker50221.494005.4880.11 +vllm/agg-b200-mtp-bf16/deploy.yamlMTPSingle worker18165.432454.72136.37 vllm/agg-h100-dspark-kv-router/deploy.yamlDSparkKV router80303.842037.43101.87 vllm/agg-h100-dspark/deploy.yamlDSparkSingle worker20238.641878.1193.91 vllm/agg-h100-dflash/deploy.yamlDFlashSingle worker16216.441757.75109.86 @@ -590,24 +636,21 @@ TensorRT-LLM rows are experimental smoke or inventory results in the source READ vllm/agg-h200-dspark/deploy.yamlDSparkSingle worker20243.972193.59109.68 vllm/agg-h200-dflash/deploy.yamlDFlashSingle worker16243.301744.04109.00 vllm/agg-h200-mtp/deploy.yamlMTPSingle worker12250.661478.67123.22 -vllm/agg-gb200-dspark/deploy.yamlDSparkSingle worker24296.822860.20119.18 -vllm/agg-gb200-mtp/deploy.yamlMTPSingle worker18402.991758.2297.68 -vllm/agg-gb200-dflash/deploy.yamlDFlashSingle worker20429.97842.4242.12 -vllm/disagg-b200-dspark/deploy.yamlDSpark1P1D8236.43904.03226.01 +vllm/agg-gb200-dspark-bf16/deploy.yamlDSparkSingle worker58307.734327.9074.62 +vllm/agg-gb200-mtp-bf16/deploy.yamlMTPSingle worker18228.772289.75127.21 +vllm/disagg-b200-dspark-bf16/deploy.yamlDSpark1P1D801059.123711.3792.78 vllm/disagg-h100-dflash/deploy.yamlDFlash1P1D12542.711117.29186.21 vllm/disagg-h100-dspark/deploy.yamlDSpark1P1D242028.231445.29120.44 -vllm/disagg-h100-mtp/deploy.yamlMTP1P1D12493.521055.68175.95 vllm/disagg-h200-dflash/deploy.yamlDFlash1P1D12408.301277.86212.98 vllm/disagg-h200-dspark/deploy.yamlDSpark1P1D24834.801656.90138.08 -vllm/disagg-h200-mtp/deploy.yamlMTP1P1D12424.661203.97200.66 -vllm/disagg-gb200-dflash/deploy.yamlDFlash1P1D4491.40584.85292.43 -vllm/disagg-gb200-dspark/deploy.yamlDSpark1P1D8599.17939.28234.82 -trtllm/agg-b200/deploy.yamlNoneSingle worker18377.68386.99821.50 -trtllm/agg-b200-mtp/deploy.yamlMTPSingle worker18446.88427.8223.77 +vllm/disagg-gb200-dflash-bf16/deploy.yamlDFlash1P1D2410641.101148.3395.69 +vllm/disagg-gb200-dspark-bf16/deploy.yamlDSpark1P1D8569.70802.30200.58 +trtllm/agg-b200-bf16/deploy.yamlNoneSingle worker18219.181869.54103.86 +trtllm/agg-b200-mtp-bf16/deploy.yamlMTPSingle worker18244.953295.50183.08 trtllm/agg-h100-mtp/deploy.yamlMTPSingle worker121757.00264.8722.07 -trtllm/agg-h200-mtp/deploy.yamlMTPSingle worker121630.96269.2622.44 -trtllm/agg-gb200/deploy.yamlNoneSingle worker18318.88280.1015.56 -trtllm/agg-gb200-mtp/deploy.yamlMTPSingle worker18400.88580.0932.23 +trtllm/agg-h200-mtp/deploy.yamlMTPSingle worker121692.98250.0520.84 +trtllm/agg-gb200-bf16/deploy.yamlNoneSingle worker18216.771985.84110.32 +trtllm/agg-gb200-mtp-bf16/deploy.yamlMTPSingle worker18231.343556.40197.58 @@ -620,18 +663,18 @@ TensorRT-LLM rows are experimental smoke or inventory results in the source READ RuntimeTopologyGPUSpec decode optionsChecked-in manifests -vLLMAggregatedB200MTP, DFlash, DSparkvllm/agg-b200-dspark-kv-router/deploy.yaml (DSpark + KV router)
vllm/agg-b200-dspark/deploy.yaml
vllm/agg-b200-dflash/deploy.yaml
vllm/agg-b200-mtp/deploy.yaml +vLLMAggregatedB200MTP, DSparkvllm/agg-b200-dspark-kv-router-bf16/deploy.yaml (DSpark + KV router)
vllm/agg-b200-dspark-bf16/deploy.yaml
vllm/agg-b200-mtp-bf16/deploy.yaml vLLMAggregatedH100MTP, DFlash, DSparkvllm/agg-h100-dspark-kv-router/deploy.yaml (DSpark + KV router)
vllm/agg-h100-dspark/deploy.yaml
vllm/agg-h100-dflash/deploy.yaml
vllm/agg-h100-mtp/deploy.yaml vLLMAggregatedH200MTP, DFlash, DSparkvllm/agg-h200-dspark-kv-router/deploy.yaml (DSpark + KV router)
vllm/agg-h200-dspark/deploy.yaml
vllm/agg-h200-dflash/deploy.yaml
vllm/agg-h200-mtp/deploy.yaml -vLLMAggregatedGB200MTP, DFlash, DSparkvllm/agg-gb200-dspark/deploy.yaml
vllm/agg-gb200-dflash/deploy.yaml
vllm/agg-gb200-mtp/deploy.yaml -vLLMDisaggregated 1P1DB200DSparkvllm/disagg-b200-dspark/deploy.yaml -vLLMDisaggregated 1P1DH100MTP, DFlash, DSparkvllm/disagg-h100-dspark/deploy.yaml
vllm/disagg-h100-dflash/deploy.yaml
vllm/disagg-h100-mtp/deploy.yaml -vLLMDisaggregated 1P1DH200MTP, DFlash, DSparkvllm/disagg-h200-dspark/deploy.yaml
vllm/disagg-h200-dflash/deploy.yaml
vllm/disagg-h200-mtp/deploy.yaml -vLLMDisaggregated 1P1DGB200DFlash, DSparkvllm/disagg-gb200-dspark/deploy.yaml
vllm/disagg-gb200-dflash/deploy.yaml -TensorRT-LLMAggregatedB200MTP, no-spec fallbacktrtllm/agg-b200-mtp/deploy.yaml
trtllm/agg-b200/deploy.yaml +vLLMAggregatedGB200MTP, DSparkvllm/agg-gb200-dspark-bf16/deploy.yaml
vllm/agg-gb200-mtp-bf16/deploy.yaml +vLLMDisaggregated 1P1DB200DSparkvllm/disagg-b200-dspark-bf16/deploy.yaml +vLLMDisaggregated 1P1DH100DFlash, DSparkvllm/disagg-h100-dspark/deploy.yaml
vllm/disagg-h100-dflash/deploy.yaml +vLLMDisaggregated 1P1DH200DFlash, DSparkvllm/disagg-h200-dspark/deploy.yaml
vllm/disagg-h200-dflash/deploy.yaml +vLLMDisaggregated 1P1DGB200DFlash, DSparkvllm/disagg-gb200-dflash-bf16/deploy.yaml
vllm/disagg-gb200-dspark-bf16/deploy.yaml +TensorRT-LLMAggregatedB200MTP, no-spec fallbacktrtllm/agg-b200-mtp-bf16/deploy.yaml
trtllm/agg-b200-bf16/deploy.yaml TensorRT-LLMAggregatedH100MTPtrtllm/agg-h100-mtp/deploy.yaml TensorRT-LLMAggregatedH200MTPtrtllm/agg-h200-mtp/deploy.yaml -TensorRT-LLMAggregatedGB200MTP, no-spec fallbacktrtllm/agg-gb200-mtp/deploy.yaml
trtllm/agg-gb200/deploy.yaml +TensorRT-LLMAggregatedGB200MTP, no-spec fallbacktrtllm/agg-gb200-mtp-bf16/deploy.yaml
trtllm/agg-gb200-bf16/deploy.yaml @@ -662,3 +705,4 @@ TensorRT-LLM rows are experimental smoke or inventory results in the source READ - TensorRT-LLM README: [recipes/nemotron-3.5-lightning/trtllm/README.md](https://github.com/ai-dynamo/dynamo/blob/main/recipes/nemotron-3.5-lightning/trtllm/README.md) - Performance notes: [recipes/nemotron-3.5-lightning/perf/README.md](https://github.com/ai-dynamo/dynamo/blob/main/recipes/nemotron-3.5-lightning/perf/README.md) - Model cache manifests: [model-cache](https://github.com/ai-dynamo/dynamo/tree/main/recipes/nemotron-3.5-lightning/model-cache) +- BF16 model download: [model-download-bf16.yaml](https://github.com/ai-dynamo/dynamo/blob/main/recipes/nemotron-3.5-lightning/model-cache/model-download-bf16.yaml) diff --git a/docs/fern/pages/recipes/model-recipes/overview.mdx b/docs/fern/pages/recipes/model-recipes/overview.mdx index 8c07495fd3f4..1ad0c2e408e2 100644 --- a/docs/fern/pages/recipes/model-recipes/overview.mdx +++ b/docs/fern/pages/recipes/model-recipes/overview.mdx @@ -108,11 +108,11 @@ Start with the model, runtime, and hardware you need to run. Model cards are sor
-

Nemotron-3.5-Lightning

NVIDIA

+

Nemotron 3.5 Lightning

NVIDIA

-

30B hybrid Mamba/Attention/MoE recipe matrix for H100, H200, B200, and GB200, with vLLM MTP/DFlash/DSpark targets, DSpark KV-routing aggregate variants on H100/H200/B200, and experimental TensorRT-LLM MTP/no-spec aggregates.

+

30B hybrid Mamba/Attention/MoE recipe matrix for H100, H200, B200, and GB200, with NVFP4 and BF16 variants, vLLM MTP/DFlash/DSpark targets, DSpark with KV-routing, and experimental TensorRT-LLM MTP/no-spec.

vLLM + TRT-LLM1-4x GPUAgentic · 1M context
- Open recipe + Open recipe
diff --git a/recipes/nemotron-3.5-lightning/README.md b/recipes/nemotron-3.5-lightning/README.md index b41cccacd738..346665e5d37f 100644 --- a/recipes/nemotron-3.5-lightning/README.md +++ b/recipes/nemotron-3.5-lightning/README.md @@ -5,20 +5,21 @@ SPDX-License-Identifier: Apache-2.0 # Nemotron 3.5 Lightning Recipes -Kubernetes recipes for serving -`nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4` with Dynamo. The repository -contains vLLM recipes for aggregate and disaggregated deployments, plus aggregate +Kubernetes recipes for serving the NVFP4 and BF16 variants of +`NVIDIA-Nemotron-3.5-Lightning-30B-A3B` with Dynamo. The repository contains +vLLM recipes for aggregate and disaggregated deployments, plus aggregate TensorRT-LLM recipes. ## Configurations | Backend | Hardware | Deployment modes | Speculative decoding | |---|---|---|---| -| vLLM | H100, H200, B200, GB200 | Aggregate and selected 1P/1D disaggregated | MTP, DFlash, DSpark | -| TensorRT-LLM | H100, H200, B200, GB200 | Aggregate | None, MTP | +| vLLM | H100, H200, B200, GB200 | Aggregate and selected 1P/1D disaggregated | MTP, DFlash, DSpark; NVFP4 on H100/H200 and BF16 on B200/GB200 | +| TensorRT-LLM | H100, H200, B200, GB200 | Aggregate | None, MTP; NVFP4 on H100/H200 and BF16 on B200/GB200 | -The vLLM recipes are organized as `vllm/{agg,disagg}-{hardware}-{specdec}`. -TensorRT-LLM recipes are organized as `trtllm/agg-{hardware}[-{specdec}]`. +The NVFP4 recipes are organized as `vllm/{agg,disagg}-{hardware}-{specdec}` and +`trtllm/agg-{hardware}[-{specdec}]`. B200 and GB200 recipes use BF16 weights +and include `-bf16` in the recipe directory and Kubernetes resource names. ## Repository layout @@ -50,34 +51,39 @@ Create the shared model cache and populate it with the target and draft models: export NAMESPACE=your-namespace # Replace your-storage-class-name in model-cache/model-cache.yaml first. kubectl apply -n "${NAMESPACE}" -f model-cache/model-cache.yaml -kubectl apply -n "${NAMESPACE}" -f model-cache/model-download.yaml -kubectl wait -n "${NAMESPACE}" --for=condition=Complete job/model-download --timeout=3600s +kubectl apply -n "${NAMESPACE}" -f model-cache/model-download-bf16.yaml +kubectl wait -n "${NAMESPACE}" --for=condition=Complete job/model-download-bf16 --timeout=3600s ``` Deploy a recipe: ```bash -RECIPE=vllm/agg-b200-dspark +RECIPE=vllm/agg-b200-dspark-bf16 kubectl apply -n "${NAMESPACE}" -f "${RECIPE}/deploy.yaml" -kubectl get dgd -n "${NAMESPACE}" vllm-agg-b200-dspark -w +kubectl get dgd -n "${NAMESPACE}" vllm-agg-b200-dspark-bf16 -w ``` Port-forward the frontend service and send a chat request: ```bash kubectl port-forward -n "${NAMESPACE}" \ - svc/vllm-agg-b200-dspark-frontend 8000:8000 + svc/vllm-agg-b200-dspark-bf16-frontend 8000:8000 ``` ```bash +MODEL=nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 +# Replace -BF16 with -NVFP4 for H100 or H200 recipes. + curl http://localhost:8000/v1/chat/completions \ -H 'Content-Type: application/json' \ - -d '{ - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", - "messages": [{"role": "user", "content": "Say OK."}], - "max_tokens": 128, - "temperature": 0 - }' + -d @- <= 50` and TTFT p50 < 5s -The trace symlink points to the shared trace under -`recipes/kimi-k2.6/perf/traces/`. Some trace requests exceed the model's native -262144-token context length. Those requests returned errors and are excluded -from valid-request latency and throughput metrics. - Rows use this AIPerf shape with the request count shown in the table: ```text @@ -33,35 +29,40 @@ aiperf profile --custom-dataset-type mooncake_trace --num-requests \ ## Trace Results -| Recipe | FW | Mode | GPU | Routing | Spec method | Spec tok | Synthetic AL | Concurrency | Requests | Valid requests | Errors | TTFT p50 (ms) | Output tok/s/GPU | Tok/s/user | -|--------|----|------|-----|---------|-------------|----------|--------------|-------------|----------|----------------|--------|---------------|------------------|------------| -| `vllm/agg-b200-dflash/deploy.yaml` | vLLM | agg | B200 | Single worker | DFlash | 5 | 3.18 | 20 | 3541 | 3411 | 130 | 212.57 | 2004.06 | 100.20 | -| `vllm/agg-b200-dspark/deploy.yaml` | vLLM | agg | B200 | Single worker | DSpark | 7 | 3.69 | 24 | 3541 | 3411 | 130 | 207.44 | 2471.12 | 102.96 | -| `vllm/agg-b200-dspark-kv-router/deploy.yaml` | vLLM | agg | B200 | KV router | DSpark | 7 | 3.69 | 96 | 3541 | 3411 | 130 | 295.03 | 2507.70 | 104.49 | -| `vllm/agg-b200-mtp/deploy.yaml` | vLLM | agg | B200 | Single worker | MTP | 3 | 2.874 | 18 | 3541 | 3411 | 130 | 217.12 | 1829.94 | 101.66 | -| `vllm/agg-gb200-dflash/deploy.yaml` | vLLM | agg | GB200 | Single worker | DFlash | 5 | 3.18 | 20 | 3541 | 3411 | 130 | 429.97 | 842.42 | 42.12 | -| `vllm/agg-gb200-dspark/deploy.yaml` | vLLM | agg | GB200 | Single worker | DSpark | 7 | 3.69 | 24 | 3541 | 3411 | 130 | 296.82 | 2860.20 | 119.18 | -| `vllm/agg-gb200-mtp/deploy.yaml` | vLLM | agg | GB200 | Single worker | MTP | 3 | 2.874 | 18 | 3541 | 3411 | 130 | 402.99 | 1758.22 | 97.68 | -| `vllm/agg-h100-dflash/deploy.yaml` | vLLM | agg | H100 | Single worker | DFlash | 5 | 3.18 | 16 | 3541 | 3411 | 130 | 216.44 | 1757.75 | 109.86 | -| `vllm/agg-h100-dspark/deploy.yaml` | vLLM | agg | H100 | Single worker | DSpark | 7 | 3.69 | 20 | 3541 | 3411 | 130 | 238.64 | 1878.11 | 93.91 | -| `vllm/agg-h100-dspark-kv-router/deploy.yaml` | vLLM | agg | H100 | KV router | DSpark | 7 | 3.69 | 80 | 3541 | 3411 | 130 | 303.84 | 2037.43 | 101.87 | -| `vllm/agg-h100-mtp/deploy.yaml` | vLLM | agg | H100 | Single worker | MTP | 7 | 3.687 | 12 | 3541 | 3411 | 130 | 241.21 | 1275.37 | 106.28 | -| `vllm/agg-h200-dflash/deploy.yaml` | vLLM | agg | H200 | Single worker | DFlash | 5 | 3.18 | 16 | 3541 | 3411 | 130 | 243.30 | 1744.04 | 109.00 | -| `vllm/agg-h200-dspark/deploy.yaml` | vLLM | agg | H200 | Single worker | DSpark | 7 | 3.69 | 20 | 3541 | 3411 | 130 | 243.97 | 2193.59 | 109.68 | -| `vllm/agg-h200-dspark-kv-router/deploy.yaml` | vLLM | agg | H200 | KV router | DSpark | 7 | 3.69 | 80 | 3541 | 3411 | 130 | 286.67 | 2171.00 | 108.55 | -| `vllm/agg-h200-mtp/deploy.yaml` | vLLM | agg | H200 | Single worker | MTP | 7 | 3.687 | 12 | 3541 | 3411 | 130 | 250.66 | 1478.67 | 123.22 | -| `vllm/disagg-b200-dspark/deploy.yaml` | vLLM | disagg | B200 | 1P1D | DSpark | 1 | 1.83 | 8 | 3541 | 3410 | 131 | 235.95 | 984.09 | 246.02 | -| `vllm/disagg-gb200-dflash/deploy.yaml` | vLLM | disagg | GB200 | 1P1D | DFlash | 7 | 3.41 | 4 | 3541 | 3411 | 130 | 491.40 | 584.85 | 292.43 | -| `vllm/disagg-gb200-dspark/deploy.yaml` | vLLM | disagg | GB200 | 1P1D | DSpark | 1 | 1.83 | 8 | 3541 | 3411 | 130 | 599.17 | 939.28 | 234.82 | -| `vllm/disagg-h100-dflash/deploy.yaml` | vLLM | disagg | H100 | 1P1D | DFlash | 3 | 2.73 | 12 | 3541 | 3404 | 137 | 542.71 | 1117.29 | 186.21 | -| `vllm/disagg-h100-dspark/deploy.yaml` | vLLM | disagg | H100 | 1P1D | DSpark | 1 | 1.83 | 24 | 3541 | 3411 | 130 | 2028.23 | 1445.29 | 120.44 | -| `vllm/disagg-h100-mtp/deploy.yaml` | vLLM | disagg | H100 | 1P1D | MTP | 5 | 3.421 | 12 | 3541 | 3398 | 143 | 493.52 | 1055.68 | 175.95 | -| `vllm/disagg-h200-dflash/deploy.yaml` | vLLM | disagg | H200 | 1P1D | DFlash | 3 | 2.73 | 12 | 3541 | 3401 | 140 | 408.30 | 1277.86 | 212.98 | -| `vllm/disagg-h200-dspark/deploy.yaml` | vLLM | disagg | H200 | 1P1D | DSpark | 1 | 1.83 | 24 | 3541 | 3411 | 130 | 834.80 | 1656.90 | 138.08 | -| `vllm/disagg-h200-mtp/deploy.yaml` | vLLM | disagg | H200 | 1P1D | MTP | 5 | 3.421 | 12 | 3541 | 3342 | 199 | 424.66 | 1203.97 | 200.66 | -| `trtllm/agg-b200/deploy.yaml` | TensorRT-LLM | agg | B200 | Single worker | None | 0 | 0.0 | 18 | 3541 | 3411 | 130 | 378.04 | 378.27 | 21.01 | -| `trtllm/agg-b200-mtp/deploy.yaml` | TensorRT-LLM | agg | B200 | Single worker | MTP | 3 | 2.874 | 18 | 3541 | 3411 | 130 | 448.71 | 490.28 | 27.24 | -| `trtllm/agg-gb200/deploy.yaml` | TensorRT-LLM | agg | GB200 | Single worker | None | 0 | 0.0 | 18 | 500 | 499 | 1 | 318.88 | 280.10 | 15.56 | -| `trtllm/agg-gb200-mtp/deploy.yaml` | TensorRT-LLM | agg | GB200 | Single worker | MTP | 3 | 2.874 | 18 | 500 | 499 | 1 | 400.88 | 580.09 | 32.23 | -| `trtllm/agg-h100-mtp/deploy.yaml` | TensorRT-LLM | agg | H100 | Single worker | MTP | 7 | 3.687 | 12 | 500 | 499 | 1 | 1757.00 | 264.87 | 22.07 | -| `trtllm/agg-h200-mtp/deploy.yaml` | TensorRT-LLM | agg | H200 | Single worker | MTP | 7 | 3.687 | 12 | 3541 | 3527 | 14 | 1692.98 | 250.05 | 20.84 | +### H100 and H200 + +| Precision | Recipe | FW | Mode | GPU | Routing | Spec method | Spec tok | Synthetic AL | Concurrency | Requests | Valid requests | Errors | TTFT p50 (ms) | Output tok/s/GPU | Tok/s/user | +|----------|--------|----|------|-----|---------|-------------|----------|--------------|-------------|----------|----------------|--------|---------------|------------------|------------| +| NVFP4 | `vllm/agg-h100-dflash/deploy.yaml` | vLLM | agg | H100 | Single worker | DFlash | 5 | 3.18 | 16 | 3541 | 3411 | 130 | 216.44 | 1757.75 | 109.86 | +| NVFP4 | `vllm/agg-h100-dspark/deploy.yaml` | vLLM | agg | H100 | Single worker | DSpark | 7 | 3.69 | 20 | 3541 | 3411 | 130 | 238.64 | 1878.11 | 93.91 | +| NVFP4 | `vllm/agg-h100-dspark-kv-router/deploy.yaml` | vLLM | agg | H100 | KV router | DSpark | 7 | 3.69 | 80 | 3541 | 3411 | 130 | 303.84 | 2037.43 | 101.87 | +| NVFP4 | `vllm/agg-h100-mtp/deploy.yaml` | vLLM | agg | H100 | Single worker | MTP | 7 | 3.687 | 12 | 3541 | 3411 | 130 | 241.21 | 1275.37 | 106.28 | +| NVFP4 | `vllm/agg-h200-dflash/deploy.yaml` | vLLM | agg | H200 | Single worker | DFlash | 5 | 3.18 | 16 | 3541 | 3411 | 130 | 243.30 | 1744.04 | 109.00 | +| NVFP4 | `vllm/agg-h200-dspark/deploy.yaml` | vLLM | agg | H200 | Single worker | DSpark | 7 | 3.69 | 20 | 3541 | 3411 | 130 | 243.97 | 2193.59 | 109.68 | +| NVFP4 | `vllm/agg-h200-dspark-kv-router/deploy.yaml` | vLLM | agg | H200 | KV router | DSpark | 7 | 3.69 | 80 | 3541 | 3411 | 130 | 286.67 | 2171.00 | 108.55 | +| NVFP4 | `vllm/agg-h200-mtp/deploy.yaml` | vLLM | agg | H200 | Single worker | MTP | 7 | 3.687 | 12 | 3541 | 3411 | 130 | 250.66 | 1478.67 | 123.22 | +| NVFP4 | `vllm/disagg-h100-dflash/deploy.yaml` | vLLM | disagg | H100 | 1P1D | DFlash | 3 | 2.73 | 12 | 3541 | 3404 | 137 | 542.71 | 1117.29 | 186.21 | +| NVFP4 | `vllm/disagg-h100-dspark/deploy.yaml` | vLLM | disagg | H100 | 1P1D | DSpark | 1 | 1.83 | 24 | 3541 | 3411 | 130 | 2028.23 | 1445.29 | 120.44 | +| NVFP4 | `vllm/disagg-h200-dflash/deploy.yaml` | vLLM | disagg | H200 | 1P1D | DFlash | 3 | 2.73 | 12 | 3541 | 3401 | 140 | 408.30 | 1277.86 | 212.98 | +| NVFP4 | `vllm/disagg-h200-dspark/deploy.yaml` | vLLM | disagg | H200 | 1P1D | DSpark | 1 | 1.83 | 24 | 3541 | 3411 | 130 | 834.80 | 1656.90 | 138.08 | +| NVFP4 | `trtllm/agg-h100-mtp/deploy.yaml` | TensorRT-LLM | agg | H100 | Single worker | MTP | 7 | 3.687 | 12 | 500 | 499 | 1 | 1757.00 | 264.87 | 22.07 | +| NVFP4 | `trtllm/agg-h200-mtp/deploy.yaml` | TensorRT-LLM | agg | H200 | Single worker | MTP | 7 | 3.687 | 12 | 3541 | 3527 | 14 | 1692.98 | 250.05 | 20.84 | + +### B200 and GB200 + +The following table contains BF16 rows for B200 and GB200 recipes. + +| Precision | Recipe | FW | Mode | GPU | Routing | Spec method | Spec tok | Synthetic AL | Concurrency | Requests | Valid requests | Errors | TTFT p50 (ms) | Output tok/s/GPU | Tok/s/user | +|----------|--------|----|------|-----|---------|-------------|----------|--------------|-------------|----------|----------------|--------|---------------|------------------|------------| +| BF16 | `vllm/agg-b200-dspark-bf16/deploy.yaml` | vLLM | agg | B200 | Single worker | DSpark | 7 | 3.69 | 50 | 3541 | 3410 | 131 | 221.49 | 4005.48 | 80.11 | +| BF16 | `vllm/agg-b200-dspark-kv-router-bf16/deploy.yaml` | vLLM | agg | B200 | KV router | DSpark | 7 | 3.69 | 192 | 3541 | 3410 | 131 | 254.26 | 4162.09 | 86.71 | +| BF16 | `vllm/agg-b200-mtp-bf16/deploy.yaml` | vLLM | agg | B200 | Single worker | MTP | 3 | 2.874 | 18 | 3541 | 3410 | 131 | 165.43 | 2454.72 | 136.37 | +| BF16 | `vllm/disagg-b200-dspark-bf16/deploy.yaml` | vLLM | disagg | B200 | 1P1D | DSpark | 1 | 1.83 | 80 | 3541 | 3410 | 131 | 1059.12 | 3711.37 | 92.78 | +| BF16 | `vllm/agg-gb200-dspark-bf16/deploy.yaml` | vLLM | agg | GB200 | Single worker | DSpark | 7 | 3.69 | 58 | 3541 | 3410 | 131 | 307.73 | 4327.90 | 74.62 | +| BF16 | `vllm/agg-gb200-mtp-bf16/deploy.yaml` | vLLM | agg | GB200 | Single worker | MTP | 3 | 2.874 | 18 | 3541 | 3410 | 131 | 228.77 | 2289.75 | 127.21 | +| BF16 | `vllm/disagg-gb200-dflash-bf16/deploy.yaml` | vLLM | disagg | GB200 | 1P1D | DFlash | 7 | 3.41 | 24 | 3541 | 3410 | 131 | 10641.10 | 1148.33 | 95.69 | +| BF16 | `vllm/disagg-gb200-dspark-bf16/deploy.yaml` | vLLM | disagg | GB200 | 1P1D | DSpark | 1 | 1.83 | 8 | 3541 | 3410 | 131 | 569.70 | 802.30 | 200.58 | +| BF16 | `trtllm/agg-b200-bf16/deploy.yaml` | TensorRT-LLM | agg | B200 | Single worker | None | 0 | 0.0 | 18 | 3541 | 3411 | 130 | 219.18 | 1869.54 | 103.86 | +| BF16 | `trtllm/agg-b200-mtp-bf16/deploy.yaml` | TensorRT-LLM | agg | B200 | Single worker | MTP | 3 | 2.874 | 18 | 3541 | 3411 | 130 | 244.95 | 3295.50 | 183.08 | +| BF16 | `trtllm/agg-gb200-bf16/deploy.yaml` | TensorRT-LLM | agg | GB200 | Single worker | None | 0 | 0.0 | 18 | 3541 | 3411 | 130 | 216.77 | 1985.84 | 110.32 | +| BF16 | `trtllm/agg-gb200-mtp-bf16/deploy.yaml` | TensorRT-LLM | agg | GB200 | Single worker | MTP | 3 | 2.874 | 18 | 3541 | 3411 | 130 | 231.34 | 3556.40 | 197.58 | diff --git a/recipes/nemotron-3.5-lightning/trtllm/README.md b/recipes/nemotron-3.5-lightning/trtllm/README.md index f07348f04bcd..cd74d9f92857 100644 --- a/recipes/nemotron-3.5-lightning/trtllm/README.md +++ b/recipes/nemotron-3.5-lightning/trtllm/README.md @@ -5,8 +5,9 @@ SPDX-License-Identifier: Apache-2.0 # Nemotron 3.5 Lightning TensorRT-LLM Recipes -Aggregate Dynamo TensorRT-LLM recipes for -`nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4`. +Aggregate Dynamo TensorRT-LLM recipes for Nemotron 3.5 Lightning. H100 and H200 +use the NVFP4 checkpoint. B200 and GB200 use +`nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16`. ## Configurations @@ -14,13 +15,13 @@ Aggregate Dynamo TensorRT-LLM recipes for |---|---|---| | H100 | `agg-h100-mtp/deploy.yaml` | MTP | | H200 | `agg-h200-mtp/deploy.yaml` | MTP | -| B200 | `agg-b200/deploy.yaml` | None | -| B200 | `agg-b200-mtp/deploy.yaml` | MTP | -| GB200 | `agg-gb200/deploy.yaml` | None | -| GB200 | `agg-gb200-mtp/deploy.yaml` | MTP | +| B200 | `agg-b200-bf16/deploy.yaml` | None | +| B200 | `agg-b200-mtp-bf16/deploy.yaml` | MTP | +| GB200 | `agg-gb200-bf16/deploy.yaml` | None | +| GB200 | `agg-gb200-mtp-bf16/deploy.yaml` | MTP | -The B200 and GB200 no-spec manifests are named `agg-b200` and `agg-gb200` on -disk. Their Kubernetes resource names retain the `-nospec` suffix. +B200 and GB200 recipes include `-bf16` in the recipe directory and Kubernetes +resource names, for example `agg-b200-mtp-bf16/deploy.yaml`. ## Platform notes @@ -31,7 +32,7 @@ GB200 manifests select ARM nodes. 1. Dynamo Platform and the `DynamoGraphDeployment` CRD installed on the target Kubernetes cluster. 2. The shared model cache created from `../model-cache/model-cache.yaml` and - populated with `../model-cache/model-download.yaml`. + populated with the matching NVFP4 or BF16 download Job. 3. A Hugging Face secret named `hf-token-secret` and an image-pull secret when required by the cluster. @@ -39,24 +40,24 @@ GB200 manifests select ARM nodes. ```bash export NAMESPACE=your-namespace -RECIPE=agg-b200-mtp +RECIPE=agg-b200-mtp-bf16 kubectl apply -n "${NAMESPACE}" -f "${RECIPE}/deploy.yaml" -kubectl get dgd -n "${NAMESPACE}" trtllm-agg-b200-mtp -w +kubectl get dgd -n "${NAMESPACE}" trtllm-agg-b200-mtp-bf16 -w ``` Port-forward the frontend service: ```bash kubectl port-forward -n "${NAMESPACE}" \ - svc/trtllm-agg-b200-mtp-frontend 8000:8000 + svc/trtllm-agg-b200-mtp-bf16-frontend 8000:8000 ``` ```bash curl http://localhost:8000/v1/chat/completions \ -H 'Content-Type: application/json' \ -d '{ - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "messages": [{"role": "user", "content": "Say OK."}], "max_tokens": 128, "temperature": 0 diff --git a/recipes/nemotron-3.5-lightning/trtllm/agg-b200/deploy.yaml b/recipes/nemotron-3.5-lightning/trtllm/agg-b200-bf16/deploy.yaml similarity index 94% rename from recipes/nemotron-3.5-lightning/trtllm/agg-b200/deploy.yaml rename to recipes/nemotron-3.5-lightning/trtllm/agg-b200-bf16/deploy.yaml index 1b9e7cf60638..8657dd2c4762 100644 --- a/recipes/nemotron-3.5-lightning/trtllm/agg-b200/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/trtllm/agg-b200-bf16/deploy.yaml @@ -1,12 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Experimental aggregated TRT-LLM non-speculative deployment for Nemotron 3.5 Lightning NVFP4 on 1x B200. +# Experimental aggregated TRT-LLM non-speculative deployment for Nemotron 3.5 Lightning BF16 on 1x B200. # Runs the checkpoint without speculative decoding. apiVersion: v1 kind: ConfigMap metadata: - name: trtllm-agg-b200-nospec-config + name: trtllm-agg-b200-bf16-config data: agg.yaml: | backend: pytorch @@ -29,9 +29,9 @@ data: apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: - name: trtllm-agg-b200-nospec + name: trtllm-agg-b200-bf16 labels: - app.kubernetes.io/name: trtllm-agg-b200-nospec + app.kubernetes.io/name: trtllm-agg-b200-bf16 app.kubernetes.io/part-of: dynamo spec: backendFramework: trtllm @@ -118,9 +118,9 @@ spec: - -m - dynamo.trtllm - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --tensor-parallel-size - "1" - --max-num-tokens @@ -199,4 +199,4 @@ spec: sizeLimit: 40Gi - name: trtllm-config configMap: - name: trtllm-agg-b200-nospec-config + name: trtllm-agg-b200-bf16-config diff --git a/recipes/nemotron-3.5-lightning/trtllm/agg-b200-mtp/deploy.yaml b/recipes/nemotron-3.5-lightning/trtllm/agg-b200-mtp-bf16/deploy.yaml similarity index 94% rename from recipes/nemotron-3.5-lightning/trtllm/agg-b200-mtp/deploy.yaml rename to recipes/nemotron-3.5-lightning/trtllm/agg-b200-mtp-bf16/deploy.yaml index d2e4cf1d1d3e..881e9866e58d 100644 --- a/recipes/nemotron-3.5-lightning/trtllm/agg-b200-mtp/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/trtllm/agg-b200-mtp-bf16/deploy.yaml @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Experimental aggregated TRT-LLM MTP deployment for Nemotron 3.5 Lightning NVFP4 on 1x B200. +# Experimental aggregated TRT-LLM MTP deployment for Nemotron 3.5 Lightning BF16 on 1x B200. apiVersion: v1 kind: ConfigMap metadata: - name: trtllm-agg-b200-mtp-config + name: trtllm-agg-b200-mtp-bf16-config data: agg.yaml: | backend: pytorch @@ -29,14 +29,14 @@ data: decoding_type: MTP max_draft_len: 3 mtp_eagle_one_model: true - speculative_model: nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + speculative_model: nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 --- apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: - name: trtllm-agg-b200-mtp + name: trtllm-agg-b200-mtp-bf16 labels: - app.kubernetes.io/name: trtllm-agg-b200-mtp + app.kubernetes.io/name: trtllm-agg-b200-mtp-bf16 app.kubernetes.io/part-of: dynamo spec: backendFramework: trtllm @@ -123,9 +123,9 @@ spec: - -m - dynamo.trtllm - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --tensor-parallel-size - "1" - --max-num-tokens @@ -207,4 +207,4 @@ spec: sizeLimit: 40Gi - name: trtllm-config configMap: - name: trtllm-agg-b200-mtp-config + name: trtllm-agg-b200-mtp-bf16-config diff --git a/recipes/nemotron-3.5-lightning/trtllm/agg-gb200/deploy.yaml b/recipes/nemotron-3.5-lightning/trtllm/agg-gb200-bf16/deploy.yaml similarity index 94% rename from recipes/nemotron-3.5-lightning/trtllm/agg-gb200/deploy.yaml rename to recipes/nemotron-3.5-lightning/trtllm/agg-gb200-bf16/deploy.yaml index 08b0921987f5..bcae18c23909 100644 --- a/recipes/nemotron-3.5-lightning/trtllm/agg-gb200/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/trtllm/agg-gb200-bf16/deploy.yaml @@ -1,12 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Experimental aggregated TRT-LLM non-speculative deployment for Nemotron 3.5 Lightning NVFP4 on 1x GB200. +# Experimental aggregated TRT-LLM non-speculative deployment for Nemotron 3.5 Lightning BF16 on 1x GB200. # Runs the checkpoint without speculative decoding. apiVersion: v1 kind: ConfigMap metadata: - name: trtllm-agg-gb200-nospec-config + name: trtllm-agg-gb200-bf16-config data: agg.yaml: | backend: pytorch @@ -29,9 +29,9 @@ data: apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: - name: trtllm-agg-gb200-nospec + name: trtllm-agg-gb200-bf16 labels: - app.kubernetes.io/name: trtllm-agg-gb200-nospec + app.kubernetes.io/name: trtllm-agg-gb200-bf16 app.kubernetes.io/part-of: dynamo spec: backendFramework: trtllm @@ -118,9 +118,9 @@ spec: - -m - dynamo.trtllm - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --tensor-parallel-size - "1" - --max-num-tokens @@ -199,4 +199,4 @@ spec: sizeLimit: 40Gi - name: trtllm-config configMap: - name: trtllm-agg-gb200-nospec-config + name: trtllm-agg-gb200-bf16-config diff --git a/recipes/nemotron-3.5-lightning/trtllm/agg-gb200-mtp/deploy.yaml b/recipes/nemotron-3.5-lightning/trtllm/agg-gb200-mtp-bf16/deploy.yaml similarity index 94% rename from recipes/nemotron-3.5-lightning/trtllm/agg-gb200-mtp/deploy.yaml rename to recipes/nemotron-3.5-lightning/trtllm/agg-gb200-mtp-bf16/deploy.yaml index 6dbfe24f755a..73208ecf027c 100644 --- a/recipes/nemotron-3.5-lightning/trtllm/agg-gb200-mtp/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/trtllm/agg-gb200-mtp-bf16/deploy.yaml @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Experimental aggregated TRT-LLM MTP deployment for Nemotron 3.5 Lightning NVFP4 on 1x GB200. +# Experimental aggregated TRT-LLM MTP deployment for Nemotron 3.5 Lightning BF16 on 1x GB200. apiVersion: v1 kind: ConfigMap metadata: - name: trtllm-agg-gb200-mtp-config + name: trtllm-agg-gb200-mtp-bf16-config data: agg.yaml: | backend: pytorch @@ -29,14 +29,14 @@ data: decoding_type: MTP max_draft_len: 3 mtp_eagle_one_model: true - speculative_model: nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + speculative_model: nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 --- apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: - name: trtllm-agg-gb200-mtp + name: trtllm-agg-gb200-mtp-bf16 labels: - app.kubernetes.io/name: trtllm-agg-gb200-mtp + app.kubernetes.io/name: trtllm-agg-gb200-mtp-bf16 app.kubernetes.io/part-of: dynamo spec: backendFramework: trtllm @@ -123,9 +123,9 @@ spec: - -m - dynamo.trtllm - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --tensor-parallel-size - "1" - --max-num-tokens @@ -207,4 +207,4 @@ spec: sizeLimit: 40Gi - name: trtllm-config configMap: - name: trtllm-agg-gb200-mtp-config + name: trtllm-agg-gb200-mtp-bf16-config diff --git a/recipes/nemotron-3.5-lightning/vllm/README.md b/recipes/nemotron-3.5-lightning/vllm/README.md index e20ee5e0303b..d5cced2e4e7e 100644 --- a/recipes/nemotron-3.5-lightning/vllm/README.md +++ b/recipes/nemotron-3.5-lightning/vllm/README.md @@ -5,32 +5,34 @@ SPDX-License-Identifier: Apache-2.0 # Nemotron 3.5 Lightning vLLM Recipes -Dynamo vLLM recipes for -`nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4`. Every hardware family has -aggregate configurations with MTP, DFlash, and DSpark speculative decoding. -Disaggregated 1-prefill/1-decode configurations are provided for selected -hardware and speculative decoding combinations. +Dynamo vLLM recipes for Nemotron 3.5 Lightning. H100 and H200 use +`nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4`, while B200 and GB200 use +`nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16`. Aggregate and +disaggregated configurations are available as explained below. ## Configurations | Hardware | Aggregate recipes | Disaggregated recipes | P/D transport | |---|---|---|---| -| H100 | `agg-h100-{mtp,dflash,dspark}` | `disagg-h100-{mtp,dflash,dspark}` | AWS EFA / Libfabric | -| H200 | `agg-h200-{mtp,dflash,dspark}` | `disagg-h200-{mtp,dflash,dspark}` | UCX over IB/RDMA | -| B200 | `agg-b200-{mtp,dflash,dspark}` | `disagg-b200-dspark` | UCX over IB/RDMA | -| GB200 | `agg-gb200-{mtp,dflash,dspark}` | `disagg-gb200-{dflash,dspark}` | UCX/NIXL, cluster-specific fabric resources | +| H100 | `agg-h100-{mtp,dflash,dspark}` | `disagg-h100-{dflash,dspark}` | AWS EFA / Libfabric | +| H200 | `agg-h200-{mtp,dflash,dspark}` | `disagg-h200-{dflash,dspark}` | UCX over IB/RDMA | +| B200 | `agg-b200-{mtp,dspark}-bf16` | `disagg-b200-dspark-bf16` | UCX over IB/RDMA | +| GB200 | `agg-gb200-{mtp,dspark}-bf16` | `disagg-gb200-{dflash,dspark}-bf16` | UCX/NIXL, cluster-specific fabric resources | + +B200 and GB200 directories use BF16 weights and include a `-bf16` suffix, for +example `agg-b200-dspark-bf16` and `disagg-gb200-dspark-bf16`. Each recipe identifier corresponds to `identifier/deploy.yaml` in this directory. ### KV-aware aggregate DSpark variants -For users who want the highest-performing aggregate DSpark shape with KV-aware routing ready to deploy, the repo also includes event-backed KV-router variants. These keep the vLLM aggregate DSpark worker configuration, run four aggregate replicas, and publish KV-cache events to the frontend KV router. Start benchmark concurrency at 4x the matching non-KV aggregate recommendation. +For users who want the highest-performing aggregate DSpark shape with KV-aware routing ready to deploy, the repo also includes event-backed KV-router variants. These keep the vLLM aggregate DSpark worker configuration, run four aggregate replicas, and publish KV-cache events to the frontend KV router. Start benchmark concurrency at approximately 4x the matching non-KV aggregate recommendation. | Recipe | Worker replicas | Base concurrency | Suggested KV-router concurrency | |---|---:|---:|---:| | `agg-h100-dspark-kv-router` | 4 | 20 | 80 | | `agg-h200-dspark-kv-router` | 4 | 20 | 80 | -| `agg-b200-dspark-kv-router` | 4 | 24 | 96 | +| `agg-b200-dspark-kv-router-bf16` | 4 | 50 | 192 | These recipes are provided for workloads that can benefit from distributing reusable KV state across several workers, for example when a single worker sees @@ -42,20 +44,12 @@ worker therefore had enough cache headroom to keep useful prefixes resident without meaningful eviction pressure, while adding replicas split work across more GPUs. -## Model configuration - -| Recipe type | Target model | Draft model | -|---|---|---| -| MTP | `nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4` | Embedded MTP | -| DFlash | `nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4` | `nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DFlash` | -| DSpark | `nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4` | `nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark` | - ## Prerequisites 1. Dynamo Platform and the `DynamoGraphDeployment` CRD installed on the target Kubernetes cluster. 2. The shared model cache created from `../model-cache/model-cache.yaml` and - populated with `../model-cache/model-download.yaml`. + populated with the matching NVFP4 or BF16 download Job. 3. A Hugging Face secret named `hf-token-secret` and an image-pull secret when required by the cluster. 4. For disaggregated recipes that use AWS EFA, EFA userspace must be prepared by @@ -65,58 +59,19 @@ more GPUs. AWS-specific setup. See [EFA setup](efa.md) for validation and deployment-time setup options. -## Quick start - -```bash -export NAMESPACE=your-namespace -GPU=h100 # h100, h200, b200, or gb200 -MODE=agg # agg or disagg -SPECDEC=dflash # choose a valid speculative decoding mode from the table - -kubectl apply -n "${NAMESPACE}" -f "${MODE}-${GPU}-${SPECDEC}/deploy.yaml" -kubectl get dgd -n "${NAMESPACE}" "vllm-${MODE}-${GPU}-${SPECDEC}" -w -``` - -Port-forward the frontend service: - -```bash -kubectl port-forward -n "${NAMESPACE}" \ - "svc/vllm-${MODE}-${GPU}-${SPECDEC}-frontend" 8000:8000 -``` - -```bash -curl http://localhost:8000/v1/chat/completions \ - -H 'Content-Type: application/json' \ - -d '{ - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", - "messages": [{"role": "user", "content": "Say OK."}], - "max_tokens": 128, - "temperature": 0 - }' -``` - ## Synthetic acceptance configuration -Each manifest defines two draft configurations: - -- `speculative-config` is the standard serving configuration. -- `speculative-config-synthetic` enables synthetic rejection sampling for - benchmark reproduction. - -Select the synthetic configuration by changing the `SPECULATIVE_CONFIG` -ConfigMap key. +Each manifest defines `speculative-config` for standard serving and +`speculative-config-synthetic` for benchmark reproduction. Select the +synthetic configuration by changing the `SPECULATIVE_CONFIG` ConfigMap key. -| Family | Aggregate draft tokens / expected AL | Disaggregated draft tokens / expected AL | -|---|---:|---:| -| MTP H100/H200 | 7 / 3.687 | 5 / 3.421 | -| MTP B200/GB200 | 3 / 2.874 | Not provided | -| DFlash H100/H200/B200/GB200 | 5 / 3.18 | H100/H200: 3 / 2.73; GB200: 7 / 3.41 | -| DSpark | 7 / 3.69 | 1 / 1.83 | +The synthetic acceptance lengths used in these configurations are based on +internal measurements. ## Notes - GB200 recipes use ARM-node scheduling. -- B200 DFlash recipes disable the FlashInfer FP8 ScaledMM linear kernel. +- B200 recipes disable the FlashInfer FP8 ScaledMM linear kernel. - Disaggregated recipes require matching speculative-decoding settings on the prefill and decode workers for compatible NIXL cache metadata. - AWS EFA disaggregated recipes require the EFA userspace stack before diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-b200-dflash/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-b200-dflash/deploy.yaml deleted file mode 100644 index f67441b8334c..000000000000 --- a/recipes/nemotron-3.5-lightning/vllm/agg-b200-dflash/deploy.yaml +++ /dev/null @@ -1,204 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Aggregated vLLM deployment for Nemotron 3.5 Lightning NVFP4 on 1x B200. -# The B200 recipes in this repo disable the FlashInfer FP8 ScaledMM path to -# keep CUDA graphs enabled for this configuration. -apiVersion: v1 -kind: ConfigMap -metadata: - name: vllm-agg-b200-dflash-config -data: - speculative-config: |- - { - "method": "dflash", - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DFlash", - "attention_backend": "TRITON_ATTN", - "num_speculative_tokens": 5 - } - speculative-config-synthetic: |- - { - "method": "dflash", - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DFlash", - "attention_backend": "TRITON_ATTN", - "num_speculative_tokens": 5, - "rejection_sample_method": "synthetic", - "synthetic_acceptance_length": 3.18 - } ---- -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: vllm-agg-b200-dflash - labels: - app.kubernetes.io/name: vllm-agg-b200-dflash - app.kubernetes.io/part-of: dynamo -spec: - backendFramework: vllm - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - spec: - imagePullSecrets: - - name: your-image-pull-secret - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.0-nemotron-3.5-lightning-dev.1 - imagePullPolicy: IfNotPresent - command: - - python3 - args: - - -m - - dynamo.frontend - - --trust-remote-code - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: DYN_HTTP_BODY_LIMIT_MB - value: "200" - - name: HF_HOME - value: /model-cache - - name: HF_HUB_OFFLINE - value: "1" - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - resources: {} - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 60 - failureThreshold: 60 - volumeMounts: - - name: shared-model-cache - mountPath: /model-cache - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - - name: agg - type: worker - replicas: 1 - sharedMemorySize: 40Gi - podTemplate: - spec: - imagePullSecrets: - - name: your-image-pull-secret - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.product - operator: In - values: - - NVIDIA-B200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: dshm - emptyDir: - medium: Memory - sizeLimit: 40Gi - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.0-nemotron-3.5-lightning-dev.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace/ - command: - - python3 - - -m - - dynamo.vllm - args: - - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 - - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 - - --trust-remote-code - - --tensor-parallel-size - - "1" - - --max-num-seqs - - "512" - - --enable-prefix-caching - - --async-scheduling - - --no-enable-flashinfer-autotune - - --mamba-backend - - flashinfer - - --mamba-ssm-cache-dtype - - float16 - - --enable-mamba-cache-stochastic-rounding - - --mamba-cache-philox-rounds - - "5" - - --dyn-tool-call-parser - - nemotron_nano - - --dyn-reasoning-parser - - nemotron_nano - - --reasoning-parser - - nemotron_v3 - - --speculative-config=$(SPECULATIVE_CONFIG) - - --disaggregation-mode - - agg - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: HF_HUB_OFFLINE - value: "1" - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: VLLM_CONFIG_ROOT - value: /tmp/vllm-config - - name: VLLM_CACHE_ROOT - value: /tmp/vllm-cache - - name: SPECULATIVE_CONFIG - valueFrom: - configMapKeyRef: - name: vllm-agg-b200-dflash-config - key: speculative-config # speculative-config-synthetic for benchmarking - - name: VLLM_DISABLED_KERNELS - value: FlashInferFP8ScaledMMLinearKernel - - name: NCCL_IB_DISABLE - value: "1" - - name: PYTHONHASHSEED - value: "0" - resources: - requests: - memory: 128Gi - nvidia.com/gpu: "1" - limits: - nvidia.com/gpu: "1" - securityContext: - runAsUser: 0 - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 30 - timeoutSeconds: 20 - failureThreshold: 120 - volumeMounts: - - name: shared-model-cache - mountPath: /model-cache - - name: dshm - mountPath: /dev/shm diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-bf16/deploy.yaml similarity index 91% rename from recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark/deploy.yaml rename to recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-bf16/deploy.yaml index 85258b374f96..3653fc9771de 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-bf16/deploy.yaml @@ -1,25 +1,25 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Aggregated vLLM deployment for Nemotron 3.5 Lightning NVFP4 on 1x B200 with DSpark speculative decoding. +# Aggregated vLLM deployment for Nemotron 3.5 Lightning BF16 on 1x B200 with DSpark speculative decoding. # The B200 recipes in this repo disable the FlashInfer FP8 ScaledMM path to # keep CUDA graphs enabled for this configuration. apiVersion: v1 kind: ConfigMap metadata: - name: vllm-agg-b200-dspark-config + name: vllm-agg-b200-dspark-bf16-config data: speculative-config: |- { "method": "dspark", - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark", + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-DSpark", "num_speculative_tokens": 7, "attention_backend": "TRITON_ATTN" } speculative-config-synthetic: |- { "method": "dspark", - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark", + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-DSpark", "num_speculative_tokens": 7, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 3.69, @@ -29,9 +29,9 @@ data: apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: - name: vllm-agg-b200-dspark + name: vllm-agg-b200-dspark-bf16 labels: - app.kubernetes.io/name: vllm-agg-b200-dspark + app.kubernetes.io/name: vllm-agg-b200-dspark-bf16 app.kubernetes.io/part-of: dynamo spec: backendFramework: vllm @@ -123,9 +123,9 @@ spec: - dynamo.vllm args: - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --trust-remote-code - --tensor-parallel-size - "1" @@ -172,7 +172,7 @@ spec: - name: SPECULATIVE_CONFIG valueFrom: configMapKeyRef: - name: vllm-agg-b200-dspark-config + name: vllm-agg-b200-dspark-bf16-config key: speculative-config # speculative-config-synthetic for benchmarking - name: VLLM_DISABLED_KERNELS value: FlashInferFP8ScaledMMLinearKernel diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-kv-router/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-kv-router-bf16/deploy.yaml similarity index 91% rename from recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-kv-router/deploy.yaml rename to recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-kv-router-bf16/deploy.yaml index 9770abfa62e0..4b56a05d2ba6 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-kv-router/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-b200-dspark-kv-router-bf16/deploy.yaml @@ -1,25 +1,25 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Aggregated vLLM deployment for Nemotron 3.5 Lightning NVFP4 on 1x B200 with DSpark speculative decoding. +# Aggregated vLLM deployment for Nemotron 3.5 Lightning BF16 on 1x B200 with DSpark speculative decoding. # The B200 recipes in this repo disable the FlashInfer FP8 ScaledMM path to # keep CUDA graphs enabled for this configuration. apiVersion: v1 kind: ConfigMap metadata: - name: vllm-agg-b200-dspark-kv-router-config + name: vllm-agg-b200-dspark-kvr-bf16-config data: speculative-config: |- { "method": "dspark", - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark", + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-DSpark", "num_speculative_tokens": 7, "attention_backend": "TRITON_ATTN" } speculative-config-synthetic: |- { "method": "dspark", - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark", + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-DSpark", "num_speculative_tokens": 7, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 3.69, @@ -29,9 +29,9 @@ data: apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: - name: vllm-agg-b200-dspark-kv-router + name: vllm-agg-b200-dspark-kvr-bf16 labels: - app.kubernetes.io/name: vllm-agg-b200-dspark-kv-router + app.kubernetes.io/name: vllm-agg-b200-dspark-kvr-bf16 app.kubernetes.io/part-of: dynamo spec: backendFramework: vllm @@ -128,9 +128,9 @@ spec: - dynamo.vllm args: - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --trust-remote-code - --tensor-parallel-size - "1" @@ -181,7 +181,7 @@ spec: - name: SPECULATIVE_CONFIG valueFrom: configMapKeyRef: - name: vllm-agg-b200-dspark-kv-router-config + name: vllm-agg-b200-dspark-kvr-bf16-config key: speculative-config # speculative-config-synthetic for benchmarking - name: VLLM_DISABLED_KERNELS value: FlashInferFP8ScaledMMLinearKernel diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-b200-mtp/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-b200-mtp-bf16/deploy.yaml similarity index 93% rename from recipes/nemotron-3.5-lightning/vllm/agg-b200-mtp/deploy.yaml rename to recipes/nemotron-3.5-lightning/vllm/agg-b200-mtp-bf16/deploy.yaml index 7a33d86816b7..8593f0481551 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-b200-mtp/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-b200-mtp-bf16/deploy.yaml @@ -1,13 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Aggregated vLLM deployment for Nemotron 3.5 Lightning NVFP4 on 1x B200 with built-in MTP speculative decoding. +# Aggregated vLLM deployment for Nemotron 3.5 Lightning BF16 on 1x B200 with built-in MTP speculative decoding. # The B200 recipes in this repo disable the FlashInfer FP8 ScaledMM path to # keep CUDA graphs enabled for this configuration. apiVersion: v1 kind: ConfigMap metadata: - name: vllm-agg-b200-mtp-config + name: vllm-agg-b200-mtp-bf16-config data: speculative-config: |- { @@ -25,9 +25,9 @@ data: apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: - name: vllm-agg-b200-mtp + name: vllm-agg-b200-mtp-bf16 labels: - app.kubernetes.io/name: vllm-agg-b200-mtp + app.kubernetes.io/name: vllm-agg-b200-mtp-bf16 app.kubernetes.io/part-of: dynamo spec: backendFramework: vllm @@ -119,9 +119,9 @@ spec: - dynamo.vllm args: - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --trust-remote-code - --tensor-parallel-size - "1" @@ -166,7 +166,7 @@ spec: - name: SPECULATIVE_CONFIG valueFrom: configMapKeyRef: - name: vllm-agg-b200-mtp-config + name: vllm-agg-b200-mtp-bf16-config key: speculative-config # speculative-config-synthetic for benchmarking - name: VLLM_DISABLED_KERNELS value: FlashInferFP8ScaledMMLinearKernel diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-gb200-dflash/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-gb200-dflash/deploy.yaml deleted file mode 100644 index bd8a4cac8e1c..000000000000 --- a/recipes/nemotron-3.5-lightning/vllm/agg-gb200-dflash/deploy.yaml +++ /dev/null @@ -1,239 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Aggregated vLLM deployment for Nemotron 3.5 Lightning NVFP4 on 1x GB200. -apiVersion: v1 -kind: ConfigMap -metadata: - name: vllm-agg-gb200-dflash-config -data: - speculative-config: |- - { - "method": "dflash", - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DFlash", - "attention_backend": "TRITON_ATTN", - "num_speculative_tokens": 5 - } - speculative-config-synthetic: |- - { - "method": "dflash", - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DFlash", - "attention_backend": "TRITON_ATTN", - "num_speculative_tokens": 5, - "rejection_sample_method": "synthetic", - "synthetic_acceptance_length": 3.18 - } ---- -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: vllm-agg-gb200-dflash - labels: - app.kubernetes.io/name: vllm-agg-gb200-dflash - app.kubernetes.io/part-of: dynamo -spec: - backendFramework: vllm - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - spec: - imagePullSecrets: - - name: your-image-pull-secret - nodeSelector: - nvidia.com/gpu.product: NVIDIA-GB200 - tolerations: - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - - key: kubernetes.io/arch - operator: Equal - value: arm64 - effect: NoSchedule - - key: dedicated - operator: Exists - effect: NoSchedule - - key: skyhook.nvidia.com - operator: Exists - effect: NoSchedule - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.0-nemotron-3.5-lightning-dev.1 - imagePullPolicy: IfNotPresent - command: - - python3 - args: - - -m - - dynamo.frontend - - --trust-remote-code - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: DYN_HTTP_BODY_LIMIT_MB - value: "200" - - name: HF_HOME - value: /model-cache - - name: HF_HUB_OFFLINE - value: "1" - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - resources: {} - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 60 - failureThreshold: 60 - volumeMounts: - - name: shared-model-cache - mountPath: /model-cache - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - - name: agg - type: worker - replicas: 1 - sharedMemorySize: 40Gi - podTemplate: - spec: - imagePullSecrets: - - name: your-image-pull-secret - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.product - operator: In - values: - - NVIDIA-GB200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: "true" - effect: NoSchedule - - key: kubernetes.io/arch - operator: Equal - value: arm64 - effect: NoSchedule - - key: dedicated - operator: Exists - effect: NoSchedule - - key: skyhook.nvidia.com - operator: Exists - effect: NoSchedule - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: dshm - emptyDir: - medium: Memory - sizeLimit: 40Gi - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.0-nemotron-3.5-lightning-dev.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace/ - command: - - python3 - - -m - - dynamo.vllm - args: - - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 - - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 - - --trust-remote-code - - --tensor-parallel-size - - "1" - - --max-num-seqs - - "512" - - --max-num-batched-tokens - - "10240" - - --enable-prefix-caching - - --async-scheduling - - --mamba-backend - - flashinfer - - --mamba-ssm-cache-dtype - - float16 - - --enable-mamba-cache-stochastic-rounding - - --mamba-cache-philox-rounds - - "5" - - --dyn-tool-call-parser - - nemotron_nano - - --dyn-reasoning-parser - - nemotron_nano - - --reasoning-parser - - nemotron_v3 - - --speculative-config=$(SPECULATIVE_CONFIG) - - --disaggregation-mode - - agg - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: HF_HUB_OFFLINE - value: "1" - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: VLLM_CONFIG_ROOT - value: /tmp/vllm-config - - name: VLLM_CACHE_ROOT - value: /tmp/vllm-cache - - name: LIBRARY_PATH - value: /usr/local/nvidia/lib64:/usr/local/nvidia/lib:/usr/local/cuda/compat - - name: VLLM_ENGINE_READY_TIMEOUT_S - value: "3600" - - name: VLLM_OMNI_USE_QUACK_FP8 - value: "0" - - name: VLLM_WORKER_MULTIPROC_METHOD - value: "spawn" - - name: VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS - value: "1" - - name: SPECULATIVE_CONFIG - valueFrom: - configMapKeyRef: - name: vllm-agg-gb200-dflash-config - key: speculative-config # speculative-config-synthetic for benchmarking - - name: NCCL_IB_DISABLE - value: "1" - - name: PYTHONHASHSEED - value: "0" - - name: PYTHONUNBUFFERED - value: "1" - resources: - requests: - memory: 128Gi - nvidia.com/gpu: "1" - limits: - nvidia.com/gpu: "1" - securityContext: - runAsUser: 0 - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 30 - timeoutSeconds: 20 - failureThreshold: 120 - volumeMounts: - - name: shared-model-cache - mountPath: /model-cache - - name: dshm - mountPath: /dev/shm diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-gb200-dspark/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-gb200-dspark-bf16/deploy.yaml similarity index 92% rename from recipes/nemotron-3.5-lightning/vllm/agg-gb200-dspark/deploy.yaml rename to recipes/nemotron-3.5-lightning/vllm/agg-gb200-dspark-bf16/deploy.yaml index 4d399a25dba3..91c8a6f83578 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-gb200-dspark/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-gb200-dspark-bf16/deploy.yaml @@ -1,23 +1,23 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Aggregated vLLM deployment for Nemotron 3.5 Lightning NVFP4 on 1x GB200 with DSpark speculative decoding. +# Aggregated vLLM deployment for Nemotron 3.5 Lightning BF16 on 1x GB200 with DSpark speculative decoding. apiVersion: v1 kind: ConfigMap metadata: - name: vllm-agg-gb200-dspark-config + name: vllm-agg-gb200-dspark-bf16-config data: speculative-config: |- { "method": "dspark", - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark", + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-DSpark", "num_speculative_tokens": 7, "attention_backend": "TRITON_ATTN" } speculative-config-synthetic: |- { "method": "dspark", - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark", + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-DSpark", "num_speculative_tokens": 7, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 3.69, @@ -27,9 +27,9 @@ data: apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: - name: vllm-agg-gb200-dspark + name: vllm-agg-gb200-dspark-bf16 labels: - app.kubernetes.io/name: vllm-agg-gb200-dspark + app.kubernetes.io/name: vllm-agg-gb200-dspark-bf16 app.kubernetes.io/part-of: dynamo spec: backendFramework: vllm @@ -147,9 +147,9 @@ spec: - dynamo.vllm args: - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --trust-remote-code - --tensor-parallel-size - "1" @@ -207,7 +207,7 @@ spec: - name: SPECULATIVE_CONFIG valueFrom: configMapKeyRef: - name: vllm-agg-gb200-dspark-config + name: vllm-agg-gb200-dspark-bf16-config key: speculative-config # speculative-config-synthetic for benchmarking - name: NCCL_IB_DISABLE value: "1" diff --git a/recipes/nemotron-3.5-lightning/vllm/agg-gb200-mtp/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/agg-gb200-mtp-bf16/deploy.yaml similarity index 94% rename from recipes/nemotron-3.5-lightning/vllm/agg-gb200-mtp/deploy.yaml rename to recipes/nemotron-3.5-lightning/vllm/agg-gb200-mtp-bf16/deploy.yaml index adfeee814650..b031d4887cf4 100644 --- a/recipes/nemotron-3.5-lightning/vllm/agg-gb200-mtp/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/agg-gb200-mtp-bf16/deploy.yaml @@ -1,11 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # -# Aggregated vLLM deployment for Nemotron 3.5 Lightning NVFP4 on 1x GB200 with built-in MTP speculative decoding. +# Aggregated vLLM deployment for Nemotron 3.5 Lightning BF16 on 1x GB200 with built-in MTP speculative decoding. apiVersion: v1 kind: ConfigMap metadata: - name: vllm-agg-gb200-mtp-config + name: vllm-agg-gb200-mtp-bf16-config data: speculative-config: |- { @@ -23,9 +23,9 @@ data: apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: - name: vllm-agg-gb200-mtp + name: vllm-agg-gb200-mtp-bf16 labels: - app.kubernetes.io/name: vllm-agg-gb200-mtp + app.kubernetes.io/name: vllm-agg-gb200-mtp-bf16 app.kubernetes.io/part-of: dynamo spec: backendFramework: vllm @@ -143,9 +143,9 @@ spec: - dynamo.vllm args: - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --trust-remote-code - --tensor-parallel-size - "1" @@ -201,7 +201,7 @@ spec: - name: SPECULATIVE_CONFIG valueFrom: configMapKeyRef: - name: vllm-agg-gb200-mtp-config + name: vllm-agg-gb200-mtp-bf16-config key: speculative-config # speculative-config-synthetic for benchmarking - name: NCCL_IB_DISABLE value: "1" diff --git a/recipes/nemotron-3.5-lightning/vllm/disagg-b200-dspark/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/disagg-b200-dspark-bf16/deploy.yaml similarity index 91% rename from recipes/nemotron-3.5-lightning/vllm/disagg-b200-dspark/deploy.yaml rename to recipes/nemotron-3.5-lightning/vllm/disagg-b200-dspark-bf16/deploy.yaml index 7f5b1f5cd2fe..9a2f43c5e4b3 100644 --- a/recipes/nemotron-3.5-lightning/vllm/disagg-b200-dspark/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/disagg-b200-dspark-bf16/deploy.yaml @@ -4,19 +4,19 @@ apiVersion: v1 kind: ConfigMap metadata: - name: vllm-disagg-b200-dspark-config + name: vllm-disagg-b200-dspark-bf16-config data: speculative-config: |- { "method": "dspark", - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark", + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-DSpark", "num_speculative_tokens": 1, "attention_backend": "TRITON_ATTN" } speculative-config-synthetic: |- { "method": "dspark", - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark", + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-DSpark", "num_speculative_tokens": 1, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 1.83, @@ -26,9 +26,9 @@ data: apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: - name: vllm-disagg-b200-dspark + name: vllm-disagg-b200-dspark-bf16 labels: - app.kubernetes.io/name: vllm-disagg-b200-dspark + app.kubernetes.io/name: vllm-disagg-b200-dspark-bf16 app.kubernetes.io/part-of: dynamo spec: backendFramework: vllm @@ -119,14 +119,14 @@ spec: - -m - dynamo.vllm - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --trust-remote-code - --tensor-parallel-size - '1' - --max-num-seqs - - '32' + - '80' - --async-scheduling - --no-enable-flashinfer-autotune - --enable-prefix-caching @@ -146,6 +146,7 @@ spec: - nemotron_nano - --reasoning-parser - nemotron_v3 + - --speculative-config=$(SPECULATIVE_CONFIG) - --kv-transfer-config - '{"kv_connector":"NixlConnector","kv_role":"kv_producer"}' - --disaggregation-mode @@ -167,6 +168,11 @@ spec: value: /tmp/vllm-config - name: VLLM_CACHE_ROOT value: /tmp/vllm-cache + - name: SPECULATIVE_CONFIG + valueFrom: + configMapKeyRef: + name: vllm-disagg-b200-dspark-bf16-config + key: speculative-config # speculative-config-synthetic for benchmarking - name: VLLM_DISABLED_KERNELS value: FlashInferFP8ScaledMMLinearKernel - name: NCCL_IB_DISABLE @@ -250,14 +256,14 @@ spec: - -m - dynamo.vllm - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --trust-remote-code - --tensor-parallel-size - '1' - --max-num-seqs - - '32' + - '80' - --async-scheduling - --no-enable-flashinfer-autotune - --enable-prefix-caching @@ -302,7 +308,7 @@ spec: - name: SPECULATIVE_CONFIG valueFrom: configMapKeyRef: - name: vllm-disagg-b200-dspark-config + name: vllm-disagg-b200-dspark-bf16-config key: speculative-config # speculative-config-synthetic for benchmarking - name: VLLM_DISABLED_KERNELS value: FlashInferFP8ScaledMMLinearKernel diff --git a/recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dflash/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dflash-bf16/deploy.yaml similarity index 94% rename from recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dflash/deploy.yaml rename to recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dflash-bf16/deploy.yaml index cba84430501b..e2814ffbe87b 100644 --- a/recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dflash/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dflash-bf16/deploy.yaml @@ -4,18 +4,18 @@ apiVersion: v1 kind: ConfigMap metadata: - name: vllm-disagg-gb200-dflash-config + name: vllm-disagg-gb200-dflash-bf16-config data: speculative-config: |- { "method": "dflash", - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DFlash", + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-DFlash", "num_speculative_tokens": 7 } speculative-config-synthetic: |- { "method": "dflash", - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DFlash", + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-DFlash", "num_speculative_tokens": 7, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 3.41 @@ -24,9 +24,9 @@ data: apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: - name: vllm-disagg-gb200-dflash + name: vllm-disagg-gb200-dflash-bf16 labels: - app.kubernetes.io/name: vllm-disagg-gb200-dflash + app.kubernetes.io/name: vllm-disagg-gb200-dflash-bf16 app.kubernetes.io/part-of: dynamo spec: backendFramework: vllm @@ -137,9 +137,9 @@ spec: - dynamo.vllm args: - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --trust-remote-code - --tensor-parallel-size - '1' @@ -192,7 +192,7 @@ spec: - name: SPECULATIVE_CONFIG valueFrom: configMapKeyRef: - name: vllm-disagg-gb200-dflash-config + name: vllm-disagg-gb200-dflash-bf16-config key: speculative-config # speculative-config-synthetic for benchmarking - name: VLLM_DISABLED_KERNELS value: FlashInferFP8ScaledMMLinearKernel @@ -285,9 +285,9 @@ spec: - dynamo.vllm args: - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --trust-remote-code - --tensor-parallel-size - '1' @@ -340,7 +340,7 @@ spec: - name: SPECULATIVE_CONFIG valueFrom: configMapKeyRef: - name: vllm-disagg-gb200-dflash-config + name: vllm-disagg-gb200-dflash-bf16-config key: speculative-config # speculative-config-synthetic for benchmarking - name: VLLM_DISABLED_KERNELS value: FlashInferFP8ScaledMMLinearKernel diff --git a/recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dspark/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dspark-bf16/deploy.yaml similarity index 94% rename from recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dspark/deploy.yaml rename to recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dspark-bf16/deploy.yaml index 7e1c8a78376f..0164fb875bb8 100644 --- a/recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dspark/deploy.yaml +++ b/recipes/nemotron-3.5-lightning/vllm/disagg-gb200-dspark-bf16/deploy.yaml @@ -4,19 +4,19 @@ apiVersion: v1 kind: ConfigMap metadata: - name: vllm-disagg-gb200-dspark-config + name: vllm-disagg-gb200-dspark-bf16-config data: speculative-config: |- { "method": "dspark", - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark", + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-DSpark", "num_speculative_tokens": 1, "attention_backend": "TRITON_ATTN" } speculative-config-synthetic: |- { "method": "dspark", - "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark", + "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16-DSpark", "num_speculative_tokens": 1, "rejection_sample_method": "synthetic", "synthetic_acceptance_length": 1.83, @@ -26,9 +26,9 @@ data: apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment metadata: - name: vllm-disagg-gb200-dspark + name: vllm-disagg-gb200-dspark-bf16 labels: - app.kubernetes.io/name: vllm-disagg-gb200-dspark + app.kubernetes.io/name: vllm-disagg-gb200-dspark-bf16 app.kubernetes.io/part-of: dynamo spec: backendFramework: vllm @@ -139,9 +139,9 @@ spec: - -m - dynamo.vllm - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --trust-remote-code - --tensor-parallel-size - '1' @@ -194,7 +194,7 @@ spec: - name: SPECULATIVE_CONFIG valueFrom: configMapKeyRef: - name: vllm-disagg-gb200-dspark-config + name: vllm-disagg-gb200-dspark-bf16-config key: speculative-config # speculative-config-synthetic for benchmarking - name: VLLM_DISABLED_KERNELS value: FlashInferFP8ScaledMMLinearKernel @@ -287,9 +287,9 @@ spec: - -m - dynamo.vllm - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 - --trust-remote-code - --tensor-parallel-size - '1' @@ -342,7 +342,7 @@ spec: - name: SPECULATIVE_CONFIG valueFrom: configMapKeyRef: - name: vllm-disagg-gb200-dspark-config + name: vllm-disagg-gb200-dspark-bf16-config key: speculative-config # speculative-config-synthetic for benchmarking - name: VLLM_DISABLED_KERNELS value: FlashInferFP8ScaledMMLinearKernel diff --git a/recipes/nemotron-3.5-lightning/vllm/disagg-h100-mtp/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/disagg-h100-mtp/deploy.yaml deleted file mode 100644 index b625583c214c..000000000000 --- a/recipes/nemotron-3.5-lightning/vllm/disagg-h100-mtp/deploy.yaml +++ /dev/null @@ -1,374 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -apiVersion: v1 -kind: ConfigMap -metadata: - name: vllm-disagg-h100-mtp-config -data: - speculative-config: |- - { - "method": "mtp", - "num_speculative_tokens": 5 - } - speculative-config-synthetic: |- - { - "method": "mtp", - "num_speculative_tokens": 5, - "rejection_sample_method": "synthetic", - "synthetic_acceptance_length": 3.421 - } ---- -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: vllm-disagg-h100-mtp - labels: - app.kubernetes.io/name: vllm-disagg-h100-mtp - app.kubernetes.io/part-of: dynamo -spec: - backendFramework: vllm - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - spec: - imagePullSecrets: - - name: your-image-pull-secret - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.0-nemotron-3.5-lightning-dev.1 - imagePullPolicy: IfNotPresent - command: - - python3 - args: - - -m - - dynamo.frontend - - --trust-remote-code - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: DYN_HTTP_BODY_LIMIT_MB - value: '200' - - name: HF_HOME - value: /model-cache - - name: HF_HUB_OFFLINE - value: '1' - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - resources: {} - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 60 - failureThreshold: 60 - volumeMounts: - - name: shared-model-cache - mountPath: /model-cache - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: VllmPrefillWorker - type: prefill - replicas: 1 - sharedMemorySize: 40Gi - podTemplate: - spec: - imagePullSecrets: - - name: your-image-pull-secret - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.product - operator: In - values: - - NVIDIA-H100-80GB-HBM3 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: 'true' - effect: NoSchedule - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: dshm - emptyDir: - medium: Memory - sizeLimit: 40Gi - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.0-nemotron-3.5-lightning-dev.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace/ - command: - - python3 - args: - - -m - - dynamo.vllm - - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 - - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 - - --trust-remote-code - - --tensor-parallel-size - - '1' - - --max-num-seqs - - '32' - - --max-num-batched-tokens - - '32768' - - --async-scheduling - - --enable-prefix-caching - - --mamba-cache-mode - - align - - --no-disable-hybrid-kv-cache-manager - - --dyn-tool-call-parser - - nemotron_nano - - --dyn-reasoning-parser - - nemotron_nano - - --reasoning-parser - - nemotron_v3 - - --mamba-backend - - flashinfer - - --mamba-ssm-cache-dtype - - float16 - - --enable-mamba-cache-stochastic-rounding - - --mamba-cache-philox-rounds - - '5' - - --speculative-config - - $(SPECULATIVE_CONFIG) - - --kv-transfer-config - - '{"kv_connector":"NixlConnector","kv_role":"kv_producer","kv_connector_extra_config":{"backends":["LIBFABRIC"]}}' - - --disaggregation-mode - - prefill - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: HF_HUB_OFFLINE - value: '1' - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: VLLM_CONFIG_ROOT - value: /tmp/vllm-config - - name: VLLM_CACHE_ROOT - value: /tmp/vllm-cache - - name: SPECULATIVE_CONFIG - valueFrom: - configMapKeyRef: - name: vllm-disagg-h100-mtp-config - key: speculative-config - - name: NCCL_IB_DISABLE - value: '1' - - name: NIXL_LOG_LEVEL - value: INFO - # Assumes AWS EFA userspace is present in the image or installed by cluster bootstrap. - # If EFA is unpacked under /opt/amazon/efa without ldconfig registration, - # add PATH and LD_LIBRARY_PATH entries for that location here. - # This manifest does not download or install EFA packages at pod startup. - - name: FI_PROVIDER - value: efa - - name: FI_EFA_USE_DEVICE_RDMA - value: '1' - - name: FI_EFA_FORK_SAFE - value: '1' - - name: FI_EFA_ENABLE_SHM_TRANSFER - value: '0' - - name: FI_HMEM_CUDA_USE_DMABUF - value: '1' - - name: NIXL_LIBFABRIC_USE_DEVICE_RDMA - value: '1' - - name: UCX_RCACHE_MAX_UNRELEASED - value: '1024' - - name: VLLM_SSM_CONV_STATE_LAYOUT - value: DS - - name: PYTHONHASHSEED - value: '0' - resources: - requests: - memory: 128Gi - nvidia.com/gpu: '1' - vpc.amazonaws.com/efa: '4' - limits: - nvidia.com/gpu: '1' - vpc.amazonaws.com/efa: '4' - securityContext: - runAsUser: 0 - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 30 - timeoutSeconds: 20 - failureThreshold: 120 - volumeMounts: - - name: shared-model-cache - mountPath: /model-cache - - name: dshm - mountPath: /dev/shm - - name: VllmDecodeWorker - type: decode - replicas: 1 - sharedMemorySize: 40Gi - podTemplate: - spec: - imagePullSecrets: - - name: your-image-pull-secret - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.product - operator: In - values: - - NVIDIA-H100-80GB-HBM3 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: 'true' - effect: NoSchedule - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: dshm - emptyDir: - medium: Memory - sizeLimit: 40Gi - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.0-nemotron-3.5-lightning-dev.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace/ - command: - - python3 - args: - - -m - - dynamo.vllm - - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 - - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 - - --trust-remote-code - - --tensor-parallel-size - - '1' - - --max-num-seqs - - '32' - - --max-num-batched-tokens - - '32768' - - --async-scheduling - - --enable-prefix-caching - - --mamba-cache-mode - - align - - --no-disable-hybrid-kv-cache-manager - - --dyn-tool-call-parser - - nemotron_nano - - --dyn-reasoning-parser - - nemotron_nano - - --reasoning-parser - - nemotron_v3 - - --mamba-backend - - flashinfer - - --mamba-ssm-cache-dtype - - float16 - - --enable-mamba-cache-stochastic-rounding - - --mamba-cache-philox-rounds - - '5' - - --speculative-config - - $(SPECULATIVE_CONFIG) - - --kv-transfer-config - - '{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"backends":["LIBFABRIC"]}}' - - --disaggregation-mode - - decode - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: HF_HUB_OFFLINE - value: '1' - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: VLLM_CONFIG_ROOT - value: /tmp/vllm-config - - name: VLLM_CACHE_ROOT - value: /tmp/vllm-cache - - name: SPECULATIVE_CONFIG - valueFrom: - configMapKeyRef: - name: vllm-disagg-h100-mtp-config - key: speculative-config # speculative-config-synthetic for benchmarking - - name: NCCL_IB_DISABLE - value: '1' - - name: NIXL_LOG_LEVEL - value: INFO - # Assumes AWS EFA userspace is present in the image or installed by cluster bootstrap. - # If EFA is unpacked under /opt/amazon/efa without ldconfig registration, - # add PATH and LD_LIBRARY_PATH entries for that location here. - # This manifest does not download or install EFA packages at pod startup. - - name: FI_PROVIDER - value: efa - - name: FI_EFA_USE_DEVICE_RDMA - value: '1' - - name: FI_EFA_FORK_SAFE - value: '1' - - name: FI_EFA_ENABLE_SHM_TRANSFER - value: '0' - - name: FI_HMEM_CUDA_USE_DMABUF - value: '1' - - name: NIXL_LIBFABRIC_USE_DEVICE_RDMA - value: '1' - - name: UCX_RCACHE_MAX_UNRELEASED - value: '1024' - - name: VLLM_SSM_CONV_STATE_LAYOUT - value: DS - - name: PYTHONHASHSEED - value: '0' - resources: - requests: - memory: 128Gi - nvidia.com/gpu: '1' - vpc.amazonaws.com/efa: '4' - limits: - nvidia.com/gpu: '1' - vpc.amazonaws.com/efa: '4' - securityContext: - runAsUser: 0 - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 30 - timeoutSeconds: 20 - failureThreshold: 120 - volumeMounts: - - name: shared-model-cache - mountPath: /model-cache - - name: dshm - mountPath: /dev/shm diff --git a/recipes/nemotron-3.5-lightning/vllm/disagg-h200-mtp/deploy.yaml b/recipes/nemotron-3.5-lightning/vllm/disagg-h200-mtp/deploy.yaml deleted file mode 100644 index 9868aa39603c..000000000000 --- a/recipes/nemotron-3.5-lightning/vllm/disagg-h200-mtp/deploy.yaml +++ /dev/null @@ -1,346 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -apiVersion: v1 -kind: ConfigMap -metadata: - name: vllm-disagg-h200-mtp-config -data: - speculative-config: |- - { - "method": "mtp", - "num_speculative_tokens": 5 - } - speculative-config-synthetic: |- - { - "method": "mtp", - "num_speculative_tokens": 5, - "rejection_sample_method": "synthetic", - "synthetic_acceptance_length": 3.421 - } ---- -apiVersion: nvidia.com/v1beta1 -kind: DynamoGraphDeployment -metadata: - name: vllm-disagg-h200-mtp - labels: - app.kubernetes.io/name: vllm-disagg-h200-mtp - app.kubernetes.io/part-of: dynamo -spec: - backendFramework: vllm - components: - - name: Frontend - type: frontend - replicas: 1 - podTemplate: - spec: - imagePullSecrets: - - name: your-image-pull-secret - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.0-nemotron-3.5-lightning-dev.1 - imagePullPolicy: IfNotPresent - command: - - python3 - args: - - -m - - dynamo.frontend - - --trust-remote-code - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: DYN_HTTP_BODY_LIMIT_MB - value: '200' - - name: HF_HOME - value: /model-cache - - name: HF_HUB_OFFLINE - value: '1' - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - resources: {} - startupProbe: - httpGet: - path: /health - port: 8000 - periodSeconds: 10 - timeoutSeconds: 60 - failureThreshold: 60 - volumeMounts: - - name: shared-model-cache - mountPath: /model-cache - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: VllmPrefillWorker - type: prefill - replicas: 1 - sharedMemorySize: 40Gi - podTemplate: - spec: - imagePullSecrets: - - name: your-image-pull-secret - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.product - operator: In - values: - - NVIDIA-H200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: 'true' - effect: NoSchedule - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: dshm - emptyDir: - medium: Memory - sizeLimit: 40Gi - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.0-nemotron-3.5-lightning-dev.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace/ - command: - - python3 - - -m - - dynamo.vllm - args: - - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 - - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 - - --trust-remote-code - - --tensor-parallel-size - - '1' - - --max-num-seqs - - '32' - - --async-scheduling - - --no-enable-flashinfer-autotune - - --enable-prefix-caching - - --mamba-cache-mode - - align - - --no-disable-hybrid-kv-cache-manager - - --mamba-backend - - flashinfer - - --mamba-ssm-cache-dtype - - float16 - - --enable-mamba-cache-stochastic-rounding - - --mamba-cache-philox-rounds - - '5' - - --dyn-tool-call-parser - - nemotron_nano - - --dyn-reasoning-parser - - nemotron_nano - - --reasoning-parser - - nemotron_v3 - - --speculative-config=$(SPECULATIVE_CONFIG) - - --kv-transfer-config - - '{"kv_connector":"NixlConnector","kv_role":"kv_producer"}' - - --disaggregation-mode - - prefill - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: HF_HUB_OFFLINE - value: '1' - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: VLLM_CONFIG_ROOT - value: /tmp/vllm-config - - name: VLLM_CACHE_ROOT - value: /tmp/vllm-cache - - name: SPECULATIVE_CONFIG - valueFrom: - configMapKeyRef: - name: vllm-disagg-h200-mtp-config - key: speculative-config # speculative-config-synthetic for benchmarking - - name: NCCL_IB_DISABLE - value: '0' - - name: NIXL_LOG_LEVEL - value: INFO - - name: UCX_TLS - value: ^cuda_ipc - - name: UCX_RNDV_SCHEME - value: get_zcopy - - name: UCX_MAX_RNDV_RAILS - value: '4' - - name: VLLM_SSM_CONV_STATE_LAYOUT - value: DS - - name: PYTHONHASHSEED - value: '0' - resources: - requests: - memory: 128Gi - nvidia.com/gpu: '1' - rdma/ib: '1' - limits: - nvidia.com/gpu: '1' - rdma/ib: '1' - securityContext: - runAsUser: 0 - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 30 - timeoutSeconds: 20 - failureThreshold: 120 - volumeMounts: - - name: shared-model-cache - mountPath: /model-cache - - name: dshm - mountPath: /dev/shm - - name: VllmDecodeWorker - type: decode - replicas: 1 - sharedMemorySize: 40Gi - podTemplate: - spec: - imagePullSecrets: - - name: your-image-pull-secret - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: nvidia.com/gpu.product - operator: In - values: - - NVIDIA-H200 - tolerations: - - key: nvidia.com/gpu - operator: Equal - value: 'true' - effect: NoSchedule - volumes: - - name: shared-model-cache - persistentVolumeClaim: - claimName: shared-model-cache - - name: dshm - emptyDir: - medium: Memory - sizeLimit: 40Gi - containers: - - name: main - image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.0-nemotron-3.5-lightning-dev.1 - imagePullPolicy: IfNotPresent - workingDir: /workspace/ - command: - - python3 - - -m - - dynamo.vllm - args: - - --model - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 - - --served-model-name - - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 - - --trust-remote-code - - --tensor-parallel-size - - '1' - - --max-num-seqs - - '32' - - --async-scheduling - - --no-enable-flashinfer-autotune - - --enable-prefix-caching - - --mamba-cache-mode - - align - - --no-disable-hybrid-kv-cache-manager - - --mamba-backend - - flashinfer - - --mamba-ssm-cache-dtype - - float16 - - --enable-mamba-cache-stochastic-rounding - - --mamba-cache-philox-rounds - - '5' - - --dyn-tool-call-parser - - nemotron_nano - - --dyn-reasoning-parser - - nemotron_nano - - --reasoning-parser - - nemotron_v3 - - --speculative-config=$(SPECULATIVE_CONFIG) - - --kv-transfer-config - - '{"kv_connector":"NixlConnector","kv_role":"kv_consumer"}' - - --disaggregation-mode - - decode - env: - - name: POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: HF_HOME - value: /model-cache - - name: HF_HUB_OFFLINE - value: '1' - - name: HF_MODULES_CACHE - value: /tmp/hf_modules - - name: TRITON_CACHE_DIR - value: /tmp/.triton-cache - - name: VLLM_CONFIG_ROOT - value: /tmp/vllm-config - - name: VLLM_CACHE_ROOT - value: /tmp/vllm-cache - - name: SPECULATIVE_CONFIG - valueFrom: - configMapKeyRef: - name: vllm-disagg-h200-mtp-config - key: speculative-config # speculative-config-synthetic for benchmarking - - name: NCCL_IB_DISABLE - value: '0' - - name: NIXL_LOG_LEVEL - value: INFO - - name: UCX_TLS - value: ^cuda_ipc - - name: UCX_RNDV_SCHEME - value: get_zcopy - - name: UCX_MAX_RNDV_RAILS - value: '4' - - name: VLLM_SSM_CONV_STATE_LAYOUT - value: DS - - name: PYTHONHASHSEED - value: '0' - resources: - requests: - memory: 128Gi - nvidia.com/gpu: '1' - rdma/ib: '1' - limits: - nvidia.com/gpu: '1' - rdma/ib: '1' - securityContext: - runAsUser: 0 - capabilities: - add: - - IPC_LOCK - - SYS_RESOURCE - startupProbe: - httpGet: - path: /live - port: 9090 - periodSeconds: 30 - timeoutSeconds: 20 - failureThreshold: 120 - volumeMounts: - - name: shared-model-cache - mountPath: /model-cache - - name: dshm - mountPath: /dev/shm