ai-dynamo · scheckerNV · Sep 1, 2026
@@ -66,3 +66,17 @@ The following table contains BF16 rows for B200 and GB200 recipes.
 | BF16 | `trtllm/agg-b200-mtp-bf16/deploy.yaml` | TensorRT-LLM | agg | B200 | Single worker | MTP | 3 | 2.874 | 18 | 3541 | 3411 | 130 | 244.95 | 3295.50 | 183.08 |
 | BF16 | `trtllm/agg-gb200-bf16/deploy.yaml` | TensorRT-LLM | agg | GB200 | Single worker | None | 0 | 0.0 | 18 | 3541 | 3411 | 130 | 216.77 | 1985.84 | 110.32 |
 | BF16 | `trtllm/agg-gb200-mtp-bf16/deploy.yaml` | TensorRT-LLM | agg | GB200 | Single worker | MTP | 3 | 2.874 | 18 | 3541 | 3411 | 130 | 231.34 | 3556.40 | 197.58 |
+
+### RTX PRO 6000
+
+| Precision | Recipe | FW | Mode | GPU | Routing | Spec method | Spec tok | Synthetic AL | Concurrency | Requests | Valid requests | Errors | TTFT p50 (ms) | Output tok/s/GPU | Tok/s/user |
+|----------|--------|----|------|-----|---------|-------------|----------|--------------|-------------|----------|----------------|--------|---------------|------------------|------------|
+| NVFP4 | `vllm/agg-rtxpro6000-dspark/deploy.yaml` | vLLM | agg | RTX PRO 6000 | Single worker | DSpark | 7 | n/m | 20 | 3541 | 3526 | 15 | 542.50 | 776.03 | 53.42 |
+
+`Synthetic AL` is `n/m` (not measured): the value in the manifest is inherited from the H100
+recipe and was not re-derived on sm_120. This row shows 15 errors rather than 130 because the
+deployment resolves `max_seq_len` to 1048576, so the long trace requests that exceed a
+262144-token context on the other SKUs completed here.
+
+RTX PRO 6000 is single-node only (PCIe, no NVLink) and uses NVFP4 rather than the BF16 weights
+used on B200 and GB200; BF16 measured 3.3x slower on sm_120.
@@ -7,7 +7,9 @@ SPDX-License-Identifier: Apache-2.0
 
 Dynamo vLLM recipes for Nemotron 3.5 Lightning. H100 and H200 use
 `nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4`, while B200 and GB200 use
-`nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16`. Aggregate and
+`nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16`. RTX PRO 6000 (sm_120) uses NVFP4:
+BF16 measured 3.3x slower there, the opposite of the B200 result, because decode is
+memory-bandwidth-bound and RTX PRO 6000 uses GDDR7 rather than HBM. Aggregate and
 disaggregated configurations are available as explained below.
 
 ## Configurations
@@ -18,6 +20,7 @@ disaggregated configurations are available as explained below.
 | H200 | `agg-h200-{mtp,dflash,dspark}` | `disagg-h200-{dflash,dspark}` | UCX over IB/RDMA |
 | B200 | `agg-b200-{mtp,dspark}-bf16` | `disagg-b200-dspark-bf16` | UCX over IB/RDMA |
 | GB200 | `agg-gb200-{mtp,dspark}-bf16` | `disagg-gb200-{dflash,dspark}-bf16` | UCX/NIXL, cluster-specific fabric resources |
+| RTX PRO 6000 | `agg-rtxpro6000-dspark` | -- | Single-node only (PCIe, no NVLink) |
 
 B200 and GB200 directories use BF16 weights and include a `-bf16` suffix, for
 example `agg-b200-dspark-bf16` and `disagg-gb200-dspark-bf16`.

@@ -0,0 +1,203 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Aggregated vLLM deployment for Nemotron 3.5 Lightning NVFP4 on 1x RTX PRO 6000 Blackwell with DSpark speculative decoding.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vllm-agg-rtxpro6000-dspark-config
+data:
+  speculative-config: |-
+    {
+      "method": "dspark",
+      "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark",
+      "num_speculative_tokens": 7,
+      "attention_backend": "TRITON_ATTN"
+    }
+  speculative-config-synthetic: |-
+    {
+      "method": "dspark",
+      "model": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark",
+      "num_speculative_tokens": 7,
+      "rejection_sample_method": "synthetic",
+      "synthetic_acceptance_length": 3.69,
+      "attention_backend": "TRITON_ATTN"
+    }
+---
+apiVersion: nvidia.com/v1beta1
+kind: DynamoGraphDeployment
+metadata:
+  name: vllm-agg-rtxpro6000-dspark
+  labels:
+    app.kubernetes.io/name: vllm-agg-rtxpro6000-dspark
+    app.kubernetes.io/part-of: dynamo
+spec:
+  backendFramework: vllm
+  components:
+    - name: Frontend
+      type: frontend
+      replicas: 1
+      podTemplate:
+        spec:
+          imagePullSecrets:
+            - name: your-image-pull-secret
+          containers:
+            - name: main
+              image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.0-nemotron-3.5-lightning-dev.1
+              imagePullPolicy: IfNotPresent
+              command:
+                - python3
+              args:
+                - -m
+                - dynamo.frontend
+                - --trust-remote-code
+              env:
+                - name: POD_UID
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: metadata.uid
+                - name: DYN_HTTP_BODY_LIMIT_MB
+                  value: "200"
+                - name: HF_HOME
+                  value: /model-cache
+                - name: HF_HUB_OFFLINE
+                  value: "1"
+                - name: HF_MODULES_CACHE
+                  value: /tmp/hf_modules
+              resources: {}
+              startupProbe:
+                httpGet:
+                  path: /health
+                  port: 8000
+                periodSeconds: 10
+                timeoutSeconds: 60
+                failureThreshold: 60
+              volumeMounts:
+                - name: shared-model-cache
+                  mountPath: /model-cache
+          volumes:
+            - name: shared-model-cache
+              persistentVolumeClaim:
+                claimName: shared-model-cache
+
+    - name: agg
+      type: worker
+      replicas: 1
+      sharedMemorySize: 40Gi
+      podTemplate:
+        spec:
+          imagePullSecrets:
+            - name: your-image-pull-secret
+          affinity:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                  - matchExpressions:
+                      - key: nvidia.com/gpu.product
+                        operator: In
+                        values:
+                          - NVIDIA-RTX-PRO-6000-Blackwell-Server-Edition
+          tolerations:
+            - key: nvidia.com/gpu
+              operator: Equal
+              value: "true"
+              effect: NoSchedule
+          volumes:
+            - name: shared-model-cache
+              persistentVolumeClaim:
+                claimName: shared-model-cache
+            - name: dshm
+              emptyDir:
+                medium: Memory
+                sizeLimit: 40Gi
+          containers:
+            - name: main
+              image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.0-nemotron-3.5-lightning-dev.1
+              imagePullPolicy: IfNotPresent
+              workingDir: /workspace/
+              command:
+                - python3
+                - -m
+                - dynamo.vllm
+              args:
+                - --model
+                - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4
+                - --served-model-name
+                - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4
+                - --trust-remote-code
+                - --tensor-parallel-size
+                - "1"
+                - --max-num-seqs
+                - "512"
+                - --max-num-batched-tokens
+                - "32768"
+                - --enable-prefix-caching
+                - --async-scheduling
+                - --mamba-backend
+                - flashinfer
+                - --mamba-ssm-cache-dtype
+                - float16
+                - --mamba-cache-mode
+                - align
+                - --enable-mamba-cache-stochastic-rounding
+                - --mamba-cache-philox-rounds
+                - "5"
+                - --dyn-tool-call-parser
+                - nemotron_nano
+                - --dyn-reasoning-parser
+                - nemotron_nano
+                - --reasoning-parser
+                - nemotron_v3
+                - --speculative-config=$(SPECULATIVE_CONFIG)
+                - --disaggregation-mode
+                - agg
+              env:
+                - name: POD_UID
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: metadata.uid
+                - name: HF_HOME
+                  value: /model-cache
+                - name: HF_HUB_OFFLINE
+                  value: "1"
+                - name: HF_MODULES_CACHE
+                  value: /tmp/hf_modules
+                - name: TRITON_CACHE_DIR
+                  value: /tmp/.triton-cache
+                - name: VLLM_CONFIG_ROOT
+                  value: /tmp/vllm-config
+                - name: VLLM_CACHE_ROOT
+                  value: /tmp/vllm-cache
+                - name: SPECULATIVE_CONFIG
+                  valueFrom:
+                    configMapKeyRef:
+                      name: vllm-agg-rtxpro6000-dspark-config
+                      key: speculative-config # speculative-config-synthetic for benchmarking
+                - name: NCCL_IB_DISABLE
+                  value: "1"
+                - name: PYTHONHASHSEED
+                  value: "0"
+              resources:
+                requests:
+                  memory: 128Gi
+                  nvidia.com/gpu: "1"
+                limits:
+                  nvidia.com/gpu: "1"
+              securityContext:
+                runAsUser: 0
+                capabilities:
+                  add:
+                    - IPC_LOCK
+                    - SYS_RESOURCE
+              startupProbe:
+                httpGet:
+                  path: /live
+                  port: 9090
+                periodSeconds: 30
+                timeoutSeconds: 20
+                failureThreshold: 120
+              volumeMounts:
+                - name: shared-model-cache
+                  mountPath: /model-cache
+                - name: dshm
+                  mountPath: /dev/shm