ci(recipes): benchmark DeepSeek V4 Pro disaggregated nightly

.github/ci/deepseek-v4-pro-disagg/kustomization.yaml

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: dynamo-nightly-dsv4
+resources:
+- ../../../recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/deploy.yaml
+- ../../../recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml
+patches:
+- path: patch-deploy.yaml
+- path: patch-perf.yaml

.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml

@@ -0,0 +1,94 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: nvidia.com/v1alpha1
+kind: DynamoGraphDeployment
+metadata:
+  name: dsv4-pro-disagg
+spec:
+  priorityClassName: ci-nightly-high-priority
+  pvcs:
+  - name: shared-model-cache
+    create: false
+  services:
+    Frontend:
+      extraPodMetadata: &podMetadata
+        annotations:
+          nvidia.com/disable-image-pull-secret-discovery: "true"
+      volumeMounts: &modelVolumeMounts
+      - name: shared-model-cache
+        mountPoint: /opt/models
+      extraPodSpec:
+        priorityClassName: ci-nightly-high-priority
+        imagePullSecrets:
+        - name: acr-token-secret
+        nodeSelector: &arm64NodeSelector
+          kubernetes.io/arch: arm64
+        tolerations: &gpuTolerations
+        - key: dedicated
+          operator: Equal
+          value: user-workload
+          effect: NoExecute
+        - key: nvidia.com/gpu
+          operator: Exists
+          effect: NoSchedule
+        - key: kubernetes.io/arch
+          operator: Equal
+          value: arm64
+          effect: NoSchedule
+        affinity:
+          nodeAffinity: &gb200NodeAffinity
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+              - matchExpressions:
+                - key: cloud.google.com/gke-nodepool
+                  operator: In
+                  values: [customer-gpu-o7v, customer-gpu-w0e]
+    prefill:
+      extraPodMetadata: *podMetadata
+      envs: &workerEnvs
+      - name: DYN_HEALTH_CHECK_REQUEST_TIMEOUT
+        value: "1200"
+      # vLLM reads per-transfer NIXL telemetry. The NOP exporter keeps that
+      # data available without binding one Prometheus port per DP process.
+      - name: NIXL_TELEMETRY_ENABLE
+        value: "y"
+      - name: NIXL_TELEMETRY_EXPORTER
+        value: NOP
+      volumeMounts: *modelVolumeMounts
+      extraPodSpec:
+        priorityClassName: ci-nightly-high-priority
+        mainContainer: &workerMainContainer
+          # DeepSeek V4 Pro loads an 805 GiB checkpoint from Lustre. Preserve
+          # the first cold-start attempt instead of restarting it at 90 minutes.
+          startupProbe:
+            failureThreshold: 900
+        imagePullSecrets:
+        - name: acr-token-secret
+        nodeSelector: *arm64NodeSelector
+        tolerations: *gpuTolerations
+        affinity:
+          nodeAffinity: *gb200NodeAffinity
+          podAffinity: &workerPodAffinity
+            requiredDuringSchedulingIgnoredDuringExecution:
+            - topologyKey: nvidia.com/gpu.clique
+              matchLabelKeys: [nvidia.com/dynamo-graph-deployment-name]
+              labelSelector:
+                matchExpressions:
+                - key: nvidia.com/dynamo-component-type
+                  operator: In
+                  values: [prefill, decode]
+    decode:
+      extraPodMetadata: *podMetadata
+      envs: *workerEnvs
+      volumeMounts: *modelVolumeMounts
+      extraPodSpec:
+        priorityClassName: ci-nightly-high-priority
+        mainContainer: *workerMainContainer
+        imagePullSecrets:
+        - name: acr-token-secret
+        nodeSelector: *arm64NodeSelector
+        tolerations: *gpuTolerations
+        affinity:
+          nodeAffinity: *gb200NodeAffinity
+          podAffinity: *workerPodAffinity

.github/ci/deepseek-v4-pro-disagg/patch-perf.yaml

@@ -0,0 +1,33 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: dsv4-pro-disagg-bench
+spec:
+  backoffLimit: 0
+  activeDeadlineSeconds: 7200
+  template:
+    spec:
+      affinity: null
+      priorityClassName: ci-nightly-high-priority
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - key: dedicated
+        operator: Equal
+        value: user-workload
+        effect: NoExecute
+      containers:
+      - name: bench
+        env:
+        # Chat framing adds about 80 tokens; leave room under max-model-len=9280.
+        - name: ISL
+          value: "8064"
+        - name: CONCURRENCIES
+          value: "512"
+      volumes:
+      - name: model-cache
+        persistentVolumeClaim:
+          claimName: shared-model-cache

.github/workflows/deepseek-v4-pro-disagg-nightly.yml

@@ -0,0 +1,47 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: DeepSeek V4 Pro Disaggregated Perf
+
+on:
+  workflow_call:
+    inputs:
+      source_ref:
+        type: string
+        required: true
+      image_tag:
+        type: string
+        required: true
+      preflight_only:
+        type: boolean
+        required: false
+        default: false
+  workflow_dispatch:
+    inputs:
+      image_tag:
+        description: 'Published ACR vLLM nightly tag (<version>-ci-<sha>-vllm-runtime-nightly)'
+        type: string
+        required: true
+      preflight_only:
+        description: 'Render and server-side dry-run without deploying'
+        type: boolean
+        default: true
+
+permissions:
+  contents: read
+
+concurrency:
+  group: deepseek-v4-pro-nightly
+  cancel-in-progress: false
+  queue: max
+
+jobs:
+  perf:
+    name: DeepSeek V4 Pro disaggregated perf
+    uses: ./.github/workflows/shared-recipe-perf.yml
+    with:
+      recipe: deepseek-v4-pro-disagg
+      source_ref: ${{ inputs.source_ref || github.sha }}
+      image_tag: ${{ inputs.image_tag }}
+      preflight_only: ${{ inputs.preflight_only || false }}
+    secrets: inherit

.github/workflows/deepseek-v4-pro-nightly.yml

@@ -29,9 +29,11 @@ permissions:
 concurrency:
   group: deepseek-v4-pro-nightly
   cancel-in-progress: false
+  queue: max
 
 jobs:
   perf:
+    name: DeepSeek V4 Pro aggregated perf
     uses: ./.github/workflows/shared-recipe-perf.yml
     with:
       recipe: deepseek-v4-pro-agg

.github/workflows/nightly-ci.yml

@@ -537,7 +537,7 @@ jobs:
     secrets: inherit
 
   deepseek-v4-pro-agg-perf:
-    name: DeepSeek V4 Pro aggregated perf
+    name: recipes
     needs: [compute-release-mode, resolve-source-sha, vllm-build, vllm-copy-to-acr]
     if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
     uses: ./.github/workflows/deepseek-v4-pro-nightly.yml
@@ -546,6 +546,16 @@ jobs:
       image_tag: ${{ needs.vllm-copy-to-acr.outputs.image_tag }}
     secrets: inherit
 
+  deepseek-v4-pro-disagg-perf:
+    name: recipes
+    needs: [compute-release-mode, resolve-source-sha, vllm-build, vllm-copy-to-acr]
+    if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
+    uses: ./.github/workflows/deepseek-v4-pro-disagg-nightly.yml
+    with:
+      source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
+      image_tag: ${{ needs.vllm-copy-to-acr.outputs.image_tag }}
+    secrets: inherit
+
   sglang-copy-to-acr:
     name: sglang-runtime-nightly
     needs: [compute-release-mode, sglang-build]
@@ -1700,7 +1710,7 @@ jobs:
   ############################## SLACK NOTIFICATION ##############################
   notify-slack:
     if: always()
-    needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, sidecar-build, sidecar-vllm-test, sidecar-sglang-test, sidecar-trtllm-test, deploy-cleanup, deepseek-v4-pro-agg-perf, rust-tests, kimi-k25-agg-perf]
+    needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, sidecar-build, sidecar-vllm-test, sidecar-sglang-test, sidecar-trtllm-test, deploy-cleanup, deepseek-v4-pro-agg-perf, deepseek-v4-pro-disagg-perf, rust-tests, kimi-k25-agg-perf]
     permissions:
       contents: read
       actions: read   # grant the reusable notifier read access to list this run's jobs

.github/workflows/shared-recipe-perf.yml

@@ -7,7 +7,7 @@ on:
   workflow_call:
     inputs:
       recipe:
-        description: 'Kustomize overlay: kimi-k25-agg or deepseek-v4-pro-agg'
+        description: 'Kustomize overlay: kimi-k25-agg, deepseek-v4-pro-agg, or deepseek-v4-pro-disagg'
         type: string
         required: true
       source_ref:
@@ -56,24 +56,31 @@ jobs:
           case "$RECIPE" in
             kimi-k25-agg)
               dgd=kimi-k25-agg-kv-eagle
-              worker=agg
+              workers=agg
               benchmark=kimi-k25-agg-kv-eagle-bench
               container=perf
               resources=dgd,job,configmap,computedomain
               ;;
             deepseek-v4-pro-agg)
               dgd=dsv4-pro-agg
-              worker=worker
+              workers=worker
               benchmark=dsv4-bench
               container=bench
               resources=dgd,job,computedomain
               ;;
+            deepseek-v4-pro-disagg)
+              dgd=dsv4-pro-disagg
+              workers=prefill,decode
+              benchmark=dsv4-pro-disagg-bench
+              container=bench
+              resources=dgd,job,computedomain
+              ;;
             *) echo "Unsupported recipe: $RECIPE" >&2; exit 1 ;;
           esac
           mkdir -p "$RUNNER_TEMP/$RUN_KEY-artifacts"
           {
             echo "DGD=$dgd"
-            echo "WORKER=$worker"
+            echo "WORKERS=$workers"
             echo "BENCHMARK=$benchmark"
             echo "PERF_CONTAINER=$container"
             echo "RESOURCE_KINDS=$resources"
@@ -100,8 +107,11 @@ jobs:
             .manifests[] | select(.platform.os == "linux" and .platform.architecture == "arm64") | .digest')
           export RUNTIME_IMAGE="$image@$digest"
           yq -i '.spec.services[].extraPodSpec.mainContainer.image = strenv(RUNTIME_IMAGE) |
-            .spec.services[].extraPodSpec.imagePullSecrets = [{"name": strenv(ACR_SECRET)}] |
-            .spec.services[strenv(WORKER)].envFromSecret = strenv(HF_SECRET)' "$OVERLAY/patch-deploy.yaml"
+            .spec.services[].extraPodSpec.imagePullSecrets = [{"name": strenv(ACR_SECRET)}]' "$OVERLAY/patch-deploy.yaml"
+          for worker in ${WORKERS//,/ }; do
+            export WORKER="$worker"
+            yq -i '.spec.services[strenv(WORKER)].envFromSecret = strenv(HF_SECRET)' "$OVERLAY/patch-deploy.yaml"
+          done
           yq -i '.commonLabels.ci-run = strenv(RUN_KEY) |
             .commonLabels.ci-workflow = strenv(RECIPE)' "$OVERLAY/kustomization.yaml"
           kubectl kustomize "$OVERLAY" --load-restrictor LoadRestrictionsNone > "$ARTIFACTS/rendered.yaml"
@@ -112,7 +122,7 @@ jobs:
           if [[ "$RECIPE" == kimi-k25-agg ]]; then
             yq -i 'with(select(.kind == "Job" and .metadata.name == strenv(BENCHMARK));
               (.spec.template.spec.containers[].env[] | select(.name == "ROOT_ARTIFACT_DIR").value) = "/model-cache/perf/" + strenv(RUN_KEY))' "$ARTIFACTS/rendered.yaml"
-          elif [[ "$RECIPE" == deepseek-v4-pro-agg ]]; then
+          elif [[ "$RECIPE" == deepseek-v4-pro-agg || "$RECIPE" == deepseek-v4-pro-disagg ]]; then
             yq -i 'with(select(.kind == "Job" and .metadata.name == strenv(BENCHMARK));
               .spec.template.spec.containers[0].command[2] = strenv(PERF_CLEANUP) + "\n" + .spec.template.spec.containers[0].command[2])' "$ARTIFACTS/rendered.yaml"
           fi
@@ -176,7 +186,9 @@ jobs:
           "${k[@]}" create -f "$ARTIFACTS/prerequisites.yaml"
       - name: Deploy and wait for readiness
         if: ${{ !inputs.preflight_only }}
-        timeout-minutes: 105
+        # DeepSeek V4 Pro disagg needs extra margin for four-node scheduling
+        # and loading its 805 GiB checkpoint from the shared Lustre volume.
+        timeout-minutes: ${{ inputs.recipe == 'deepseek-v4-pro-disagg' && 180 || 105 }}
         run: |
           k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
           selector=nvidia.com/dynamo-graph-deployment-name=$DGD
@@ -217,6 +229,12 @@ jobs:
           if [[ -n "$("${k[@]}" get dgd -l "$selector" -o name)" ]]; then
             "${k[@]}" logs -l "nvidia.com/dynamo-graph-deployment-name=$DGD" \
               --all-containers --prefix --tail=1000 > "$ARTIFACTS/deployment.log" 2>&1 || true
+            for worker_pod in $("${k[@]}" get pods \
+              -l "nvidia.com/dynamo-graph-deployment-name=$DGD,nvidia.com/dynamo-component-type in (worker,prefill,decode)" \
+              -o name); do
+              "${k[@]}" logs "$worker_pod" -c main --prefix --previous --tail=1000 \
+                >> "$ARTIFACTS/deployment-previous.log" 2>&1 || true
+            done
           fi
           for job in $("${k[@]}" get job -l "$selector" -o name); do
             "${k[@]}" logs "$job" --all-containers > "$ARTIFACTS/${job##*/}.log" 2>&1 || true
@@ -229,7 +247,7 @@ jobs:
           fi
           pod=$("${k[@]}" get pods -l "$pod_selector" -o json |
             jq -r 'first(.items[] | select(.metadata.deletionTimestamp == null and .status.phase == "Running")) | .metadata.name // empty')
-          if [[ -n "$pod" ]]; then
+          if [[ -n "$pod" ]] && "${k[@]}" exec -c "$container" "$pod" -- test -d "$directory"; then
             mkdir -p "$ARTIFACTS/perf"
             "${k[@]}" exec -c "$container" "$pod" -- \
               tar -C "$directory" --exclude=inputs.json -cf - . | tar -C "$ARTIFACTS/perf" -xf -