From a6d452d10ca910dd919ab0ce1f20f8eafc8a6dfa Mon Sep 17 00:00:00 2001
From: Lavanya <lvijayakrish@nvidia.com>
Date: Tue, 29 Sep 2026 14:59:00 -0700
Subject: [PATCH 01/14] ci(recipes): benchmark DeepSeek V4 Pro disaggregated
 nightly

Signed-off-by: Lavanya <lvijayakrish@nvidia.com>
---
 .../deepseek-v4-pro-disagg/kustomization.yaml |  16 ++
 .../deepseek-v4-pro-disagg/patch-deploy.yaml  | 136 +++++++++++++
 .../ci/deepseek-v4-pro-disagg/patch-perf.yaml |  28 +++
 .../deepseek-v4-pro-disagg-nightly.yml        | 187 ++++++++++++++++++
 .github/workflows/nightly-ci.yml              |  12 +-
 5 files changed, 378 insertions(+), 1 deletion(-)
 create mode 100644 .github/ci/deepseek-v4-pro-disagg/kustomization.yaml
 create mode 100644 .github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
 create mode 100644 .github/ci/deepseek-v4-pro-disagg/patch-perf.yaml
 create mode 100644 .github/workflows/deepseek-v4-pro-disagg-nightly.yml

diff --git a/.github/ci/deepseek-v4-pro-disagg/kustomization.yaml b/.github/ci/deepseek-v4-pro-disagg/kustomization.yaml
new file mode 100644
index 000000000000..896729f41495
--- /dev/null
+++ b/.github/ci/deepseek-v4-pro-disagg/kustomization.yaml
@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: dynamo-nightly-dsv4
+resources:
+- ../../../recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/deploy.yaml
+- ../../../recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml
+images:
+- name: python
+  newName: docker.io/library/python
+  digest: sha256:44ff437bba879d4941b710a369a8f19266aea34b29002807f0c487fabc9eec9b
+patches:
+- path: patch-deploy.yaml
+- path: patch-perf.yaml
diff --git a/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml b/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
new file mode 100644
index 000000000000..32b431a7422a
--- /dev/null
+++ b/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
@@ -0,0 +1,136 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: nvidia.com/v1alpha1
+kind: DynamoGraphDeployment
+metadata:
+  name: dsv4-pro-disagg
+spec:
+  priorityClassName: ci-nightly-high-priority
+  pvcs:
+  - name: shared-model-cache
+    create: false
+  services:
+    Frontend:
+      extraPodMetadata:
+        annotations:
+          nvidia.com/disable-image-pull-secret-discovery: "true"
+      volumeMounts:
+      - name: shared-model-cache
+        mountPoint: /opt/models
+      extraPodSpec:
+        priorityClassName: ci-nightly-high-priority
+        imagePullSecrets:
+        - name: acr-token-secret
+        nodeSelector:
+          kubernetes.io/arch: arm64
+        tolerations:
+        - key: dedicated
+          operator: Equal
+          value: user-workload
+          effect: NoExecute
+        - key: nvidia.com/gpu
+          operator: Exists
+          effect: NoSchedule
+        - key: kubernetes.io/arch
+          operator: Equal
+          value: arm64
+          effect: NoSchedule
+        affinity:
+          nodeAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+              - matchExpressions:
+                - key: cloud.google.com/gke-nodepool
+                  operator: In
+                  values: [customer-gpu-o7v, customer-gpu-w0e]
+    prefill:
+      extraPodMetadata:
+        annotations:
+          nvidia.com/disable-image-pull-secret-discovery: "true"
+      envs:
+      - name: DYN_HEALTH_CHECK_REQUEST_TIMEOUT
+        value: "1200"
+      volumeMounts:
+      - name: shared-model-cache
+        mountPoint: /opt/models
+      extraPodSpec:
+        priorityClassName: ci-nightly-high-priority
+        imagePullSecrets:
+        - name: acr-token-secret
+        nodeSelector:
+          kubernetes.io/arch: arm64
+        tolerations:
+        - key: dedicated
+          operator: Equal
+          value: user-workload
+          effect: NoExecute
+        - key: nvidia.com/gpu
+          operator: Exists
+          effect: NoSchedule
+        - key: kubernetes.io/arch
+          operator: Equal
+          value: arm64
+          effect: NoSchedule
+        affinity:
+          nodeAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+              - matchExpressions:
+                - key: cloud.google.com/gke-nodepool
+                  operator: In
+                  values: [customer-gpu-o7v, customer-gpu-w0e]
+          podAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+            - topologyKey: nvidia.com/gpu.clique
+              matchLabelKeys: [nvidia.com/dynamo-graph-deployment-name]
+              labelSelector:
+                matchExpressions:
+                - key: nvidia.com/dynamo-component-type
+                  operator: In
+                  values: [worker]
+    decode:
+      extraPodMetadata:
+        annotations:
+          nvidia.com/disable-image-pull-secret-discovery: "true"
+      envs:
+      - name: DYN_HEALTH_CHECK_REQUEST_TIMEOUT
+        value: "1200"
+      volumeMounts:
+      - name: shared-model-cache
+        mountPoint: /opt/models
+      extraPodSpec:
+        priorityClassName: ci-nightly-high-priority
+        imagePullSecrets:
+        - name: acr-token-secret
+        nodeSelector:
+          kubernetes.io/arch: arm64
+        tolerations:
+        - key: dedicated
+          operator: Equal
+          value: user-workload
+          effect: NoExecute
+        - key: nvidia.com/gpu
+          operator: Exists
+          effect: NoSchedule
+        - key: kubernetes.io/arch
+          operator: Equal
+          value: arm64
+          effect: NoSchedule
+        affinity:
+          nodeAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+              - matchExpressions:
+                - key: cloud.google.com/gke-nodepool
+                  operator: In
+                  values: [customer-gpu-o7v, customer-gpu-w0e]
+          podAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+            - topologyKey: nvidia.com/gpu.clique
+              matchLabelKeys: [nvidia.com/dynamo-graph-deployment-name]
+              labelSelector:
+                matchExpressions:
+                - key: nvidia.com/dynamo-component-type
+                  operator: In
+                  values: [worker]
diff --git a/.github/ci/deepseek-v4-pro-disagg/patch-perf.yaml b/.github/ci/deepseek-v4-pro-disagg/patch-perf.yaml
new file mode 100644
index 000000000000..43e898630fba
--- /dev/null
+++ b/.github/ci/deepseek-v4-pro-disagg/patch-perf.yaml
@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: dsv4-pro-disagg-bench
+spec:
+  template:
+    spec:
+      affinity: null
+      priorityClassName: ci-nightly-high-priority
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - key: dedicated
+        operator: Equal
+        value: user-workload
+        effect: NoExecute
+      containers:
+      - name: bench
+        env:
+        - name: CONCURRENCIES
+          value: "512"
+      volumes:
+      - name: model-cache
+        persistentVolumeClaim:
+          claimName: shared-model-cache
diff --git a/.github/workflows/deepseek-v4-pro-disagg-nightly.yml b/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
new file mode 100644
index 000000000000..8b50882b386d
--- /dev/null
+++ b/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
@@ -0,0 +1,187 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: DeepSeek V4 Pro Disaggregated Perf
+
+on:
+  workflow_call:
+    inputs:
+      source_ref:
+        type: string
+        required: true
+      image_tag:
+        type: string
+        required: true
+  workflow_dispatch:
+    inputs:
+      image_tag:
+        description: 'Published ACR vLLM nightly tag (<version>-ci-<sha>-vllm-runtime-nightly)'
+        type: string
+        required: true
+      preflight_only:
+        description: 'Render and server-side dry-run without deploying'
+        type: boolean
+        default: true
+
+permissions:
+  contents: read
+
+concurrency:
+  group: deepseek-v4-pro-nightly
+  cancel-in-progress: false
+
+jobs:
+  perf:
+    runs-on: prod-deploy-tester-v1
+    timeout-minutes: 300
+    defaults:
+      run:
+        shell: bash
+    env:
+      # Read tbot's renewed credentials; every kubectl command selects its context.
+      KUBECONFIG: /teleport/kubeconfig.yaml
+      KUBE_CONTEXT: nv-prd-dgxc.teleport.sh-dynamo-gcp-dev-02
+      NAMESPACE: dynamo-nightly-dsv4
+      RUN_KEY: ci-${{ github.run_id }}-${{ github.run_attempt }}
+      OVERLAY: .github/ci/deepseek-v4-pro-disagg
+    steps:
+      - timeout-minutes: 5
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          ref: ${{ inputs.source_ref || github.sha }}
+          persist-credentials: false
+      - name: Prepare artifacts
+        timeout-minutes: 5
+        run: |
+          mkdir -p "$RUNNER_TEMP/dsv4-disagg-artifacts"
+          echo "ARTIFACTS=$RUNNER_TEMP/dsv4-disagg-artifacts" >> "$GITHUB_ENV"
+          echo "REGISTRY_AUTH_FILE=$RUNNER_TEMP/dsv4-disagg-registry-auth.json" >> "$GITHUB_ENV"
+      - timeout-minutes: 5
+        uses: ./.github/actions/skopeo-login
+        with:
+          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
+          azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
+          azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
+      - name: Render deployment and upstream perf recipe
+        timeout-minutes: 5
+        env:
+          IMAGE_TAG: ${{ inputs.image_tag }}
+          ACR_HOST: ${{ secrets.AZURE_ACR_HOSTNAME }}
+          ACR_REPOSITORY: ${{ vars.ACR_REPOSITORY }}
+        run: |
+          image="${ACR_HOST}/${ACR_REPOSITORY}:${IMAGE_TAG}"
+          digest=$(skopeo inspect --raw "docker://$image" | jq -er '
+            .manifests[] | select(.platform.os == "linux" and .platform.architecture == "arm64") | .digest')
+          export RUNTIME_IMAGE="$image@$digest"
+          yq -i '.spec.services[].extraPodSpec.mainContainer.image = strenv(RUNTIME_IMAGE) |
+            .spec.services[].extraPodSpec.imagePullSecrets = [{"name": "acr-" + strenv(RUN_KEY)}] |
+            (.spec.services.prefill.envFromSecret, .spec.services.decode.envFromSecret) = "hf-" + strenv(RUN_KEY)' "$OVERLAY/patch-deploy.yaml"
+          yq -i '.commonLabels.ci-run = strenv(RUN_KEY)' "$OVERLAY/kustomization.yaml"
+          kubectl kustomize "$OVERLAY" --load-restrictor LoadRestrictionsNone > "$ARTIFACTS/rendered.yaml"
+          yq 'select(.kind != "Job")' "$ARTIFACTS/rendered.yaml" > "$ARTIFACTS/deploy.yaml"
+          yq 'select(.kind == "Job")' "$ARTIFACTS/rendered.yaml" > "$ARTIFACTS/perf.yaml"
+          yq -i '(.spec.template.spec.containers[0].env[] | select(.name == "ARTIFACT_ROOT").value) = "/opt/models/perf/" + strenv(RUN_KEY) |
+            .spec.template.spec.containers[0].envFrom[0].secretRef.name = "hf-" + strenv(RUN_KEY)' "$ARTIFACTS/perf.yaml"
+          git rev-parse HEAD > "$ARTIFACTS/source-sha.txt"
+      - name: Preflight and create run credentials
+        timeout-minutes: 10
+        env:
+          DRY_RUN: ${{ inputs.preflight_only && 'server' || 'none' }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          ACR_HOST: ${{ secrets.AZURE_ACR_HOSTNAME }}
+          ACR_USER: ${{ secrets.AZURE_ACR_USER }}
+          ACR_PASSWORD: ${{ secrets.AZURE_ACR_PASSWORD }}
+        run: |
+          k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
+          if [[ "$DRY_RUN" == none ]]; then
+            # Runs are serialized, so remove fixed-name resources left by an interrupted run.
+            "${k[@]}" delete --request-timeout=0 \
+              job/dsv4-pro-disagg-bench dgd/dsv4-pro-disagg computedomain/dsv4-pro-compute-domain \
+              --ignore-not-found --cascade=foreground --wait=true --timeout=5m
+          fi
+          [[ "$("${k[@]}" get pvc shared-model-cache -o jsonpath='{.status.phase}')" == Bound ]]
+          "${k[@]}" create --dry-run=server -f "$ARTIFACTS/deploy.yaml" > "$ARTIFACTS/deploy-dry-run.txt"
+          "${k[@]}" create --dry-run=server -f "$ARTIFACTS/perf.yaml" > "$ARTIFACTS/perf-dry-run.txt"
+          "${k[@]}" create secret generic "hf-$RUN_KEY" --dry-run="$DRY_RUN" -o name \
+            --from-literal=HF_TOKEN="${HF_TOKEN:?Missing HF_TOKEN}"
+          "${k[@]}" create secret docker-registry "acr-$RUN_KEY" --dry-run="$DRY_RUN" -o name \
+            --docker-server="$ACR_HOST" --docker-username="$ACR_USER" --docker-password="$ACR_PASSWORD"
+      - name: Deploy and wait for readiness
+        if: ${{ !inputs.preflight_only }}
+        timeout-minutes: 105
+        run: |
+          k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
+          selector=nvidia.com/dynamo-graph-deployment-name=dsv4-pro-disagg
+          "${k[@]}" create -f "$ARTIFACTS/deploy.yaml"
+          # Reconnect logs every two minutes to discover new pods and reload tbot certificates.
+          until "${k[@]}" wait dgd/dsv4-pro-disagg --for=condition=Ready --timeout=5s; do
+            "${k[@]}" get dgd dsv4-pro-disagg
+            "${k[@]}" get pods -l "$selector" -o wide
+            timeout 120s "${k[@]}" logs --request-timeout=0 -f -l "$selector" -c main \
+              --prefix --timestamps --since=1m --tail=20 --ignore-errors || true
+            sleep 5
+          done
+          frontend=$("${k[@]}" get pods -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-disagg,nvidia.com/dynamo-component-type=frontend -o jsonpath='{.items[0].metadata.name}')
+          echo "FRONTEND=$frontend" >> "$GITHUB_ENV"
+      - name: Run upstream AIPerf benchmark
+        if: ${{ !inputs.preflight_only }}
+        timeout-minutes: 125
+        run: |
+          k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
+          "${k[@]}" create -f "$ARTIFACTS/perf.yaml"
+          until "${k[@]}" wait job/dsv4-pro-disagg-bench --for=condition=Complete --timeout=5s; do
+            "${k[@]}" get job dsv4-pro-disagg-bench -o json |
+              jq -e 'all(.status.conditions[]?; .type != "Failed" or .status != "True")' >/dev/null
+            "${k[@]}" get pods -l job-name=dsv4-pro-disagg-bench -o wide
+            timeout 120s "${k[@]}" logs --request-timeout=0 -f job/dsv4-pro-disagg-bench \
+              --prefix --timestamps --since=1m --tail=20 --ignore-errors || true
+            sleep 5
+          done
+      - name: Collect logs and perf artifacts
+        if: always()
+        timeout-minutes: 5
+        run: |
+          k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=30s)
+          "${k[@]}" get dgd,job,computedomain -l "ci-run=$RUN_KEY" -o yaml > "$ARTIFACTS/resources.yaml" || true
+          "${k[@]}" get events --sort-by=.lastTimestamp > "$ARTIFACTS/events.txt" || true
+          if [[ -n "$("${k[@]}" get dgd -l "ci-run=$RUN_KEY" -o name)" ]]; then
+            "${k[@]}" logs -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-disagg \
+              --all-containers --prefix --tail=1000 > "$ARTIFACTS/deployment.log" 2>&1 || true
+          fi
+          if [[ -n "$("${k[@]}" get job -l "ci-run=$RUN_KEY" -o name)" ]]; then
+            "${k[@]}" logs job/dsv4-pro-disagg-bench --all-containers > "$ARTIFACTS/perf.log" 2>&1 || true
+          fi
+          if [[ -n "$FRONTEND" ]]; then
+            frontend=$("${k[@]}" get pods \
+              -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-disagg,nvidia.com/dynamo-component-type=frontend -o json |
+              jq -r 'first(.items[] | select(.metadata.deletionTimestamp == null and .status.phase == "Running")) | .metadata.name // empty')
+            if [[ -n "$frontend" ]]; then
+              "${k[@]}" cp -c main "$frontend:/opt/models/perf/$RUN_KEY/." "$ARTIFACTS/perf"
+            fi
+          fi
+      - name: Verify AIPerf results
+        if: ${{ success() && !inputs.preflight_only }}
+        timeout-minutes: 5
+        run: |
+          result=$(find "$ARTIFACTS/perf" -name profile_export_aiperf.json -print -quit)
+          jq -e '.request_count.avg == 1536 and (.error_request_count.avg // 0) == 0 and
+            .output_sequence_length.avg > 0' "$result"
+      - name: Clean up recipe resources
+        if: always()
+        timeout-minutes: 20
+        run: |
+          trap 'rm -f "$REGISTRY_AUTH_FILE"' EXIT
+          failed=0
+          kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=0 \
+            delete job,dgd,computedomain -l "ci-run=$RUN_KEY" --cascade=foreground --wait=true --timeout=15m || failed=1
+          kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=30s \
+            delete secret "hf-$RUN_KEY" "acr-$RUN_KEY" --ignore-not-found --wait=false || failed=1
+          exit "$failed"
+      - timeout-minutes: 5
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f  # v6.0.0
+        if: always()
+        with:
+          name: deepseek-v4-pro-disagg-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ env.ARTIFACTS }}
+          retention-days: 14
+          if-no-files-found: error
diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml
index df7a48ed43c0..85a8b791b33c 100644
--- a/.github/workflows/nightly-ci.yml
+++ b/.github/workflows/nightly-ci.yml
@@ -547,6 +547,16 @@ jobs:
       image_tag: ${{ needs.vllm-copy-to-acr.outputs.image_tag }}
     secrets: inherit
 
+  deepseek-v4-pro-disagg-perf:
+    name: DeepSeek V4 Pro disaggregated perf
+    needs: [compute-release-mode, resolve-source-sha, vllm-build, vllm-copy-to-acr]
+    if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
+    uses: ./.github/workflows/deepseek-v4-pro-disagg-nightly.yml
+    with:
+      source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
+      image_tag: ${{ needs.vllm-copy-to-acr.outputs.image_tag }}
+    secrets: inherit
+
   sglang-copy-to-acr:
     name: sglang-runtime-nightly
     needs: [compute-release-mode, sglang-build]
@@ -1701,7 +1711,7 @@ jobs:
   ############################## SLACK NOTIFICATION ##############################
   notify-slack:
     if: always()
-    needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, sidecar-build, sidecar-vllm-test, sidecar-sglang-test, sidecar-trtllm-test, deploy-cleanup, deepseek-v4-pro-agg-perf, rust-tests, kimi-k25-agg-perf]
+    needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, sidecar-build, sidecar-vllm-test, sidecar-sglang-test, sidecar-trtllm-test, deploy-cleanup, deepseek-v4-pro-agg-perf, deepseek-v4-pro-disagg-perf, rust-tests, kimi-k25-agg-perf]
     permissions:
       contents: read
       actions: read   # grant the reusable notifier read access to list this run's jobs

From 09aad1f3603f71cea208fd7df6287ae1f139e2cd Mon Sep 17 00:00:00 2001
From: Lavanya <lvijayakrish@nvidia.com>
Date: Tue, 29 Sep 2026 15:28:41 -0700
Subject: [PATCH 02/14] ci(recipes): enable branch-only DeepSeek disagg
 dispatch

Signed-off-by: Lavanya <lvijayakrish@nvidia.com>
---
 .github/workflows/build-on-demand.yml         | 29 ++++++++++++++++++-
 .../deepseek-v4-pro-disagg-nightly.yml        |  4 +++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-on-demand.yml b/.github/workflows/build-on-demand.yml
index 848d6d7bb6e9..08e6ff75b7f3 100644
--- a/.github/workflows/build-on-demand.yml
+++ b/.github/workflows/build-on-demand.yml
@@ -26,6 +26,18 @@ on:
         description: 'Build and push operator image, tagged with branch name'
         type: boolean
         default: false
+      run_deepseek_disagg:
+        description: 'Run only the DeepSeek V4 Pro disaggregated recipe'
+        type: boolean
+        default: false
+      deepseek_image_tag:
+        description: 'Published ACR vLLM nightly tag for the DeepSeek recipe'
+        type: string
+        default: ''
+      deepseek_preflight_only:
+        description: 'Validate the DeepSeek recipe without deploying'
+        type: boolean
+        default: true
 
 env:
   BUILDER_NAME: b-${{ github.run_id }}-${{ github.run_attempt }}
@@ -33,6 +45,7 @@ env:
 jobs:
 
   init:
+    if: ${{ !inputs.run_deepseek_disagg }}
     runs-on: ubuntu-slim
     outputs:
       builder_name: ${{ steps.export-builder-name.outputs.builder_name }}
@@ -210,6 +223,20 @@ jobs:
             echo "| \`${image_uri}\` |" >> $GITHUB_STEP_SUMMARY
           done
 
+  # ============================================================================
+  # RECIPE-ONLY MANUAL RUN
+  # ============================================================================
+
+  deepseek-disagg-perf:
+    name: DeepSeek V4 Pro disaggregated perf
+    if: ${{ inputs.run_deepseek_disagg }}
+    uses: ./.github/workflows/deepseek-v4-pro-disagg-nightly.yml
+    with:
+      source_ref: ${{ github.sha }}
+      image_tag: ${{ inputs.deepseek_image_tag }}
+      preflight_only: ${{ inputs.deepseek_preflight_only }}
+    secrets: inherit
+
   # ============================================================================
   # CLEANUP
   # ============================================================================
@@ -217,7 +244,7 @@ jobs:
   clean-k8s-builder:
     name: Clean K8s builder if exists
     runs-on: prod-default-small-v2
-    if: always()
+    if: ${{ always() && !inputs.run_deepseek_disagg }}
     needs: [init, vllm-build, sglang-build, trtllm-build, triton-build, operator]
     steps:
     - name: Checkout repository
diff --git a/.github/workflows/deepseek-v4-pro-disagg-nightly.yml b/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
index 8b50882b386d..28f1a0cfd467 100644
--- a/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
+++ b/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
@@ -12,6 +12,10 @@ on:
       image_tag:
         type: string
         required: true
+      preflight_only:
+        type: boolean
+        required: false
+        default: false
   workflow_dispatch:
     inputs:
       image_tag:

From bada9e25816b2ab57bd86ee45e12a56d297f7c50 Mon Sep 17 00:00:00 2001
From: Lavanya <lvijayakrish@nvidia.com>
Date: Tue, 29 Sep 2026 15:44:17 -0700
Subject: [PATCH 03/14] fix(ci): match disaggregated worker pod labels

Signed-off-by: Lavanya <lvijayakrish@nvidia.com>
---
 .github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml b/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
index 32b431a7422a..4828193db36d 100644
--- a/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
+++ b/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
@@ -88,7 +88,7 @@ spec:
                 matchExpressions:
                 - key: nvidia.com/dynamo-component-type
                   operator: In
-                  values: [worker]
+                  values: [prefill, decode]
     decode:
       extraPodMetadata:
         annotations:
@@ -133,4 +133,4 @@ spec:
                 matchExpressions:
                 - key: nvidia.com/dynamo-component-type
                   operator: In
-                  values: [worker]
+                  values: [prefill, decode]

From f3c98813f581af0b064c56942924d1978ab7adb8 Mon Sep 17 00:00:00 2001
From: Lavanya <lvijayakrish@nvidia.com>
Date: Tue, 29 Sep 2026 16:42:07 -0700
Subject: [PATCH 04/14] fix(ci): keep DeepSeek benchmark within context limit

Signed-off-by: Lavanya <lvijayakrish@nvidia.com>
---
 .github/ci/deepseek-v4-pro-disagg/patch-perf.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/ci/deepseek-v4-pro-disagg/patch-perf.yaml b/.github/ci/deepseek-v4-pro-disagg/patch-perf.yaml
index 43e898630fba..0ba85d68d818 100644
--- a/.github/ci/deepseek-v4-pro-disagg/patch-perf.yaml
+++ b/.github/ci/deepseek-v4-pro-disagg/patch-perf.yaml
@@ -20,6 +20,9 @@ spec:
       containers:
       - name: bench
         env:
+        # Chat framing adds about 80 tokens; leave room under max-model-len=9280.
+        - name: ISL
+          value: "8064"
         - name: CONCURRENCIES
           value: "512"
       volumes:

From aa5f1d2f1000d2035b05ffb192bb65db1dccef4b Mon Sep 17 00:00:00 2001
From: Lavanya <lvijayakrish@nvidia.com>
Date: Wed, 30 Sep 2026 06:51:22 -0700
Subject: [PATCH 05/14] chore(ci): remove temporary recipe dispatch

Signed-off-by: Lavanya <lvijayakrish@nvidia.com>
---
 .github/workflows/build-on-demand.yml | 29 +--------------------------
 1 file changed, 1 insertion(+), 28 deletions(-)

diff --git a/.github/workflows/build-on-demand.yml b/.github/workflows/build-on-demand.yml
index 08e6ff75b7f3..848d6d7bb6e9 100644
--- a/.github/workflows/build-on-demand.yml
+++ b/.github/workflows/build-on-demand.yml
@@ -26,18 +26,6 @@ on:
         description: 'Build and push operator image, tagged with branch name'
         type: boolean
         default: false
-      run_deepseek_disagg:
-        description: 'Run only the DeepSeek V4 Pro disaggregated recipe'
-        type: boolean
-        default: false
-      deepseek_image_tag:
-        description: 'Published ACR vLLM nightly tag for the DeepSeek recipe'
-        type: string
-        default: ''
-      deepseek_preflight_only:
-        description: 'Validate the DeepSeek recipe without deploying'
-        type: boolean
-        default: true
 
 env:
   BUILDER_NAME: b-${{ github.run_id }}-${{ github.run_attempt }}
@@ -45,7 +33,6 @@ env:
 jobs:
 
   init:
-    if: ${{ !inputs.run_deepseek_disagg }}
     runs-on: ubuntu-slim
     outputs:
       builder_name: ${{ steps.export-builder-name.outputs.builder_name }}
@@ -223,20 +210,6 @@ jobs:
             echo "| \`${image_uri}\` |" >> $GITHUB_STEP_SUMMARY
           done
 
-  # ============================================================================
-  # RECIPE-ONLY MANUAL RUN
-  # ============================================================================
-
-  deepseek-disagg-perf:
-    name: DeepSeek V4 Pro disaggregated perf
-    if: ${{ inputs.run_deepseek_disagg }}
-    uses: ./.github/workflows/deepseek-v4-pro-disagg-nightly.yml
-    with:
-      source_ref: ${{ github.sha }}
-      image_tag: ${{ inputs.deepseek_image_tag }}
-      preflight_only: ${{ inputs.deepseek_preflight_only }}
-    secrets: inherit
-
   # ============================================================================
   # CLEANUP
   # ============================================================================
@@ -244,7 +217,7 @@ jobs:
   clean-k8s-builder:
     name: Clean K8s builder if exists
     runs-on: prod-default-small-v2
-    if: ${{ always() && !inputs.run_deepseek_disagg }}
+    if: always()
     needs: [init, vllm-build, sglang-build, trtllm-build, triton-build, operator]
     steps:
     - name: Checkout repository

From b20553c8b81cb7e4aa8c6d32f825ff4bad892f08 Mon Sep 17 00:00:00 2001
From: Lavanya <lvijayakrish@nvidia.com>
Date: Wed, 30 Sep 2026 09:03:28 -0700
Subject: [PATCH 06/14] chore(ci): harden DeepSeek disagg nightly cleanup

Signed-off-by: Lavanya <lvijayakrish@nvidia.com>
---
 .../ci/deepseek-v4-pro-disagg/patch-perf.yaml |  2 ++
 .../deepseek-v4-pro-disagg-nightly.yml        | 25 ++++++++++++++-----
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/.github/ci/deepseek-v4-pro-disagg/patch-perf.yaml b/.github/ci/deepseek-v4-pro-disagg/patch-perf.yaml
index 0ba85d68d818..d3b434f8f704 100644
--- a/.github/ci/deepseek-v4-pro-disagg/patch-perf.yaml
+++ b/.github/ci/deepseek-v4-pro-disagg/patch-perf.yaml
@@ -6,6 +6,8 @@ kind: Job
 metadata:
   name: dsv4-pro-disagg-bench
 spec:
+  backoffLimit: 0
+  activeDeadlineSeconds: 7200
   template:
     spec:
       affinity: null
diff --git a/.github/workflows/deepseek-v4-pro-disagg-nightly.yml b/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
index 28f1a0cfd467..a28cde29d4d3 100644
--- a/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
+++ b/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
@@ -72,6 +72,8 @@ jobs:
           IMAGE_TAG: ${{ inputs.image_tag }}
           ACR_HOST: ${{ secrets.AZURE_ACR_HOSTNAME }}
           ACR_REPOSITORY: ${{ vars.ACR_REPOSITORY }}
+          PERF_CLEANUP: |
+            trap 'find "$ARTIFACT_ROOT" -type f -name inputs.json -delete || true' EXIT
         run: |
           image="${ACR_HOST}/${ACR_REPOSITORY}:${IMAGE_TAG}"
           digest=$(skopeo inspect --raw "docker://$image" | jq -er '
@@ -85,7 +87,8 @@ jobs:
           yq 'select(.kind != "Job")' "$ARTIFACTS/rendered.yaml" > "$ARTIFACTS/deploy.yaml"
           yq 'select(.kind == "Job")' "$ARTIFACTS/rendered.yaml" > "$ARTIFACTS/perf.yaml"
           yq -i '(.spec.template.spec.containers[0].env[] | select(.name == "ARTIFACT_ROOT").value) = "/opt/models/perf/" + strenv(RUN_KEY) |
-            .spec.template.spec.containers[0].envFrom[0].secretRef.name = "hf-" + strenv(RUN_KEY)' "$ARTIFACTS/perf.yaml"
+            .spec.template.spec.containers[0].envFrom[0].secretRef.name = "hf-" + strenv(RUN_KEY) |
+            .spec.template.spec.containers[0].command[2] = strenv(PERF_CLEANUP) + "\n" + .spec.template.spec.containers[0].command[2]' "$ARTIFACTS/perf.yaml"
           git rev-parse HEAD > "$ARTIFACTS/source-sha.txt"
       - name: Preflight and create run credentials
         timeout-minutes: 10
@@ -102,14 +105,20 @@ jobs:
             "${k[@]}" delete --request-timeout=0 \
               job/dsv4-pro-disagg-bench dgd/dsv4-pro-disagg computedomain/dsv4-pro-compute-domain \
               --ignore-not-found --cascade=foreground --wait=true --timeout=5m
+            "${k[@]}" delete secret -l ci-workflow=deepseek-v4-pro-disagg-nightly \
+              --ignore-not-found --wait=false
           fi
           [[ "$("${k[@]}" get pvc shared-model-cache -o jsonpath='{.status.phase}')" == Bound ]]
           "${k[@]}" create --dry-run=server -f "$ARTIFACTS/deploy.yaml" > "$ARTIFACTS/deploy-dry-run.txt"
           "${k[@]}" create --dry-run=server -f "$ARTIFACTS/perf.yaml" > "$ARTIFACTS/perf-dry-run.txt"
-          "${k[@]}" create secret generic "hf-$RUN_KEY" --dry-run="$DRY_RUN" -o name \
-            --from-literal=HF_TOKEN="${HF_TOKEN:?Missing HF_TOKEN}"
-          "${k[@]}" create secret docker-registry "acr-$RUN_KEY" --dry-run="$DRY_RUN" -o name \
-            --docker-server="$ACR_HOST" --docker-username="$ACR_USER" --docker-password="$ACR_PASSWORD"
+          "${k[@]}" create secret generic "hf-$RUN_KEY" --dry-run=client -o json \
+            --from-literal=HF_TOKEN="${HF_TOKEN:?Missing HF_TOKEN}" |
+            jq '.metadata.labels = {"ci-workflow": "deepseek-v4-pro-disagg-nightly"}' |
+            "${k[@]}" create --dry-run="$DRY_RUN" -f - -o name
+          "${k[@]}" create secret docker-registry "acr-$RUN_KEY" --dry-run=client -o json \
+            --docker-server="$ACR_HOST" --docker-username="$ACR_USER" --docker-password="$ACR_PASSWORD" |
+            jq '.metadata.labels = {"ci-workflow": "deepseek-v4-pro-disagg-nightly"}' |
+            "${k[@]}" create --dry-run="$DRY_RUN" -f - -o name
       - name: Deploy and wait for readiness
         if: ${{ !inputs.preflight_only }}
         timeout-minutes: 105
@@ -160,7 +169,11 @@ jobs:
               -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-disagg,nvidia.com/dynamo-component-type=frontend -o json |
               jq -r 'first(.items[] | select(.metadata.deletionTimestamp == null and .status.phase == "Running")) | .metadata.name // empty')
             if [[ -n "$frontend" ]]; then
-              "${k[@]}" cp -c main "$frontend:/opt/models/perf/$RUN_KEY/." "$ARTIFACTS/perf"
+              # Exclude generated prompts even if cleanup in the benchmark Job failed.
+              mkdir -p "$ARTIFACTS/perf"
+              "${k[@]}" exec -c main "$frontend" -- \
+                tar -C "/opt/models/perf/$RUN_KEY" --exclude=inputs.json -cf - . |
+                tar -C "$ARTIFACTS/perf" -xf -
             fi
           fi
       - name: Verify AIPerf results

From 974a0dd37afb3f0f01d2c2e19e50f06133c7999c Mon Sep 17 00:00:00 2001
From: Lavanya <lvijayakrish@nvidia.com>
Date: Wed, 30 Sep 2026 16:12:03 -0700
Subject: [PATCH 07/14] fix(ci): address DeepSeek nightly review feedback

Signed-off-by: Lavanya <lvijayakrish@nvidia.com>
---
 .../deepseek-v4-pro-disagg/kustomization.yaml |  4 -
 .../deepseek-v4-pro-disagg/patch-deploy.yaml  | 92 ++++---------------
 .../deepseek-v4-pro-disagg-nightly.yml        | 12 ++-
 .github/workflows/deepseek-v4-pro-nightly.yml |  1 +
 .github/workflows/nightly-ci.yml              |  4 +-
 5 files changed, 30 insertions(+), 83 deletions(-)

diff --git a/.github/ci/deepseek-v4-pro-disagg/kustomization.yaml b/.github/ci/deepseek-v4-pro-disagg/kustomization.yaml
index 896729f41495..1e6eb4611d3d 100644
--- a/.github/ci/deepseek-v4-pro-disagg/kustomization.yaml
+++ b/.github/ci/deepseek-v4-pro-disagg/kustomization.yaml
@@ -7,10 +7,6 @@ namespace: dynamo-nightly-dsv4
 resources:
 - ../../../recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/deploy.yaml
 - ../../../recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml
-images:
-- name: python
-  newName: docker.io/library/python
-  digest: sha256:44ff437bba879d4941b710a369a8f19266aea34b29002807f0c487fabc9eec9b
 patches:
 - path: patch-deploy.yaml
 - path: patch-perf.yaml
diff --git a/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml b/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
index 4828193db36d..ae2119ebb03d 100644
--- a/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
+++ b/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
@@ -12,19 +12,19 @@ spec:
     create: false
   services:
     Frontend:
-      extraPodMetadata:
+      extraPodMetadata: &podMetadata
         annotations:
           nvidia.com/disable-image-pull-secret-discovery: "true"
-      volumeMounts:
+      volumeMounts: &modelVolumeMounts
       - name: shared-model-cache
         mountPoint: /opt/models
       extraPodSpec:
         priorityClassName: ci-nightly-high-priority
         imagePullSecrets:
         - name: acr-token-secret
-        nodeSelector:
+        nodeSelector: &arm64NodeSelector
           kubernetes.io/arch: arm64
-        tolerations:
+        tolerations: &gpuTolerations
         - key: dedicated
           operator: Equal
           value: user-workload
@@ -37,7 +37,7 @@ spec:
           value: arm64
           effect: NoSchedule
         affinity:
-          nodeAffinity:
+          nodeAffinity: &gb200NodeAffinity
             requiredDuringSchedulingIgnoredDuringExecution:
               nodeSelectorTerms:
               - matchExpressions:
@@ -45,42 +45,20 @@ spec:
                   operator: In
                   values: [customer-gpu-o7v, customer-gpu-w0e]
     prefill:
-      extraPodMetadata:
-        annotations:
-          nvidia.com/disable-image-pull-secret-discovery: "true"
-      envs:
+      extraPodMetadata: *podMetadata
+      envs: &workerEnvs
       - name: DYN_HEALTH_CHECK_REQUEST_TIMEOUT
         value: "1200"
-      volumeMounts:
-      - name: shared-model-cache
-        mountPoint: /opt/models
+      volumeMounts: *modelVolumeMounts
       extraPodSpec:
         priorityClassName: ci-nightly-high-priority
         imagePullSecrets:
         - name: acr-token-secret
-        nodeSelector:
-          kubernetes.io/arch: arm64
-        tolerations:
-        - key: dedicated
-          operator: Equal
-          value: user-workload
-          effect: NoExecute
-        - key: nvidia.com/gpu
-          operator: Exists
-          effect: NoSchedule
-        - key: kubernetes.io/arch
-          operator: Equal
-          value: arm64
-          effect: NoSchedule
+        nodeSelector: *arm64NodeSelector
+        tolerations: *gpuTolerations
         affinity:
-          nodeAffinity:
-            requiredDuringSchedulingIgnoredDuringExecution:
-              nodeSelectorTerms:
-              - matchExpressions:
-                - key: cloud.google.com/gke-nodepool
-                  operator: In
-                  values: [customer-gpu-o7v, customer-gpu-w0e]
-          podAffinity:
+          nodeAffinity: *gb200NodeAffinity
+          podAffinity: &workerPodAffinity
             requiredDuringSchedulingIgnoredDuringExecution:
             - topologyKey: nvidia.com/gpu.clique
               matchLabelKeys: [nvidia.com/dynamo-graph-deployment-name]
@@ -90,47 +68,15 @@ spec:
                   operator: In
                   values: [prefill, decode]
     decode:
-      extraPodMetadata:
-        annotations:
-          nvidia.com/disable-image-pull-secret-discovery: "true"
-      envs:
-      - name: DYN_HEALTH_CHECK_REQUEST_TIMEOUT
-        value: "1200"
-      volumeMounts:
-      - name: shared-model-cache
-        mountPoint: /opt/models
+      extraPodMetadata: *podMetadata
+      envs: *workerEnvs
+      volumeMounts: *modelVolumeMounts
       extraPodSpec:
         priorityClassName: ci-nightly-high-priority
         imagePullSecrets:
         - name: acr-token-secret
-        nodeSelector:
-          kubernetes.io/arch: arm64
-        tolerations:
-        - key: dedicated
-          operator: Equal
-          value: user-workload
-          effect: NoExecute
-        - key: nvidia.com/gpu
-          operator: Exists
-          effect: NoSchedule
-        - key: kubernetes.io/arch
-          operator: Equal
-          value: arm64
-          effect: NoSchedule
+        nodeSelector: *arm64NodeSelector
+        tolerations: *gpuTolerations
         affinity:
-          nodeAffinity:
-            requiredDuringSchedulingIgnoredDuringExecution:
-              nodeSelectorTerms:
-              - matchExpressions:
-                - key: cloud.google.com/gke-nodepool
-                  operator: In
-                  values: [customer-gpu-o7v, customer-gpu-w0e]
-          podAffinity:
-            requiredDuringSchedulingIgnoredDuringExecution:
-            - topologyKey: nvidia.com/gpu.clique
-              matchLabelKeys: [nvidia.com/dynamo-graph-deployment-name]
-              labelSelector:
-                matchExpressions:
-                - key: nvidia.com/dynamo-component-type
-                  operator: In
-                  values: [prefill, decode]
+          nodeAffinity: *gb200NodeAffinity
+          podAffinity: *workerPodAffinity
diff --git a/.github/workflows/deepseek-v4-pro-disagg-nightly.yml b/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
index a28cde29d4d3..6c476300e066 100644
--- a/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
+++ b/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
@@ -36,6 +36,7 @@ concurrency:
 
 jobs:
   perf:
+    name: DeepSeek V4 Pro disaggregated perf
     runs-on: prod-deploy-tester-v1
     timeout-minutes: 300
     defaults:
@@ -46,7 +47,7 @@ jobs:
       KUBECONFIG: /teleport/kubeconfig.yaml
       KUBE_CONTEXT: nv-prd-dgxc.teleport.sh-dynamo-gcp-dev-02
       NAMESPACE: dynamo-nightly-dsv4
-      RUN_KEY: ci-${{ github.run_id }}-${{ github.run_attempt }}
+      RUN_KEY: ci-disagg-${{ github.run_id }}-${{ github.run_attempt }}
       OVERLAY: .github/ci/deepseek-v4-pro-disagg
     steps:
       - timeout-minutes: 5
@@ -180,9 +181,12 @@ jobs:
         if: ${{ success() && !inputs.preflight_only }}
         timeout-minutes: 5
         run: |
-          result=$(find "$ARTIFACTS/perf" -name profile_export_aiperf.json -print -quit)
-          jq -e '.request_count.avg == 1536 and (.error_request_count.avg // 0) == 0 and
-            .output_sequence_length.avg > 0' "$result"
+          concurrencies=$(yq '.spec.template.spec.containers[0].env[] | select(.name == "CONCURRENCIES").value' "$ARTIFACTS/perf.yaml")
+          for concurrency in $concurrencies; do
+            jq -e --argjson requests "$((concurrency * 3))" '
+              .request_count.avg == $requests and (.error_request_count.avg // 0) == 0 and
+              .output_sequence_length.avg > 0' "$ARTIFACTS/perf/c$concurrency/profile_export_aiperf.json"
+          done
       - name: Clean up recipe resources
         if: always()
         timeout-minutes: 20
diff --git a/.github/workflows/deepseek-v4-pro-nightly.yml b/.github/workflows/deepseek-v4-pro-nightly.yml
index 58bf285c3d4f..7411de2b606b 100644
--- a/.github/workflows/deepseek-v4-pro-nightly.yml
+++ b/.github/workflows/deepseek-v4-pro-nightly.yml
@@ -32,6 +32,7 @@ concurrency:
 
 jobs:
   perf:
+    name: DeepSeek V4 Pro aggregated perf
     uses: ./.github/workflows/shared-recipe-perf.yml
     with:
       recipe: deepseek-v4-pro-agg
diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml
index 85a8b791b33c..53824ec477dd 100644
--- a/.github/workflows/nightly-ci.yml
+++ b/.github/workflows/nightly-ci.yml
@@ -538,7 +538,7 @@ jobs:
     secrets: inherit
 
   deepseek-v4-pro-agg-perf:
-    name: DeepSeek V4 Pro aggregated perf
+    name: recipes
     needs: [compute-release-mode, resolve-source-sha, vllm-build, vllm-copy-to-acr]
     if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
     uses: ./.github/workflows/deepseek-v4-pro-nightly.yml
@@ -548,7 +548,7 @@ jobs:
     secrets: inherit
 
   deepseek-v4-pro-disagg-perf:
-    name: DeepSeek V4 Pro disaggregated perf
+    name: recipes
     needs: [compute-release-mode, resolve-source-sha, vllm-build, vllm-copy-to-acr]
     if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
     uses: ./.github/workflows/deepseek-v4-pro-disagg-nightly.yml

From b841fc246d0dac63c1b8c893ffc0dc607eb88253 Mon Sep 17 00:00:00 2001
From: Lavanya <lvijayakrish@nvidia.com>
Date: Wed, 30 Sep 2026 17:36:25 -0700
Subject: [PATCH 08/14] fix(ci): enable NIXL transfer telemetry

Signed-off-by: Lavanya <lvijayakrish@nvidia.com>
---
 .github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml b/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
index ae2119ebb03d..49800ef7f4a1 100644
--- a/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
+++ b/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
@@ -49,6 +49,8 @@ spec:
       envs: &workerEnvs
       - name: DYN_HEALTH_CHECK_REQUEST_TIMEOUT
         value: "1200"
+      - name: NIXL_TELEMETRY_ENABLE
+        value: "1"
       volumeMounts: *modelVolumeMounts
       extraPodSpec:
         priorityClassName: ci-nightly-high-priority

From 58c36dbf1aeeebdc7b9a29b02c51be9b4e739ee0 Mon Sep 17 00:00:00 2001
From: Lavanya <lvijayakrish@nvidia.com>
Date: Wed, 30 Sep 2026 19:04:04 -0700
Subject: [PATCH 09/14] refactor(ci): use shared recipe performance workflow

Signed-off-by: Lavanya <lvijayakrish@nvidia.com>
---
 .../deepseek-v4-pro-disagg-nightly.yml        | 176 +-----------------
 .github/workflows/shared-recipe-perf.yml      |  24 ++-
 2 files changed, 24 insertions(+), 176 deletions(-)

diff --git a/.github/workflows/deepseek-v4-pro-disagg-nightly.yml b/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
index 6c476300e066..251a5701e06b 100644
--- a/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
+++ b/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
@@ -37,172 +37,10 @@ concurrency:
 jobs:
   perf:
     name: DeepSeek V4 Pro disaggregated perf
-    runs-on: prod-deploy-tester-v1
-    timeout-minutes: 300
-    defaults:
-      run:
-        shell: bash
-    env:
-      # Read tbot's renewed credentials; every kubectl command selects its context.
-      KUBECONFIG: /teleport/kubeconfig.yaml
-      KUBE_CONTEXT: nv-prd-dgxc.teleport.sh-dynamo-gcp-dev-02
-      NAMESPACE: dynamo-nightly-dsv4
-      RUN_KEY: ci-disagg-${{ github.run_id }}-${{ github.run_attempt }}
-      OVERLAY: .github/ci/deepseek-v4-pro-disagg
-    steps:
-      - timeout-minutes: 5
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-        with:
-          ref: ${{ inputs.source_ref || github.sha }}
-          persist-credentials: false
-      - name: Prepare artifacts
-        timeout-minutes: 5
-        run: |
-          mkdir -p "$RUNNER_TEMP/dsv4-disagg-artifacts"
-          echo "ARTIFACTS=$RUNNER_TEMP/dsv4-disagg-artifacts" >> "$GITHUB_ENV"
-          echo "REGISTRY_AUTH_FILE=$RUNNER_TEMP/dsv4-disagg-registry-auth.json" >> "$GITHUB_ENV"
-      - timeout-minutes: 5
-        uses: ./.github/actions/skopeo-login
-        with:
-          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
-          azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
-          azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
-      - name: Render deployment and upstream perf recipe
-        timeout-minutes: 5
-        env:
-          IMAGE_TAG: ${{ inputs.image_tag }}
-          ACR_HOST: ${{ secrets.AZURE_ACR_HOSTNAME }}
-          ACR_REPOSITORY: ${{ vars.ACR_REPOSITORY }}
-          PERF_CLEANUP: |
-            trap 'find "$ARTIFACT_ROOT" -type f -name inputs.json -delete || true' EXIT
-        run: |
-          image="${ACR_HOST}/${ACR_REPOSITORY}:${IMAGE_TAG}"
-          digest=$(skopeo inspect --raw "docker://$image" | jq -er '
-            .manifests[] | select(.platform.os == "linux" and .platform.architecture == "arm64") | .digest')
-          export RUNTIME_IMAGE="$image@$digest"
-          yq -i '.spec.services[].extraPodSpec.mainContainer.image = strenv(RUNTIME_IMAGE) |
-            .spec.services[].extraPodSpec.imagePullSecrets = [{"name": "acr-" + strenv(RUN_KEY)}] |
-            (.spec.services.prefill.envFromSecret, .spec.services.decode.envFromSecret) = "hf-" + strenv(RUN_KEY)' "$OVERLAY/patch-deploy.yaml"
-          yq -i '.commonLabels.ci-run = strenv(RUN_KEY)' "$OVERLAY/kustomization.yaml"
-          kubectl kustomize "$OVERLAY" --load-restrictor LoadRestrictionsNone > "$ARTIFACTS/rendered.yaml"
-          yq 'select(.kind != "Job")' "$ARTIFACTS/rendered.yaml" > "$ARTIFACTS/deploy.yaml"
-          yq 'select(.kind == "Job")' "$ARTIFACTS/rendered.yaml" > "$ARTIFACTS/perf.yaml"
-          yq -i '(.spec.template.spec.containers[0].env[] | select(.name == "ARTIFACT_ROOT").value) = "/opt/models/perf/" + strenv(RUN_KEY) |
-            .spec.template.spec.containers[0].envFrom[0].secretRef.name = "hf-" + strenv(RUN_KEY) |
-            .spec.template.spec.containers[0].command[2] = strenv(PERF_CLEANUP) + "\n" + .spec.template.spec.containers[0].command[2]' "$ARTIFACTS/perf.yaml"
-          git rev-parse HEAD > "$ARTIFACTS/source-sha.txt"
-      - name: Preflight and create run credentials
-        timeout-minutes: 10
-        env:
-          DRY_RUN: ${{ inputs.preflight_only && 'server' || 'none' }}
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-          ACR_HOST: ${{ secrets.AZURE_ACR_HOSTNAME }}
-          ACR_USER: ${{ secrets.AZURE_ACR_USER }}
-          ACR_PASSWORD: ${{ secrets.AZURE_ACR_PASSWORD }}
-        run: |
-          k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
-          if [[ "$DRY_RUN" == none ]]; then
-            # Runs are serialized, so remove fixed-name resources left by an interrupted run.
-            "${k[@]}" delete --request-timeout=0 \
-              job/dsv4-pro-disagg-bench dgd/dsv4-pro-disagg computedomain/dsv4-pro-compute-domain \
-              --ignore-not-found --cascade=foreground --wait=true --timeout=5m
-            "${k[@]}" delete secret -l ci-workflow=deepseek-v4-pro-disagg-nightly \
-              --ignore-not-found --wait=false
-          fi
-          [[ "$("${k[@]}" get pvc shared-model-cache -o jsonpath='{.status.phase}')" == Bound ]]
-          "${k[@]}" create --dry-run=server -f "$ARTIFACTS/deploy.yaml" > "$ARTIFACTS/deploy-dry-run.txt"
-          "${k[@]}" create --dry-run=server -f "$ARTIFACTS/perf.yaml" > "$ARTIFACTS/perf-dry-run.txt"
-          "${k[@]}" create secret generic "hf-$RUN_KEY" --dry-run=client -o json \
-            --from-literal=HF_TOKEN="${HF_TOKEN:?Missing HF_TOKEN}" |
-            jq '.metadata.labels = {"ci-workflow": "deepseek-v4-pro-disagg-nightly"}' |
-            "${k[@]}" create --dry-run="$DRY_RUN" -f - -o name
-          "${k[@]}" create secret docker-registry "acr-$RUN_KEY" --dry-run=client -o json \
-            --docker-server="$ACR_HOST" --docker-username="$ACR_USER" --docker-password="$ACR_PASSWORD" |
-            jq '.metadata.labels = {"ci-workflow": "deepseek-v4-pro-disagg-nightly"}' |
-            "${k[@]}" create --dry-run="$DRY_RUN" -f - -o name
-      - name: Deploy and wait for readiness
-        if: ${{ !inputs.preflight_only }}
-        timeout-minutes: 105
-        run: |
-          k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
-          selector=nvidia.com/dynamo-graph-deployment-name=dsv4-pro-disagg
-          "${k[@]}" create -f "$ARTIFACTS/deploy.yaml"
-          # Reconnect logs every two minutes to discover new pods and reload tbot certificates.
-          until "${k[@]}" wait dgd/dsv4-pro-disagg --for=condition=Ready --timeout=5s; do
-            "${k[@]}" get dgd dsv4-pro-disagg
-            "${k[@]}" get pods -l "$selector" -o wide
-            timeout 120s "${k[@]}" logs --request-timeout=0 -f -l "$selector" -c main \
-              --prefix --timestamps --since=1m --tail=20 --ignore-errors || true
-            sleep 5
-          done
-          frontend=$("${k[@]}" get pods -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-disagg,nvidia.com/dynamo-component-type=frontend -o jsonpath='{.items[0].metadata.name}')
-          echo "FRONTEND=$frontend" >> "$GITHUB_ENV"
-      - name: Run upstream AIPerf benchmark
-        if: ${{ !inputs.preflight_only }}
-        timeout-minutes: 125
-        run: |
-          k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
-          "${k[@]}" create -f "$ARTIFACTS/perf.yaml"
-          until "${k[@]}" wait job/dsv4-pro-disagg-bench --for=condition=Complete --timeout=5s; do
-            "${k[@]}" get job dsv4-pro-disagg-bench -o json |
-              jq -e 'all(.status.conditions[]?; .type != "Failed" or .status != "True")' >/dev/null
-            "${k[@]}" get pods -l job-name=dsv4-pro-disagg-bench -o wide
-            timeout 120s "${k[@]}" logs --request-timeout=0 -f job/dsv4-pro-disagg-bench \
-              --prefix --timestamps --since=1m --tail=20 --ignore-errors || true
-            sleep 5
-          done
-      - name: Collect logs and perf artifacts
-        if: always()
-        timeout-minutes: 5
-        run: |
-          k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=30s)
-          "${k[@]}" get dgd,job,computedomain -l "ci-run=$RUN_KEY" -o yaml > "$ARTIFACTS/resources.yaml" || true
-          "${k[@]}" get events --sort-by=.lastTimestamp > "$ARTIFACTS/events.txt" || true
-          if [[ -n "$("${k[@]}" get dgd -l "ci-run=$RUN_KEY" -o name)" ]]; then
-            "${k[@]}" logs -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-disagg \
-              --all-containers --prefix --tail=1000 > "$ARTIFACTS/deployment.log" 2>&1 || true
-          fi
-          if [[ -n "$("${k[@]}" get job -l "ci-run=$RUN_KEY" -o name)" ]]; then
-            "${k[@]}" logs job/dsv4-pro-disagg-bench --all-containers > "$ARTIFACTS/perf.log" 2>&1 || true
-          fi
-          if [[ -n "$FRONTEND" ]]; then
-            frontend=$("${k[@]}" get pods \
-              -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-disagg,nvidia.com/dynamo-component-type=frontend -o json |
-              jq -r 'first(.items[] | select(.metadata.deletionTimestamp == null and .status.phase == "Running")) | .metadata.name // empty')
-            if [[ -n "$frontend" ]]; then
-              # Exclude generated prompts even if cleanup in the benchmark Job failed.
-              mkdir -p "$ARTIFACTS/perf"
-              "${k[@]}" exec -c main "$frontend" -- \
-                tar -C "/opt/models/perf/$RUN_KEY" --exclude=inputs.json -cf - . |
-                tar -C "$ARTIFACTS/perf" -xf -
-            fi
-          fi
-      - name: Verify AIPerf results
-        if: ${{ success() && !inputs.preflight_only }}
-        timeout-minutes: 5
-        run: |
-          concurrencies=$(yq '.spec.template.spec.containers[0].env[] | select(.name == "CONCURRENCIES").value' "$ARTIFACTS/perf.yaml")
-          for concurrency in $concurrencies; do
-            jq -e --argjson requests "$((concurrency * 3))" '
-              .request_count.avg == $requests and (.error_request_count.avg // 0) == 0 and
-              .output_sequence_length.avg > 0' "$ARTIFACTS/perf/c$concurrency/profile_export_aiperf.json"
-          done
-      - name: Clean up recipe resources
-        if: always()
-        timeout-minutes: 20
-        run: |
-          trap 'rm -f "$REGISTRY_AUTH_FILE"' EXIT
-          failed=0
-          kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=0 \
-            delete job,dgd,computedomain -l "ci-run=$RUN_KEY" --cascade=foreground --wait=true --timeout=15m || failed=1
-          kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=30s \
-            delete secret "hf-$RUN_KEY" "acr-$RUN_KEY" --ignore-not-found --wait=false || failed=1
-          exit "$failed"
-      - timeout-minutes: 5
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f  # v6.0.0
-        if: always()
-        with:
-          name: deepseek-v4-pro-disagg-${{ github.run_id }}-${{ github.run_attempt }}
-          path: ${{ env.ARTIFACTS }}
-          retention-days: 14
-          if-no-files-found: error
+    uses: ./.github/workflows/shared-recipe-perf.yml
+    with:
+      recipe: deepseek-v4-pro-disagg
+      source_ref: ${{ inputs.source_ref || github.sha }}
+      image_tag: ${{ inputs.image_tag }}
+      preflight_only: ${{ inputs.preflight_only || false }}
+    secrets: inherit
diff --git a/.github/workflows/shared-recipe-perf.yml b/.github/workflows/shared-recipe-perf.yml
index 15806a5b38de..aa1783b0fcc0 100644
--- a/.github/workflows/shared-recipe-perf.yml
+++ b/.github/workflows/shared-recipe-perf.yml
@@ -7,7 +7,7 @@ on:
   workflow_call:
     inputs:
       recipe:
-        description: 'Kustomize overlay: kimi-k25-agg or deepseek-v4-pro-agg'
+        description: 'Kustomize overlay: kimi-k25-agg, deepseek-v4-pro-agg, or deepseek-v4-pro-disagg'
         type: string
         required: true
       source_ref:
@@ -56,24 +56,31 @@ jobs:
           case "$RECIPE" in
             kimi-k25-agg)
               dgd=kimi-k25-agg-kv-eagle
-              worker=agg
+              workers=agg
               benchmark=kimi-k25-agg-kv-eagle-bench
               container=perf
               resources=dgd,job,configmap,computedomain
               ;;
             deepseek-v4-pro-agg)
               dgd=dsv4-pro-agg
-              worker=worker
+              workers=worker
               benchmark=dsv4-bench
               container=bench
               resources=dgd,job,computedomain
               ;;
+            deepseek-v4-pro-disagg)
+              dgd=dsv4-pro-disagg
+              workers=prefill,decode
+              benchmark=dsv4-pro-disagg-bench
+              container=bench
+              resources=dgd,job,computedomain
+              ;;
             *) echo "Unsupported recipe: $RECIPE" >&2; exit 1 ;;
           esac
           mkdir -p "$RUNNER_TEMP/$RUN_KEY-artifacts"
           {
             echo "DGD=$dgd"
-            echo "WORKER=$worker"
+            echo "WORKERS=$workers"
             echo "BENCHMARK=$benchmark"
             echo "PERF_CONTAINER=$container"
             echo "RESOURCE_KINDS=$resources"
@@ -100,8 +107,11 @@ jobs:
             .manifests[] | select(.platform.os == "linux" and .platform.architecture == "arm64") | .digest')
           export RUNTIME_IMAGE="$image@$digest"
           yq -i '.spec.services[].extraPodSpec.mainContainer.image = strenv(RUNTIME_IMAGE) |
-            .spec.services[].extraPodSpec.imagePullSecrets = [{"name": strenv(ACR_SECRET)}] |
-            .spec.services[strenv(WORKER)].envFromSecret = strenv(HF_SECRET)' "$OVERLAY/patch-deploy.yaml"
+            .spec.services[].extraPodSpec.imagePullSecrets = [{"name": strenv(ACR_SECRET)}]' "$OVERLAY/patch-deploy.yaml"
+          for worker in ${WORKERS//,/ }; do
+            export WORKER="$worker"
+            yq -i '.spec.services[strenv(WORKER)].envFromSecret = strenv(HF_SECRET)' "$OVERLAY/patch-deploy.yaml"
+          done
           yq -i '.commonLabels.ci-run = strenv(RUN_KEY) |
             .commonLabels.ci-workflow = strenv(RECIPE)' "$OVERLAY/kustomization.yaml"
           kubectl kustomize "$OVERLAY" --load-restrictor LoadRestrictionsNone > "$ARTIFACTS/rendered.yaml"
@@ -112,7 +122,7 @@ jobs:
           if [[ "$RECIPE" == kimi-k25-agg ]]; then
             yq -i 'with(select(.kind == "Job" and .metadata.name == strenv(BENCHMARK));
               (.spec.template.spec.containers[].env[] | select(.name == "ROOT_ARTIFACT_DIR").value) = "/model-cache/perf/" + strenv(RUN_KEY))' "$ARTIFACTS/rendered.yaml"
-          elif [[ "$RECIPE" == deepseek-v4-pro-agg ]]; then
+          elif [[ "$RECIPE" == deepseek-v4-pro-agg || "$RECIPE" == deepseek-v4-pro-disagg ]]; then
             yq -i 'with(select(.kind == "Job" and .metadata.name == strenv(BENCHMARK));
               .spec.template.spec.containers[0].command[2] = strenv(PERF_CLEANUP) + "\n" + .spec.template.spec.containers[0].command[2])' "$ARTIFACTS/rendered.yaml"
           fi

From 6431d7316b63937eaffd5a9ede0b0da6466b3f87 Mon Sep 17 00:00:00 2001
From: Lavanya <lvijayakrish@nvidia.com>
Date: Thu, 1 Oct 2026 06:37:19 -0700
Subject: [PATCH 10/14] fix(ci): allow DeepSeek disagg cold start

Signed-off-by: Lavanya <lvijayakrish@nvidia.com>
---
 .github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml |  6 ++++++
 .github/workflows/shared-recipe-perf.yml            | 12 ++++++++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml b/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
index 49800ef7f4a1..624b6223c20c 100644
--- a/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
+++ b/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
@@ -54,6 +54,11 @@ spec:
       volumeMounts: *modelVolumeMounts
       extraPodSpec:
         priorityClassName: ci-nightly-high-priority
+        mainContainer: &workerMainContainer
+          # DeepSeek V4 Pro loads an 805 GiB checkpoint from Lustre. Preserve
+          # the first cold-start attempt instead of restarting it at 90 minutes.
+          startupProbe:
+            failureThreshold: 900
         imagePullSecrets:
         - name: acr-token-secret
         nodeSelector: *arm64NodeSelector
@@ -75,6 +80,7 @@ spec:
       volumeMounts: *modelVolumeMounts
       extraPodSpec:
         priorityClassName: ci-nightly-high-priority
+        mainContainer: *workerMainContainer
         imagePullSecrets:
         - name: acr-token-secret
         nodeSelector: *arm64NodeSelector
diff --git a/.github/workflows/shared-recipe-perf.yml b/.github/workflows/shared-recipe-perf.yml
index aa1783b0fcc0..fca9bfd591d5 100644
--- a/.github/workflows/shared-recipe-perf.yml
+++ b/.github/workflows/shared-recipe-perf.yml
@@ -186,7 +186,9 @@ jobs:
           "${k[@]}" create -f "$ARTIFACTS/prerequisites.yaml"
       - name: Deploy and wait for readiness
         if: ${{ !inputs.preflight_only }}
-        timeout-minutes: 105
+        # DeepSeek V4 Pro disagg needs extra margin for four-node scheduling
+        # and loading its 805 GiB checkpoint from the shared Lustre volume.
+        timeout-minutes: ${{ inputs.recipe == 'deepseek-v4-pro-disagg' && 180 || 105 }}
         run: |
           k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
           selector=nvidia.com/dynamo-graph-deployment-name=$DGD
@@ -227,6 +229,12 @@ jobs:
           if [[ -n "$("${k[@]}" get dgd -l "$selector" -o name)" ]]; then
             "${k[@]}" logs -l "nvidia.com/dynamo-graph-deployment-name=$DGD" \
               --all-containers --prefix --tail=1000 > "$ARTIFACTS/deployment.log" 2>&1 || true
+            for worker_pod in $("${k[@]}" get pods \
+              -l "nvidia.com/dynamo-graph-deployment-name=$DGD,nvidia.com/dynamo-component-type=worker" \
+              -o name); do
+              "${k[@]}" logs "$worker_pod" --all-containers --prefix --previous --tail=1000 \
+                >> "$ARTIFACTS/deployment-previous.log" 2>&1 || true
+            done
           fi
           for job in $("${k[@]}" get job -l "$selector" -o name); do
             "${k[@]}" logs "$job" --all-containers > "$ARTIFACTS/${job##*/}.log" 2>&1 || true
@@ -239,7 +247,7 @@ jobs:
           fi
           pod=$("${k[@]}" get pods -l "$pod_selector" -o json |
             jq -r 'first(.items[] | select(.metadata.deletionTimestamp == null and .status.phase == "Running")) | .metadata.name // empty')
-          if [[ -n "$pod" ]]; then
+          if [[ -n "$pod" ]] && "${k[@]}" exec -c "$container" "$pod" -- test -d "$directory"; then
             mkdir -p "$ARTIFACTS/perf"
             "${k[@]}" exec -c "$container" "$pod" -- \
               tar -C "$directory" --exclude=inputs.json -cf - . | tar -C "$ARTIFACTS/perf" -xf -

From 17ce31b9f1714ba17fd61e5aa4ddd744cdd559c2 Mon Sep 17 00:00:00 2001
From: Lavanya <lvijayakrish@nvidia.com>
Date: Thu, 1 Oct 2026 06:47:42 -0700
Subject: [PATCH 11/14] fix(ci): queue DeepSeek recipe runs

Signed-off-by: Lavanya <lvijayakrish@nvidia.com>
---
 .github/workflows/deepseek-v4-pro-disagg-nightly.yml | 1 +
 .github/workflows/deepseek-v4-pro-nightly.yml        | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.github/workflows/deepseek-v4-pro-disagg-nightly.yml b/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
index 251a5701e06b..f4c135e97f35 100644
--- a/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
+++ b/.github/workflows/deepseek-v4-pro-disagg-nightly.yml
@@ -33,6 +33,7 @@ permissions:
 concurrency:
   group: deepseek-v4-pro-nightly
   cancel-in-progress: false
+  queue: max
 
 jobs:
   perf:
diff --git a/.github/workflows/deepseek-v4-pro-nightly.yml b/.github/workflows/deepseek-v4-pro-nightly.yml
index 7411de2b606b..35aaf3363df2 100644
--- a/.github/workflows/deepseek-v4-pro-nightly.yml
+++ b/.github/workflows/deepseek-v4-pro-nightly.yml
@@ -29,6 +29,7 @@ permissions:
 concurrency:
   group: deepseek-v4-pro-nightly
   cancel-in-progress: false
+  queue: max
 
 jobs:
   perf:

From 075a9f0084c0d8c8a823389a6d1c93fa66c06f90 Mon Sep 17 00:00:00 2001
From: Lavanya <lvijayakrish@nvidia.com>
Date: Thu, 1 Oct 2026 07:19:00 -0700
Subject: [PATCH 12/14] fix(ci): disable conflicting NIXL telemetry

Signed-off-by: Lavanya <lvijayakrish@nvidia.com>
---
 .github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml b/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
index 624b6223c20c..cda2fb572478 100644
--- a/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
+++ b/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
@@ -49,8 +49,6 @@ spec:
       envs: &workerEnvs
       - name: DYN_HEALTH_CHECK_REQUEST_TIMEOUT
         value: "1200"
-      - name: NIXL_TELEMETRY_ENABLE
-        value: "1"
       volumeMounts: *modelVolumeMounts
       extraPodSpec:
         priorityClassName: ci-nightly-high-priority

From fde2a8a7ce008c4ba749b6e83a4d2d53543e1c63 Mon Sep 17 00:00:00 2001
From: Lavanya <lvijayakrish@nvidia.com>
Date: Thu, 1 Oct 2026 08:09:55 -0700
Subject: [PATCH 13/14] fix(ci): collect NIXL telemetry without exporter

Signed-off-by: Lavanya <lvijayakrish@nvidia.com>
---
 .github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml b/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
index cda2fb572478..771096be4afc 100644
--- a/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
+++ b/.github/ci/deepseek-v4-pro-disagg/patch-deploy.yaml
@@ -49,6 +49,12 @@ spec:
       envs: &workerEnvs
       - name: DYN_HEALTH_CHECK_REQUEST_TIMEOUT
         value: "1200"
+      # vLLM reads per-transfer NIXL telemetry. The NOP exporter keeps that
+      # data available without binding one Prometheus port per DP process.
+      - name: NIXL_TELEMETRY_ENABLE
+        value: "y"
+      - name: NIXL_TELEMETRY_EXPORTER
+        value: NOP
       volumeMounts: *modelVolumeMounts
       extraPodSpec:
         priorityClassName: ci-nightly-high-priority

From 014246030c78cbc7d46a3b8ede7f4e3a37ddfa3a Mon Sep 17 00:00:00 2001
From: Lavanya <lvijayakrish@nvidia.com>
Date: Thu, 1 Oct 2026 09:26:30 -0700
Subject: [PATCH 14/14] fix(ci): collect previous worker logs

Signed-off-by: Lavanya <lvijayakrish@nvidia.com>
---
 .github/workflows/shared-recipe-perf.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/shared-recipe-perf.yml b/.github/workflows/shared-recipe-perf.yml
index fca9bfd591d5..6bda1d0ada86 100644
--- a/.github/workflows/shared-recipe-perf.yml
+++ b/.github/workflows/shared-recipe-perf.yml
@@ -230,9 +230,9 @@ jobs:
             "${k[@]}" logs -l "nvidia.com/dynamo-graph-deployment-name=$DGD" \
               --all-containers --prefix --tail=1000 > "$ARTIFACTS/deployment.log" 2>&1 || true
             for worker_pod in $("${k[@]}" get pods \
-              -l "nvidia.com/dynamo-graph-deployment-name=$DGD,nvidia.com/dynamo-component-type=worker" \
+              -l "nvidia.com/dynamo-graph-deployment-name=$DGD,nvidia.com/dynamo-component-type in (worker,prefill,decode)" \
               -o name); do
-              "${k[@]}" logs "$worker_pod" --all-containers --prefix --previous --tail=1000 \
+              "${k[@]}" logs "$worker_pod" -c main --prefix --previous --tail=1000 \
                 >> "$ARTIFACTS/deployment-previous.log" 2>&1 || true
             done
           fi