From d695777270c27a276c3c04e2c1952e75994b4fba Mon Sep 17 00:00:00 2001
From: Saravana Periyasamy <saperiyasamy@nvidia.com>
Date: Mon, 28 Sep 2026 13:38:34 -0500
Subject: [PATCH 01/14] ci(recipes): run upstream DeepSeek aggregated recipes
 nightly

Signed-off-by: Saravana Periyasamy <saperiyasamy@nvidia.com>
---
 .../ci/deepseek-v4-pro-agg/kustomization.yaml |  16 ++
 .../ci/deepseek-v4-pro-agg/patch-deploy.yaml  |  82 +++++++++
 .../ci/deepseek-v4-pro-agg/patch-perf.yaml    |  45 +++++
 .github/filters.yaml                          |   1 +
 .github/workflows/deepseek-v4-pro-nightly.yml | 171 ++++++++++++++++++
 .github/workflows/nightly-ci.yml              |  12 +-
 6 files changed, 326 insertions(+), 1 deletion(-)
 create mode 100644 .github/ci/deepseek-v4-pro-agg/kustomization.yaml
 create mode 100644 .github/ci/deepseek-v4-pro-agg/patch-deploy.yaml
 create mode 100644 .github/ci/deepseek-v4-pro-agg/patch-perf.yaml
 create mode 100644 .github/workflows/deepseek-v4-pro-nightly.yml

diff --git a/.github/ci/deepseek-v4-pro-agg/kustomization.yaml b/.github/ci/deepseek-v4-pro-agg/kustomization.yaml
new file mode 100644
index 000000000000..03df96a624e0
--- /dev/null
+++ b/.github/ci/deepseek-v4-pro-agg/kustomization.yaml
@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: dynamo-nightly-dsv4
+resources:
+- ../../../recipes/deepseek-v4/deepseek-v4-pro/vllm/agg/gb200/deploy.yaml
+- ../../../recipes/deepseek-v4/perf/perf.yaml
+images:
+- name: python
+  newName: docker.io/library/python
+  digest: sha256:44ff437bba879d4941b710a369a8f19266aea34b29002807f0c487fabc9eec9b
+patches:
+- path: patch-deploy.yaml
+- path: patch-perf.yaml
diff --git a/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml b/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml
new file mode 100644
index 000000000000..34e688e8c395
--- /dev/null
+++ b/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml
@@ -0,0 +1,82 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: nvidia.com/v1alpha1
+kind: DynamoGraphDeployment
+metadata:
+  name: dsv4-pro-agg
+spec:
+  priorityClassName: ci-nightly-high-priority
+  pvcs:
+  - name: shared-model-cache
+    create: false
+  services:
+    Frontend:
+      volumeMounts:
+      - name: shared-model-cache
+        mountPoint: /opt/models
+      extraPodSpec:
+        priorityClassName: ci-nightly-high-priority
+        imagePullSecrets:
+        - name: acr-token-secret
+        nodeSelector:
+          kubernetes.io/arch: arm64
+        tolerations:
+        - key: dedicated
+          operator: Equal
+          value: user-workload
+          effect: NoExecute
+        - key: nvidia.com/gpu
+          operator: Exists
+          effect: NoSchedule
+        - key: kubernetes.io/arch
+          operator: Equal
+          value: arm64
+          effect: NoSchedule
+        affinity:
+          nodeAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+              - matchExpressions:
+                - key: cloud.google.com/gke-nodepool
+                  operator: In
+                  values: [customer-gpu-o7v, customer-gpu-w0e]
+    worker:
+      volumeMounts:
+      - name: shared-model-cache
+        mountPoint: /opt/models
+      extraPodSpec:
+        priorityClassName: ci-nightly-high-priority
+        imagePullSecrets:
+        - name: acr-token-secret
+        nodeSelector:
+          kubernetes.io/arch: arm64
+        tolerations:
+        - key: dedicated
+          operator: Equal
+          value: user-workload
+          effect: NoExecute
+        - key: nvidia.com/gpu
+          operator: Exists
+          effect: NoSchedule
+        - key: kubernetes.io/arch
+          operator: Equal
+          value: arm64
+          effect: NoSchedule
+        affinity:
+          nodeAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+              - matchExpressions:
+                - key: cloud.google.com/gke-nodepool
+                  operator: In
+                  values: [customer-gpu-o7v, customer-gpu-w0e]
+          podAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+            - topologyKey: nvidia.com/gpu.clique
+              matchLabelKeys: [nvidia.com/dynamo-graph-deployment-name]
+              labelSelector:
+                matchExpressions:
+                - key: nvidia.com/dynamo-component-type
+                  operator: In
+                  values: [worker, decode]
diff --git a/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml b/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml
new file mode 100644
index 000000000000..7bbf64d3f1a8
--- /dev/null
+++ b/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml
@@ -0,0 +1,45 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: dsv4-bench
+spec:
+  backoffLimit: 0
+  activeDeadlineSeconds: 900
+  template:
+    spec:
+      affinity: null
+      priorityClassName: ci-nightly-high-priority
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - key: dedicated
+        operator: Equal
+        value: user-workload
+        effect: NoExecute
+      containers:
+      - name: perf
+        resources:
+          requests:
+            cpu: "2"
+            memory: 4Gi
+          limits:
+            cpu: "4"
+            memory: 8Gi
+        env:
+        - name: ENDPOINT
+          value: dsv4-pro-agg-frontend:8000
+        - name: TARGET_MODEL
+          value: deepseek-ai/DeepSeek-V4-Pro
+        - name: CONCURRENCY
+          value: "1"
+        - name: REQUEST_TIMEOUT_SECONDS
+          value: "120"
+        - name: WAIT_FOR_MODEL_MAX_ATTEMPTS
+          value: "12"
+      volumes:
+      - name: model-cache
+        persistentVolumeClaim:
+          claimName: shared-model-cache
diff --git a/.github/filters.yaml b/.github/filters.yaml
index 2eb7c1a55910..3c7f3867ab43 100644
--- a/.github/filters.yaml
+++ b/.github/filters.yaml
@@ -152,6 +152,7 @@ ignore:
   - 'lib/sidecar/**/*.rst'
 
 ci: &ci
+  - '.github/ci/**'
   - '.github/workflows/**'
   - '.github/filters.yaml'
   - '.github/actions/**'
diff --git a/.github/workflows/deepseek-v4-pro-nightly.yml b/.github/workflows/deepseek-v4-pro-nightly.yml
new file mode 100644
index 000000000000..3b1cc638666e
--- /dev/null
+++ b/.github/workflows/deepseek-v4-pro-nightly.yml
@@ -0,0 +1,171 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: DeepSeek V4 Pro Aggregated Smoke
+
+on:
+  workflow_call:
+    inputs:
+      source_ref:
+        type: string
+        required: true
+      image_tag:
+        type: string
+        required: true
+  workflow_dispatch:
+    inputs:
+      image_tag:
+        description: 'Published ACR vLLM nightly tag (<version>-ci-<sha>-vllm-runtime-nightly)'
+        type: string
+        required: true
+      preflight_only:
+        description: 'Render and server-side dry-run without deploying'
+        type: boolean
+        default: true
+
+permissions:
+  contents: read
+
+concurrency:
+  group: deepseek-v4-pro-nightly
+  cancel-in-progress: false
+
+jobs:
+  smoke:
+    runs-on: prod-deploy-tester-v1
+    timeout-minutes: 210
+    defaults:
+      run:
+        shell: bash
+    env:
+      # Read tbot's renewed credentials; every kubectl command selects its context.
+      KUBECONFIG: /teleport/kubeconfig.yaml
+      KUBE_CONTEXT: nv-prd-dgxc.teleport.sh-dynamo-gcp-dev-02
+      NAMESPACE: dynamo-nightly-dsv4
+      RUN_KEY: ci-${{ github.run_id }}-${{ github.run_attempt }}
+      OVERLAY: .github/ci/deepseek-v4-pro-agg
+    steps:
+      - timeout-minutes: 5
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          ref: ${{ inputs.source_ref || github.sha }}
+          persist-credentials: false
+      - name: Prepare artifacts
+        timeout-minutes: 5
+        run: |
+          mkdir -p "$RUNNER_TEMP/dsv4-artifacts"
+          echo "ARTIFACTS=$RUNNER_TEMP/dsv4-artifacts" >> "$GITHUB_ENV"
+          echo "REGISTRY_AUTH_FILE=$RUNNER_TEMP/dsv4-registry-auth.json" >> "$GITHUB_ENV"
+      - timeout-minutes: 5
+        uses: ./.github/actions/skopeo-login
+        with:
+          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
+          azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
+          azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
+      - name: Render deployment and upstream perf recipe
+        timeout-minutes: 5
+        env:
+          IMAGE_TAG: ${{ inputs.image_tag }}
+          ACR_HOST: ${{ secrets.AZURE_ACR_HOSTNAME }}
+          ACR_REPOSITORY: ${{ vars.ACR_REPOSITORY }}
+        run: |
+          image="${ACR_HOST}/${ACR_REPOSITORY}:${IMAGE_TAG}"
+          digest=$(skopeo inspect --raw "docker://$image" | jq -er '
+            .manifests[] | select(.platform.os == "linux" and .platform.architecture == "arm64") | .digest')
+          export RUNTIME_IMAGE="$image@$digest"
+          yq -i '.spec.services[].extraPodSpec.mainContainer.image = strenv(RUNTIME_IMAGE) |
+            .spec.services[].extraPodSpec.imagePullSecrets = [{"name": "acr-" + strenv(RUN_KEY)}] |
+            .spec.services.worker.envFromSecret = "hf-" + strenv(RUN_KEY)' "$OVERLAY/patch-deploy.yaml"
+          yq -i '.commonLabels.ci-run = strenv(RUN_KEY)' "$OVERLAY/kustomization.yaml"
+          kubectl kustomize "$OVERLAY" --load-restrictor LoadRestrictionsNone > "$ARTIFACTS/rendered.yaml"
+          yq 'select(.kind != "Job")' "$ARTIFACTS/rendered.yaml" > "$ARTIFACTS/deploy.yaml"
+          yq 'select(.kind == "Job")' "$ARTIFACTS/rendered.yaml" > "$ARTIFACTS/perf.yaml"
+          yq -i '(.spec.template.spec.containers[0].env[] | select(.name == "TRACE_FILE").value) = "/model-cache/perf/" + strenv(RUN_KEY) + "/smoke.jsonl" |
+            (.spec.template.spec.containers[0].env[] | select(.name == "ROOT_ARTIFACT_DIR").value) = "/model-cache/perf/" + strenv(RUN_KEY)' "$ARTIFACTS/perf.yaml"
+          git rev-parse HEAD > "$ARTIFACTS/source-sha.txt"
+      - name: Preflight and create run credentials
+        timeout-minutes: 10
+        env:
+          DRY_RUN: ${{ inputs.preflight_only && 'server' || 'none' }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          ACR_HOST: ${{ secrets.AZURE_ACR_HOSTNAME }}
+          ACR_USER: ${{ secrets.AZURE_ACR_USER }}
+          ACR_PASSWORD: ${{ secrets.AZURE_ACR_PASSWORD }}
+        run: |
+          k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
+          [[ "$("${k[@]}" get pvc shared-model-cache -o jsonpath='{.status.phase}')" == Bound ]]
+          "${k[@]}" create --dry-run=server -f "$ARTIFACTS/deploy.yaml" > "$ARTIFACTS/deploy-dry-run.txt"
+          "${k[@]}" create --dry-run=server -f "$ARTIFACTS/perf.yaml" > "$ARTIFACTS/perf-dry-run.txt"
+          "${k[@]}" create secret generic "hf-$RUN_KEY" --dry-run="$DRY_RUN" -o name \
+            --from-literal=HF_TOKEN="${HF_TOKEN:?Missing HF_TOKEN}"
+          "${k[@]}" create secret docker-registry "acr-$RUN_KEY" --dry-run="$DRY_RUN" -o name \
+            --docker-server="$ACR_HOST" --docker-username="$ACR_USER" --docker-password="$ACR_PASSWORD"
+      - name: Deploy and wait for readiness
+        if: ${{ !inputs.preflight_only }}
+        timeout-minutes: 105
+        run: |
+          k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
+          "${k[@]}" create -f "$ARTIFACTS/deploy.yaml"
+          # Fresh clients reload tbot certificates during the long model startup.
+          until "${k[@]}" wait dgd/dsv4-pro-agg --for=condition=Ready --timeout=60s; do
+            "${k[@]}" get dgd dsv4-pro-agg -o name
+          done
+          frontend=$("${k[@]}" get pods -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-agg,nvidia.com/dynamo-component-type=frontend -o jsonpath='{.items[0].metadata.name}')
+          echo "FRONTEND=$frontend" >> "$GITHUB_ENV"
+          "${k[@]}" exec "$frontend" -c main -- mkdir -p "/opt/models/perf/$RUN_KEY"
+          printf '%s\n' '{"input_length":128,"output_length":16}' '{"input_length":128,"output_length":16}' '{"input_length":128,"output_length":16}' |
+            "${k[@]}" exec -i "$frontend" -c main -- sh -c 'cat > "$1"' sh "/opt/models/perf/$RUN_KEY/smoke.jsonl"
+      - name: Run upstream AIPerf smoke
+        if: ${{ !inputs.preflight_only }}
+        timeout-minutes: 17
+        run: |
+          k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
+          # The upstream Job checks model discovery before its three inference requests.
+          "${k[@]}" create -f "$ARTIFACTS/perf.yaml"
+          until "${k[@]}" wait job/dsv4-bench --for=condition=Complete --timeout=60s; do
+            "${k[@]}" get job dsv4-bench -o json |
+              jq -e 'all(.status.conditions[]?; .type != "Failed" or .status != "True")' >/dev/null
+          done
+      - name: Collect logs and perf artifacts
+        if: always()
+        timeout-minutes: 5
+        run: |
+          k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=30s)
+          "${k[@]}" get dgd,job,computedomain -l "ci-run=$RUN_KEY" -o yaml > "$ARTIFACTS/resources.yaml" || true
+          "${k[@]}" get events --sort-by=.lastTimestamp > "$ARTIFACTS/events.txt" || true
+          if [[ -n "$("${k[@]}" get dgd -l "ci-run=$RUN_KEY" -o name)" ]]; then
+            "${k[@]}" logs -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-agg \
+              --all-containers --prefix --tail=1000 > "$ARTIFACTS/deployment.log" 2>&1 || true
+          fi
+          if [[ -n "$("${k[@]}" get job -l "ci-run=$RUN_KEY" -o name)" ]]; then
+            "${k[@]}" logs job/dsv4-bench --all-containers > "$ARTIFACTS/perf.log" 2>&1 || true
+          fi
+          if [[ -n "$FRONTEND" ]]; then
+            "${k[@]}" cp -c main "$FRONTEND:/opt/models/perf/$RUN_KEY/." "$ARTIFACTS/perf"
+          fi
+      - name: Verify AIPerf results
+        if: ${{ success() && !inputs.preflight_only }}
+        timeout-minutes: 5
+        run: |
+          result=$(find "$ARTIFACTS/perf" -name profile_export_aiperf.json -print -quit)
+          jq -e '.request_count.avg == 3 and (.error_request_count.avg // 0) == 0 and
+            .output_sequence_length.avg > 0' "$result"
+      - name: Clean up recipe resources
+        if: always()
+        timeout-minutes: 20
+        run: |
+          trap 'rm -f "$REGISTRY_AUTH_FILE"' EXIT
+          failed=0
+          kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=0 \
+            delete job,dgd,computedomain -l "ci-run=$RUN_KEY" --cascade=foreground --wait=true --timeout=15m || failed=1
+          kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=30s \
+            delete secret "hf-$RUN_KEY" "acr-$RUN_KEY" --ignore-not-found --wait=false || failed=1
+          exit "$failed"
+      - timeout-minutes: 5
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f  # v6.0.0
+        if: always()
+        with:
+          name: deepseek-v4-pro-agg-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ env.ARTIFACTS }}
+          retention-days: 14
+          if-no-files-found: error
diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml
index 1ac90b6724c4..53dd2ff75ccd 100644
--- a/.github/workflows/nightly-ci.yml
+++ b/.github/workflows/nightly-ci.yml
@@ -536,6 +536,16 @@ jobs:
       copy_timeout_minutes: 20
     secrets: inherit
 
+  deepseek-v4-pro-agg-smoke:
+    name: DeepSeek V4 Pro aggregated smoke
+    needs: [compute-release-mode, resolve-source-sha, vllm-build, vllm-copy-to-acr]
+    if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
+    uses: ./.github/workflows/deepseek-v4-pro-nightly.yml
+    with:
+      source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
+      image_tag: ${{ needs.vllm-copy-to-acr.outputs.image_tag }}
+    secrets: inherit
+
   sglang-copy-to-acr:
     name: sglang-runtime-nightly
     needs: [compute-release-mode, sglang-build]
@@ -1656,7 +1666,7 @@ jobs:
   ############################## SLACK NOTIFICATION ##############################
   notify-slack:
     if: always()
-    needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, sidecar-build, sidecar-vllm-test, sidecar-sglang-test, deploy-cleanup, rust-tests]
+    needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, sidecar-build, sidecar-vllm-test, sidecar-sglang-test, deploy-cleanup, deepseek-v4-pro-agg-smoke, rust-tests]
     permissions:
       contents: read
       actions: read   # grant the reusable notifier read access to list this run's jobs

From 17cc32c4107689be27c3606d7ebbaa9707a5e202 Mon Sep 17 00:00:00 2001
From: Saravana Periyasamy <saperiyasamy@nvidia.com>
Date: Mon, 28 Sep 2026 17:44:04 -0500
Subject: [PATCH 02/14] ci(recipes): use upstream DeepSeek perf workload

Signed-off-by: Saravana Periyasamy <saperiyasamy@nvidia.com>
---
 .../ci/deepseek-v4-pro-agg/patch-perf.yaml    | 15 ---------------
 .github/workflows/deepseek-v4-pro-nightly.yml | 19 +++++++------------
 .github/workflows/nightly-ci.yml              |  6 +++---
 3 files changed, 10 insertions(+), 30 deletions(-)

diff --git a/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml b/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml
index 7bbf64d3f1a8..47dcaea6d6ae 100644
--- a/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml
+++ b/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml
@@ -6,8 +6,6 @@ kind: Job
 metadata:
   name: dsv4-bench
 spec:
-  backoffLimit: 0
-  activeDeadlineSeconds: 900
   template:
     spec:
       affinity: null
@@ -21,24 +19,11 @@ spec:
         effect: NoExecute
       containers:
       - name: perf
-        resources:
-          requests:
-            cpu: "2"
-            memory: 4Gi
-          limits:
-            cpu: "4"
-            memory: 8Gi
         env:
         - name: ENDPOINT
           value: dsv4-pro-agg-frontend:8000
         - name: TARGET_MODEL
           value: deepseek-ai/DeepSeek-V4-Pro
-        - name: CONCURRENCY
-          value: "1"
-        - name: REQUEST_TIMEOUT_SECONDS
-          value: "120"
-        - name: WAIT_FOR_MODEL_MAX_ATTEMPTS
-          value: "12"
       volumes:
       - name: model-cache
         persistentVolumeClaim:
diff --git a/.github/workflows/deepseek-v4-pro-nightly.yml b/.github/workflows/deepseek-v4-pro-nightly.yml
index 3b1cc638666e..50819a15e09f 100644
--- a/.github/workflows/deepseek-v4-pro-nightly.yml
+++ b/.github/workflows/deepseek-v4-pro-nightly.yml
@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-name: DeepSeek V4 Pro Aggregated Smoke
+name: DeepSeek V4 Pro Aggregated Perf
 
 on:
   workflow_call:
@@ -31,9 +31,9 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
-  smoke:
+  perf:
     runs-on: prod-deploy-tester-v1
-    timeout-minutes: 210
+    timeout-minutes: 300
     defaults:
       run:
         shell: bash
@@ -80,8 +80,7 @@ jobs:
           kubectl kustomize "$OVERLAY" --load-restrictor LoadRestrictionsNone > "$ARTIFACTS/rendered.yaml"
           yq 'select(.kind != "Job")' "$ARTIFACTS/rendered.yaml" > "$ARTIFACTS/deploy.yaml"
           yq 'select(.kind == "Job")' "$ARTIFACTS/rendered.yaml" > "$ARTIFACTS/perf.yaml"
-          yq -i '(.spec.template.spec.containers[0].env[] | select(.name == "TRACE_FILE").value) = "/model-cache/perf/" + strenv(RUN_KEY) + "/smoke.jsonl" |
-            (.spec.template.spec.containers[0].env[] | select(.name == "ROOT_ARTIFACT_DIR").value) = "/model-cache/perf/" + strenv(RUN_KEY)' "$ARTIFACTS/perf.yaml"
+          yq -i '(.spec.template.spec.containers[0].env[] | select(.name == "ROOT_ARTIFACT_DIR").value) = "/model-cache/perf/" + strenv(RUN_KEY)' "$ARTIFACTS/perf.yaml"
           git rev-parse HEAD > "$ARTIFACTS/source-sha.txt"
       - name: Preflight and create run credentials
         timeout-minutes: 10
@@ -112,15 +111,11 @@ jobs:
           done
           frontend=$("${k[@]}" get pods -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-agg,nvidia.com/dynamo-component-type=frontend -o jsonpath='{.items[0].metadata.name}')
           echo "FRONTEND=$frontend" >> "$GITHUB_ENV"
-          "${k[@]}" exec "$frontend" -c main -- mkdir -p "/opt/models/perf/$RUN_KEY"
-          printf '%s\n' '{"input_length":128,"output_length":16}' '{"input_length":128,"output_length":16}' '{"input_length":128,"output_length":16}' |
-            "${k[@]}" exec -i "$frontend" -c main -- sh -c 'cat > "$1"' sh "/opt/models/perf/$RUN_KEY/smoke.jsonl"
-      - name: Run upstream AIPerf smoke
+      - name: Run upstream AIPerf benchmark
         if: ${{ !inputs.preflight_only }}
-        timeout-minutes: 17
+        timeout-minutes: 125
         run: |
           k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
-          # The upstream Job checks model discovery before its three inference requests.
           "${k[@]}" create -f "$ARTIFACTS/perf.yaml"
           until "${k[@]}" wait job/dsv4-bench --for=condition=Complete --timeout=60s; do
             "${k[@]}" get job dsv4-bench -o json |
@@ -148,7 +143,7 @@ jobs:
         timeout-minutes: 5
         run: |
           result=$(find "$ARTIFACTS/perf" -name profile_export_aiperf.json -print -quit)
-          jq -e '.request_count.avg == 3 and (.error_request_count.avg // 0) == 0 and
+          jq -e '.request_count.avg == .input_config.phases[0].requests and (.error_request_count.avg // 0) == 0 and
             .output_sequence_length.avg > 0' "$result"
       - name: Clean up recipe resources
         if: always()
diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml
index 53dd2ff75ccd..7c1992fcf67b 100644
--- a/.github/workflows/nightly-ci.yml
+++ b/.github/workflows/nightly-ci.yml
@@ -536,8 +536,8 @@ jobs:
       copy_timeout_minutes: 20
     secrets: inherit
 
-  deepseek-v4-pro-agg-smoke:
-    name: DeepSeek V4 Pro aggregated smoke
+  deepseek-v4-pro-agg-perf:
+    name: DeepSeek V4 Pro aggregated perf
     needs: [compute-release-mode, resolve-source-sha, vllm-build, vllm-copy-to-acr]
     if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
     uses: ./.github/workflows/deepseek-v4-pro-nightly.yml
@@ -1666,7 +1666,7 @@ jobs:
   ############################## SLACK NOTIFICATION ##############################
   notify-slack:
     if: always()
-    needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, sidecar-build, sidecar-vllm-test, sidecar-sglang-test, deploy-cleanup, deepseek-v4-pro-agg-smoke, rust-tests]
+    needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, sidecar-build, sidecar-vllm-test, sidecar-sglang-test, deploy-cleanup, deepseek-v4-pro-agg-perf, rust-tests]
     permissions:
       contents: read
       actions: read   # grant the reusable notifier read access to list this run's jobs

From 7c1c060350ceef8ba23ea4b0ea4ade8016c72b3a Mon Sep 17 00:00:00 2001
From: Saravana Periyasamy <saperiyasamy@nvidia.com>
Date: Mon, 28 Sep 2026 17:56:56 -0500
Subject: [PATCH 03/14] ci(recipes): stream DeepSeek deployment startup logs

Signed-off-by: Saravana Periyasamy <saperiyasamy@nvidia.com>
---
 .github/workflows/deepseek-v4-pro-nightly.yml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/deepseek-v4-pro-nightly.yml b/.github/workflows/deepseek-v4-pro-nightly.yml
index 50819a15e09f..41b1801eff27 100644
--- a/.github/workflows/deepseek-v4-pro-nightly.yml
+++ b/.github/workflows/deepseek-v4-pro-nightly.yml
@@ -104,10 +104,15 @@ jobs:
         timeout-minutes: 105
         run: |
           k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
+          selector=nvidia.com/dynamo-graph-deployment-name=dsv4-pro-agg
           "${k[@]}" create -f "$ARTIFACTS/deploy.yaml"
-          # Fresh clients reload tbot certificates during the long model startup.
-          until "${k[@]}" wait dgd/dsv4-pro-agg --for=condition=Ready --timeout=60s; do
-            "${k[@]}" get dgd dsv4-pro-agg -o name
+          # Reconnect logs each minute to discover new pods and reload tbot certificates.
+          until "${k[@]}" wait dgd/dsv4-pro-agg --for=condition=Ready --timeout=5s; do
+            "${k[@]}" get dgd dsv4-pro-agg
+            "${k[@]}" get pods -l "$selector" -o wide
+            timeout 60s "${k[@]}" logs -f -l "$selector" -c main \
+              --prefix --timestamps --since=1m --tail=20 --ignore-errors || true
+            sleep 5
           done
           frontend=$("${k[@]}" get pods -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-agg,nvidia.com/dynamo-component-type=frontend -o jsonpath='{.items[0].metadata.name}')
           echo "FRONTEND=$frontend" >> "$GITHUB_ENV"

From 7dc4d0d181889472615474f9e4ce3e01fd0be2fc Mon Sep 17 00:00:00 2001
From: Saravana Periyasamy <saperiyasamy@nvidia.com>
Date: Mon, 28 Sep 2026 19:01:30 -0500
Subject: [PATCH 04/14] ci(recipes): stream AIPerf benchmark logs

Signed-off-by: Saravana Periyasamy <saperiyasamy@nvidia.com>
---
 .github/workflows/deepseek-v4-pro-nightly.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/deepseek-v4-pro-nightly.yml b/.github/workflows/deepseek-v4-pro-nightly.yml
index 41b1801eff27..3cd5c11a0bbd 100644
--- a/.github/workflows/deepseek-v4-pro-nightly.yml
+++ b/.github/workflows/deepseek-v4-pro-nightly.yml
@@ -122,9 +122,13 @@ jobs:
         run: |
           k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
           "${k[@]}" create -f "$ARTIFACTS/perf.yaml"
-          until "${k[@]}" wait job/dsv4-bench --for=condition=Complete --timeout=60s; do
+          until "${k[@]}" wait job/dsv4-bench --for=condition=Complete --timeout=5s; do
             "${k[@]}" get job dsv4-bench -o json |
               jq -e 'all(.status.conditions[]?; .type != "Failed" or .status != "True")' >/dev/null
+            "${k[@]}" get pods -l job-name=dsv4-bench -o wide
+            timeout 60s "${k[@]}" logs -f job/dsv4-bench \
+              --prefix --timestamps --since=1m --tail=20 --ignore-errors || true
+            sleep 5
           done
       - name: Collect logs and perf artifacts
         if: always()

From 5645cbe5572b7e004680a21b776d8f0dc60d4c29 Mon Sep 17 00:00:00 2001
From: Saravana Periyasamy <saperiyasamy@nvidia.com>
Date: Mon, 28 Sep 2026 19:30:02 -0500
Subject: [PATCH 05/14] ci(recipes): use one timeout for log streaming

Signed-off-by: Saravana Periyasamy <saperiyasamy@nvidia.com>
---
 .github/workflows/deepseek-v4-pro-nightly.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/deepseek-v4-pro-nightly.yml b/.github/workflows/deepseek-v4-pro-nightly.yml
index 3cd5c11a0bbd..1d18762d8c8d 100644
--- a/.github/workflows/deepseek-v4-pro-nightly.yml
+++ b/.github/workflows/deepseek-v4-pro-nightly.yml
@@ -106,11 +106,11 @@ jobs:
           k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
           selector=nvidia.com/dynamo-graph-deployment-name=dsv4-pro-agg
           "${k[@]}" create -f "$ARTIFACTS/deploy.yaml"
-          # Reconnect logs each minute to discover new pods and reload tbot certificates.
+          # Reconnect logs every two minutes to discover new pods and reload tbot certificates.
           until "${k[@]}" wait dgd/dsv4-pro-agg --for=condition=Ready --timeout=5s; do
             "${k[@]}" get dgd dsv4-pro-agg
             "${k[@]}" get pods -l "$selector" -o wide
-            timeout 60s "${k[@]}" logs -f -l "$selector" -c main \
+            timeout 120s "${k[@]}" logs --request-timeout=0 -f -l "$selector" -c main \
               --prefix --timestamps --since=1m --tail=20 --ignore-errors || true
             sleep 5
           done
@@ -126,7 +126,7 @@ jobs:
             "${k[@]}" get job dsv4-bench -o json |
               jq -e 'all(.status.conditions[]?; .type != "Failed" or .status != "True")' >/dev/null
             "${k[@]}" get pods -l job-name=dsv4-bench -o wide
-            timeout 60s "${k[@]}" logs -f job/dsv4-bench \
+            timeout 120s "${k[@]}" logs --request-timeout=0 -f job/dsv4-bench \
               --prefix --timestamps --since=1m --tail=20 --ignore-errors || true
             sleep 5
           done

From 764bf246c5251aa4be70084891ebfe4a67d2a399 Mon Sep 17 00:00:00 2001
From: Saravana Periyasamy <saperiyasamy@nvidia.com>
Date: Mon, 28 Sep 2026 19:50:27 -0500
Subject: [PATCH 06/14] ci(recipes): allow long DeepSeek inference health
 checks

Signed-off-by: Saravana Periyasamy <saperiyasamy@nvidia.com>
---
 .github/ci/deepseek-v4-pro-agg/patch-deploy.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml b/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml
index 34e688e8c395..5a0af53cad23 100644
--- a/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml
+++ b/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml
@@ -42,6 +42,10 @@ spec:
                   operator: In
                   values: [customer-gpu-o7v, customer-gpu-w0e]
     worker:
+      envs:
+      # Match the upstream perf request timeout for long prefills and cold JIT.
+      - name: DYN_HEALTH_CHECK_REQUEST_TIMEOUT
+        value: "1200"
       volumeMounts:
       - name: shared-model-cache
         mountPoint: /opt/models

From 790c739df0ab24673e9060678b482cc9beb78817 Mon Sep 17 00:00:00 2001
From: Saravana Periyasamy <saperiyasamy@nvidia.com>
Date: Mon, 28 Sep 2026 20:42:21 -0500
Subject: [PATCH 07/14] ci(recipes): isolate DeepSeek registry credentials

Signed-off-by: Saravana Periyasamy <saperiyasamy@nvidia.com>
---
 .github/ci/deepseek-v4-pro-agg/patch-deploy.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml b/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml
index 5a0af53cad23..f397192979ba 100644
--- a/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml
+++ b/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml
@@ -12,6 +12,8 @@ spec:
     create: false
   services:
     Frontend:
+      annotations:
+        nvidia.com/disable-image-pull-secret-discovery: "true"
       volumeMounts:
       - name: shared-model-cache
         mountPoint: /opt/models
@@ -42,6 +44,8 @@ spec:
                   operator: In
                   values: [customer-gpu-o7v, customer-gpu-w0e]
     worker:
+      annotations:
+        nvidia.com/disable-image-pull-secret-discovery: "true"
       envs:
       # Match the upstream perf request timeout for long prefills and cold JIT.
       - name: DYN_HEALTH_CHECK_REQUEST_TIMEOUT

From 23b4b2d78744bcacb9f64e49232fe82492d788ca Mon Sep 17 00:00:00 2001
From: Saravana Periyasamy <saperiyasamy@nvidia.com>
Date: Mon, 28 Sep 2026 20:49:55 -0500
Subject: [PATCH 08/14] ci(recipes): set registry isolation on pod metadata

Signed-off-by: Saravana Periyasamy <saperiyasamy@nvidia.com>
---
 .github/ci/deepseek-v4-pro-agg/patch-deploy.yaml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml b/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml
index f397192979ba..5148850822a7 100644
--- a/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml
+++ b/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml
@@ -12,8 +12,9 @@ spec:
     create: false
   services:
     Frontend:
-      annotations:
-        nvidia.com/disable-image-pull-secret-discovery: "true"
+      extraPodMetadata:
+        annotations:
+          nvidia.com/disable-image-pull-secret-discovery: "true"
       volumeMounts:
       - name: shared-model-cache
         mountPoint: /opt/models
@@ -44,8 +45,9 @@ spec:
                   operator: In
                   values: [customer-gpu-o7v, customer-gpu-w0e]
     worker:
-      annotations:
-        nvidia.com/disable-image-pull-secret-discovery: "true"
+      extraPodMetadata:
+        annotations:
+          nvidia.com/disable-image-pull-secret-discovery: "true"
       envs:
       # Match the upstream perf request timeout for long prefills and cold JIT.
       - name: DYN_HEALTH_CHECK_REQUEST_TIMEOUT

From db378553e8ef88a2d88b4ae838e2c1239c2e290b Mon Sep 17 00:00:00 2001
From: Saravana Periyasamy <saperiyasamy@nvidia.com>
Date: Tue, 29 Sep 2026 11:29:37 -0500
Subject: [PATCH 09/14] ci(recipes): recover interrupted DeepSeek nightly runs

Signed-off-by: Saravana Periyasamy <saperiyasamy@nvidia.com>
---
 .github/workflows/deepseek-v4-pro-nightly.yml | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/deepseek-v4-pro-nightly.yml b/.github/workflows/deepseek-v4-pro-nightly.yml
index 1d18762d8c8d..600d952c870f 100644
--- a/.github/workflows/deepseek-v4-pro-nightly.yml
+++ b/.github/workflows/deepseek-v4-pro-nightly.yml
@@ -92,6 +92,12 @@ jobs:
           ACR_PASSWORD: ${{ secrets.AZURE_ACR_PASSWORD }}
         run: |
           k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
+          if [[ "$DRY_RUN" == none ]]; then
+            # Runs are serialized, so remove fixed-name resources left by an interrupted run.
+            "${k[@]}" delete --request-timeout=0 \
+              job/dsv4-bench dgd/dsv4-pro-agg computedomain/dsv4-pro-agg-compute-domain \
+              --ignore-not-found --cascade=foreground --wait=true --timeout=5m
+          fi
           [[ "$("${k[@]}" get pvc shared-model-cache -o jsonpath='{.status.phase}')" == Bound ]]
           "${k[@]}" create --dry-run=server -f "$ARTIFACTS/deploy.yaml" > "$ARTIFACTS/deploy-dry-run.txt"
           "${k[@]}" create --dry-run=server -f "$ARTIFACTS/perf.yaml" > "$ARTIFACTS/perf-dry-run.txt"
@@ -145,7 +151,12 @@ jobs:
             "${k[@]}" logs job/dsv4-bench --all-containers > "$ARTIFACTS/perf.log" 2>&1 || true
           fi
           if [[ -n "$FRONTEND" ]]; then
-            "${k[@]}" cp -c main "$FRONTEND:/opt/models/perf/$RUN_KEY/." "$ARTIFACTS/perf"
+            frontend=$("${k[@]}" get pods \
+              -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-agg,nvidia.com/dynamo-component-type=frontend -o json |
+              jq -r 'first(.items[] | select(.metadata.deletionTimestamp == null and .status.phase == "Running")) | .metadata.name // empty')
+            if [[ -n "$frontend" ]]; then
+              "${k[@]}" cp -c main "$frontend:/opt/models/perf/$RUN_KEY/." "$ARTIFACTS/perf"
+            fi
           fi
       - name: Verify AIPerf results
         if: ${{ success() && !inputs.preflight_only }}

From b0055718023e7feaa0e45263cf98de1b34d111ac Mon Sep 17 00:00:00 2001
From: Saravana Periyasamy <saperiyasamy@nvidia.com>
Date: Tue, 29 Sep 2026 17:07:53 -0500
Subject: [PATCH 10/14] ci(recipes): inherit upstream DeepSeek benchmark image

Signed-off-by: Saravana Periyasamy <saperiyasamy@nvidia.com>
---
 .github/ci/deepseek-v4-pro-agg/kustomization.yaml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/ci/deepseek-v4-pro-agg/kustomization.yaml b/.github/ci/deepseek-v4-pro-agg/kustomization.yaml
index 03df96a624e0..f2e7f464657d 100644
--- a/.github/ci/deepseek-v4-pro-agg/kustomization.yaml
+++ b/.github/ci/deepseek-v4-pro-agg/kustomization.yaml
@@ -7,10 +7,6 @@ namespace: dynamo-nightly-dsv4
 resources:
 - ../../../recipes/deepseek-v4/deepseek-v4-pro/vllm/agg/gb200/deploy.yaml
 - ../../../recipes/deepseek-v4/perf/perf.yaml
-images:
-- name: python
-  newName: docker.io/library/python
-  digest: sha256:44ff437bba879d4941b710a369a8f19266aea34b29002807f0c487fabc9eec9b
 patches:
 - path: patch-deploy.yaml
 - path: patch-perf.yaml

From aba05791c7bc06c15b1eab4fd4b2ecf4a393d39f Mon Sep 17 00:00:00 2001
From: Saravana Periyasamy <saperiyasamy@nvidia.com>
Date: Tue, 29 Sep 2026 20:06:35 -0500
Subject: [PATCH 11/14] ci(recipes): match GitLab DeepSeek synthetic workload

Signed-off-by: Saravana Periyasamy <saperiyasamy@nvidia.com>
---
 .../ci/deepseek-v4-pro-agg/kustomization.yaml    |  7 ++++++-
 .github/ci/deepseek-v4-pro-agg/patch-deploy.yaml |  2 +-
 .github/ci/deepseek-v4-pro-agg/patch-perf.yaml   |  9 +++++----
 .github/workflows/deepseek-v4-pro-nightly.yml    | 16 ++++++++++------
 4 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/.github/ci/deepseek-v4-pro-agg/kustomization.yaml b/.github/ci/deepseek-v4-pro-agg/kustomization.yaml
index f2e7f464657d..6adb0e865048 100644
--- a/.github/ci/deepseek-v4-pro-agg/kustomization.yaml
+++ b/.github/ci/deepseek-v4-pro-agg/kustomization.yaml
@@ -6,7 +6,12 @@ kind: Kustomization
 namespace: dynamo-nightly-dsv4
 resources:
 - ../../../recipes/deepseek-v4/deepseek-v4-pro/vllm/agg/gb200/deploy.yaml
-- ../../../recipes/deepseek-v4/perf/perf.yaml
+- ../../../recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml
 patches:
 - path: patch-deploy.yaml
 - path: patch-perf.yaml
+  target:
+    kind: Job
+    name: dsv4-pro-disagg-bench
+  options:
+    allowNameChange: true
diff --git a/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml b/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml
index 5148850822a7..d4bfaaccb37a 100644
--- a/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml
+++ b/.github/ci/deepseek-v4-pro-agg/patch-deploy.yaml
@@ -49,7 +49,7 @@ spec:
         annotations:
           nvidia.com/disable-image-pull-secret-discovery: "true"
       envs:
-      # Match the upstream perf request timeout for long prefills and cold JIT.
+      # Allow long prefills and cold JIT before marking workers unhealthy.
       - name: DYN_HEALTH_CHECK_REQUEST_TIMEOUT
         value: "1200"
       volumeMounts:
diff --git a/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml b/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml
index 47dcaea6d6ae..6b85afc54dd0 100644
--- a/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml
+++ b/.github/ci/deepseek-v4-pro-agg/patch-perf.yaml
@@ -6,9 +6,10 @@ kind: Job
 metadata:
   name: dsv4-bench
 spec:
+  backoffLimit: 0
+  activeDeadlineSeconds: 43200
   template:
     spec:
-      affinity: null
       priorityClassName: ci-nightly-high-priority
       nodeSelector:
         kubernetes.io/arch: amd64
@@ -18,12 +19,12 @@ spec:
         value: user-workload
         effect: NoExecute
       containers:
-      - name: perf
+      - name: bench
         env:
         - name: ENDPOINT
           value: dsv4-pro-agg-frontend:8000
-        - name: TARGET_MODEL
-          value: deepseek-ai/DeepSeek-V4-Pro
+        - name: CONCURRENCIES
+          value: "1 8 64 128 256 512 1024"
       volumes:
       - name: model-cache
         persistentVolumeClaim:
diff --git a/.github/workflows/deepseek-v4-pro-nightly.yml b/.github/workflows/deepseek-v4-pro-nightly.yml
index 600d952c870f..e76bcfd921e3 100644
--- a/.github/workflows/deepseek-v4-pro-nightly.yml
+++ b/.github/workflows/deepseek-v4-pro-nightly.yml
@@ -33,7 +33,8 @@ concurrency:
 jobs:
   perf:
     runs-on: prod-deploy-tester-v1
-    timeout-minutes: 300
+    # Allow the 12-hour benchmark plus deployment, diagnostics, and cleanup.
+    timeout-minutes: 900
     defaults:
       run:
         shell: bash
@@ -80,7 +81,8 @@ jobs:
           kubectl kustomize "$OVERLAY" --load-restrictor LoadRestrictionsNone > "$ARTIFACTS/rendered.yaml"
           yq 'select(.kind != "Job")' "$ARTIFACTS/rendered.yaml" > "$ARTIFACTS/deploy.yaml"
           yq 'select(.kind == "Job")' "$ARTIFACTS/rendered.yaml" > "$ARTIFACTS/perf.yaml"
-          yq -i '(.spec.template.spec.containers[0].env[] | select(.name == "ROOT_ARTIFACT_DIR").value) = "/model-cache/perf/" + strenv(RUN_KEY)' "$ARTIFACTS/perf.yaml"
+          yq -i '(.spec.template.spec.containers[0].env[] | select(.name == "ARTIFACT_ROOT").value) = "/opt/models/perf/" + strenv(RUN_KEY) |
+            .spec.template.spec.containers[0].envFrom[0].secretRef.name = "hf-" + strenv(RUN_KEY)' "$ARTIFACTS/perf.yaml"
           git rev-parse HEAD > "$ARTIFACTS/source-sha.txt"
       - name: Preflight and create run credentials
         timeout-minutes: 10
@@ -124,7 +126,6 @@ jobs:
           echo "FRONTEND=$frontend" >> "$GITHUB_ENV"
       - name: Run upstream AIPerf benchmark
         if: ${{ !inputs.preflight_only }}
-        timeout-minutes: 125
         run: |
           k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
           "${k[@]}" create -f "$ARTIFACTS/perf.yaml"
@@ -162,9 +163,12 @@ jobs:
         if: ${{ success() && !inputs.preflight_only }}
         timeout-minutes: 5
         run: |
-          result=$(find "$ARTIFACTS/perf" -name profile_export_aiperf.json -print -quit)
-          jq -e '.request_count.avg == .input_config.phases[0].requests and (.error_request_count.avg // 0) == 0 and
-            .output_sequence_length.avg > 0' "$result"
+          concurrencies=$(yq '.spec.template.spec.containers[0].env[] | select(.name == "CONCURRENCIES").value' "$ARTIFACTS/perf.yaml")
+          for concurrency in $concurrencies; do
+            jq -e --argjson requests "$((concurrency * 3))" '
+              .request_count.avg == $requests and (.error_request_count.avg // 0) == 0 and
+              .output_sequence_length.avg > 0' "$ARTIFACTS/perf/c$concurrency/profile_export_aiperf.json"
+          done
       - name: Clean up recipe resources
         if: always()
         timeout-minutes: 20

From 4ec2547084e996b186cf86e6acd1386e0cf53feb Mon Sep 17 00:00:00 2001
From: Saravana Periyasamy <saperiyasamy@nvidia.com>
Date: Tue, 29 Sep 2026 20:21:13 -0500
Subject: [PATCH 12/14] fix(ci): clean up stale DeepSeek nightly credentials

Signed-off-by: Saravana Periyasamy <saperiyasamy@nvidia.com>
---
 .github/workflows/deepseek-v4-pro-nightly.yml | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/deepseek-v4-pro-nightly.yml b/.github/workflows/deepseek-v4-pro-nightly.yml
index e76bcfd921e3..36b222b2a46e 100644
--- a/.github/workflows/deepseek-v4-pro-nightly.yml
+++ b/.github/workflows/deepseek-v4-pro-nightly.yml
@@ -99,14 +99,19 @@ jobs:
             "${k[@]}" delete --request-timeout=0 \
               job/dsv4-bench dgd/dsv4-pro-agg computedomain/dsv4-pro-agg-compute-domain \
               --ignore-not-found --cascade=foreground --wait=true --timeout=5m
+            "${k[@]}" delete secret -l ci-workflow=deepseek-v4-pro-nightly --ignore-not-found --wait=false
           fi
           [[ "$("${k[@]}" get pvc shared-model-cache -o jsonpath='{.status.phase}')" == Bound ]]
           "${k[@]}" create --dry-run=server -f "$ARTIFACTS/deploy.yaml" > "$ARTIFACTS/deploy-dry-run.txt"
           "${k[@]}" create --dry-run=server -f "$ARTIFACTS/perf.yaml" > "$ARTIFACTS/perf-dry-run.txt"
-          "${k[@]}" create secret generic "hf-$RUN_KEY" --dry-run="$DRY_RUN" -o name \
-            --from-literal=HF_TOKEN="${HF_TOKEN:?Missing HF_TOKEN}"
-          "${k[@]}" create secret docker-registry "acr-$RUN_KEY" --dry-run="$DRY_RUN" -o name \
-            --docker-server="$ACR_HOST" --docker-username="$ACR_USER" --docker-password="$ACR_PASSWORD"
+          "${k[@]}" create secret generic "hf-$RUN_KEY" --dry-run=client -o json \
+            --from-literal=HF_TOKEN="${HF_TOKEN:?Missing HF_TOKEN}" |
+            jq '.metadata.labels = {"ci-workflow": "deepseek-v4-pro-nightly"}' |
+            "${k[@]}" create --dry-run="$DRY_RUN" -f - -o name
+          "${k[@]}" create secret docker-registry "acr-$RUN_KEY" --dry-run=client -o json \
+            --docker-server="$ACR_HOST" --docker-username="$ACR_USER" --docker-password="$ACR_PASSWORD" |
+            jq '.metadata.labels = {"ci-workflow": "deepseek-v4-pro-nightly"}' |
+            "${k[@]}" create --dry-run="$DRY_RUN" -f - -o name
       - name: Deploy and wait for readiness
         if: ${{ !inputs.preflight_only }}
         timeout-minutes: 105

From 16a41f7d20c5f449169c60b0b02e97bbe8928dcb Mon Sep 17 00:00:00 2001
From: Saravana Periyasamy <saperiyasamy@nvidia.com>
Date: Tue, 29 Sep 2026 21:14:12 -0500
Subject: [PATCH 13/14] fix(ci): discard generated DeepSeek benchmark inputs

Signed-off-by: Saravana Periyasamy <saperiyasamy@nvidia.com>
---
 .github/workflows/deepseek-v4-pro-nightly.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/deepseek-v4-pro-nightly.yml b/.github/workflows/deepseek-v4-pro-nightly.yml
index 36b222b2a46e..2dd33d553db9 100644
--- a/.github/workflows/deepseek-v4-pro-nightly.yml
+++ b/.github/workflows/deepseek-v4-pro-nightly.yml
@@ -161,6 +161,9 @@ jobs:
               -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-agg,nvidia.com/dynamo-component-type=frontend -o json |
               jq -r 'first(.items[] | select(.metadata.deletionTimestamp == null and .status.phase == "Running")) | .metadata.name // empty')
             if [[ -n "$frontend" ]]; then
+              # Synthetic prompts are large and reproducible from the benchmark seed.
+              "${k[@]}" exec -c main "$frontend" -- \
+                find "/opt/models/perf/$RUN_KEY" -type f -name inputs.json -delete
               "${k[@]}" cp -c main "$frontend:/opt/models/perf/$RUN_KEY/." "$ARTIFACTS/perf"
             fi
           fi

From 45480a87e53aa26bfd2cf5f1ef45619509d6fc36 Mon Sep 17 00:00:00 2001
From: Saravana Periyasamy <saperiyasamy@nvidia.com>
Date: Tue, 29 Sep 2026 22:06:50 -0500
Subject: [PATCH 14/14] fix(ci): clean benchmark inputs as their owner

Signed-off-by: Saravana Periyasamy <saperiyasamy@nvidia.com>
---
 .github/workflows/deepseek-v4-pro-nightly.yml | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/deepseek-v4-pro-nightly.yml b/.github/workflows/deepseek-v4-pro-nightly.yml
index 2dd33d553db9..d1fdd97939cd 100644
--- a/.github/workflows/deepseek-v4-pro-nightly.yml
+++ b/.github/workflows/deepseek-v4-pro-nightly.yml
@@ -69,6 +69,8 @@ jobs:
           IMAGE_TAG: ${{ inputs.image_tag }}
           ACR_HOST: ${{ secrets.AZURE_ACR_HOSTNAME }}
           ACR_REPOSITORY: ${{ vars.ACR_REPOSITORY }}
+          PERF_CLEANUP: |
+            trap 'find "$ARTIFACT_ROOT" -type f -name inputs.json -delete || true' EXIT
         run: |
           image="${ACR_HOST}/${ACR_REPOSITORY}:${IMAGE_TAG}"
           digest=$(skopeo inspect --raw "docker://$image" | jq -er '
@@ -82,7 +84,8 @@ jobs:
           yq 'select(.kind != "Job")' "$ARTIFACTS/rendered.yaml" > "$ARTIFACTS/deploy.yaml"
           yq 'select(.kind == "Job")' "$ARTIFACTS/rendered.yaml" > "$ARTIFACTS/perf.yaml"
           yq -i '(.spec.template.spec.containers[0].env[] | select(.name == "ARTIFACT_ROOT").value) = "/opt/models/perf/" + strenv(RUN_KEY) |
-            .spec.template.spec.containers[0].envFrom[0].secretRef.name = "hf-" + strenv(RUN_KEY)' "$ARTIFACTS/perf.yaml"
+            .spec.template.spec.containers[0].envFrom[0].secretRef.name = "hf-" + strenv(RUN_KEY) |
+            .spec.template.spec.containers[0].command[2] = strenv(PERF_CLEANUP) + "\n" + .spec.template.spec.containers[0].command[2]' "$ARTIFACTS/perf.yaml"
           git rev-parse HEAD > "$ARTIFACTS/source-sha.txt"
       - name: Preflight and create run credentials
         timeout-minutes: 10
@@ -161,10 +164,11 @@ jobs:
               -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-agg,nvidia.com/dynamo-component-type=frontend -o json |
               jq -r 'first(.items[] | select(.metadata.deletionTimestamp == null and .status.phase == "Running")) | .metadata.name // empty')
             if [[ -n "$frontend" ]]; then
-              # Synthetic prompts are large and reproducible from the benchmark seed.
+              # Exclude generated prompts even if cleanup in the benchmark Job failed.
+              mkdir -p "$ARTIFACTS/perf"
               "${k[@]}" exec -c main "$frontend" -- \
-                find "/opt/models/perf/$RUN_KEY" -type f -name inputs.json -delete
-              "${k[@]}" cp -c main "$frontend:/opt/models/perf/$RUN_KEY/." "$ARTIFACTS/perf"
+                tar -C "/opt/models/perf/$RUN_KEY" --exclude=inputs.json -cf - . |
+                tar -C "$ARTIFACTS/perf" -xf -
             fi
           fi
       - name: Verify AIPerf results