sara4dev · Sep 30, 2026 · Sep 28, 2026 · Sep 28, 2026 · Sep 28, 2026 · Sep 29, 2026
@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: dynamo-nightly-dsv4
+resources:
+- ../../../recipes/deepseek-v4/deepseek-v4-pro/vllm/agg/gb200/deploy.yaml
+- ../../../recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml
+patches:
+- path: patch-deploy.yaml
+- path: patch-perf.yaml
+  target:
+    kind: Job
+    name: dsv4-pro-disagg-bench
+  options:
+    allowNameChange: true
@@ -0,0 +1,92 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: nvidia.com/v1alpha1
+kind: DynamoGraphDeployment
+metadata:
+  name: dsv4-pro-agg
+spec:
+  priorityClassName: ci-nightly-high-priority
+  pvcs:
+  - name: shared-model-cache
+    create: false
+  services:
+    Frontend:
+      extraPodMetadata:
+        annotations:
+          nvidia.com/disable-image-pull-secret-discovery: "true"
+      volumeMounts:
+      - name: shared-model-cache
+        mountPoint: /opt/models
+      extraPodSpec:
+        priorityClassName: ci-nightly-high-priority
+        imagePullSecrets:
+        - name: acr-token-secret
+        nodeSelector:
+          kubernetes.io/arch: arm64
+        tolerations:
+        - key: dedicated
+          operator: Equal
+          value: user-workload
+          effect: NoExecute
+        - key: nvidia.com/gpu
+          operator: Exists
+          effect: NoSchedule
+        - key: kubernetes.io/arch
+          operator: Equal
+          value: arm64
+          effect: NoSchedule
+        affinity:
+          nodeAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+              - matchExpressions:
+                - key: cloud.google.com/gke-nodepool
+                  operator: In
+                  values: [customer-gpu-o7v, customer-gpu-w0e]
+    worker:
+      extraPodMetadata:
+        annotations:
+          nvidia.com/disable-image-pull-secret-discovery: "true"
+      envs:
+      # Allow long prefills and cold JIT before marking workers unhealthy.
+      - name: DYN_HEALTH_CHECK_REQUEST_TIMEOUT
+        value: "1200"
+      volumeMounts:
+      - name: shared-model-cache
+        mountPoint: /opt/models
+      extraPodSpec:
+        priorityClassName: ci-nightly-high-priority
+        imagePullSecrets:
+        - name: acr-token-secret
+        nodeSelector:
+          kubernetes.io/arch: arm64
+        tolerations:
+        - key: dedicated
+          operator: Equal
+          value: user-workload
+          effect: NoExecute
+        - key: nvidia.com/gpu
+          operator: Exists
+          effect: NoSchedule
+        - key: kubernetes.io/arch
+          operator: Equal
+          value: arm64
+          effect: NoSchedule
+        affinity:
+          nodeAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+              - matchExpressions:
+                - key: cloud.google.com/gke-nodepool
+                  operator: In
+                  values: [customer-gpu-o7v, customer-gpu-w0e]
+          podAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+            - topologyKey: nvidia.com/gpu.clique
+              matchLabelKeys: [nvidia.com/dynamo-graph-deployment-name]
+              labelSelector:
+                matchExpressions:
+                - key: nvidia.com/dynamo-component-type
+                  operator: In
+                  values: [worker, decode]
@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: dsv4-bench
+spec:
+  backoffLimit: 0
+  activeDeadlineSeconds: 43200
+  template:
+    spec:
+      priorityClassName: ci-nightly-high-priority
+      nodeSelector:
+        kubernetes.io/arch: amd64
+      tolerations:
+      - key: dedicated
+        operator: Equal
+        value: user-workload
+        effect: NoExecute
+      containers:
+      - name: bench
+        env:
+        - name: ENDPOINT
+          value: dsv4-pro-agg-frontend:8000
+        - name: CONCURRENCIES
+          value: "1 8 64 128 256 512 1024"
+      volumes:
+      - name: model-cache
+        persistentVolumeClaim:
+          claimName: shared-model-cache
@@ -152,6 +152,7 @@ ignore:
   - 'lib/sidecar/**/*.rst'
 
 ci: &ci
+  - '.github/ci/**'
   - '.github/workflows/**'
   - '.github/filters.yaml'
   - '.github/actions/**'

@@ -0,0 +1,195 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: DeepSeek V4 Pro Aggregated Perf
+
+on:
+  workflow_call:
+    inputs:
+      source_ref:
+        type: string
+        required: true
+      image_tag:
+        type: string
+        required: true
+  workflow_dispatch:
+    inputs:
+      image_tag:
+        description: 'Published ACR vLLM nightly tag (<version>-ci-<sha>-vllm-runtime-nightly)'
+        type: string
+        required: true
+      preflight_only:
+        description: 'Render and server-side dry-run without deploying'
+        type: boolean
+        default: true
+
+permissions:
+  contents: read
+
+concurrency:
+  group: deepseek-v4-pro-nightly
+  cancel-in-progress: false
+
+jobs:
+  perf:
+    runs-on: prod-deploy-tester-v1
+    # Allow the 12-hour benchmark plus deployment, diagnostics, and cleanup.
+    timeout-minutes: 900
+    defaults:
+      run:
+        shell: bash
+    env:
+      # Read tbot's renewed credentials; every kubectl command selects its context.
+      KUBECONFIG: /teleport/kubeconfig.yaml
+      KUBE_CONTEXT: nv-prd-dgxc.teleport.sh-dynamo-gcp-dev-02
+      NAMESPACE: dynamo-nightly-dsv4
+      RUN_KEY: ci-${{ github.run_id }}-${{ github.run_attempt }}
+      OVERLAY: .github/ci/deepseek-v4-pro-agg
+    steps:
+      - timeout-minutes: 5
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          ref: ${{ inputs.source_ref || github.sha }}
+          persist-credentials: false
+      - name: Prepare artifacts
+        timeout-minutes: 5
+        run: |
+          mkdir -p "$RUNNER_TEMP/dsv4-artifacts"
+          echo "ARTIFACTS=$RUNNER_TEMP/dsv4-artifacts" >> "$GITHUB_ENV"
+          echo "REGISTRY_AUTH_FILE=$RUNNER_TEMP/dsv4-registry-auth.json" >> "$GITHUB_ENV"
+      - timeout-minutes: 5
+        uses: ./.github/actions/skopeo-login
+        with:
+          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
+          azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
+          azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
+      - name: Render deployment and upstream perf recipe
+        timeout-minutes: 5
+        env:
+          IMAGE_TAG: ${{ inputs.image_tag }}
+          ACR_HOST: ${{ secrets.AZURE_ACR_HOSTNAME }}
+          ACR_REPOSITORY: ${{ vars.ACR_REPOSITORY }}
+        run: |
+          image="${ACR_HOST}/${ACR_REPOSITORY}:${IMAGE_TAG}"
+          digest=$(skopeo inspect --raw "docker://$image" | jq -er '
+            .manifests[] | select(.platform.os == "linux" and .platform.architecture == "arm64") | .digest')
+          export RUNTIME_IMAGE="$image@$digest"
+          yq -i '.spec.services[].extraPodSpec.mainContainer.image = strenv(RUNTIME_IMAGE) |
+            .spec.services[].extraPodSpec.imagePullSecrets = [{"name": "acr-" + strenv(RUN_KEY)}] |
+            .spec.services.worker.envFromSecret = "hf-" + strenv(RUN_KEY)' "$OVERLAY/patch-deploy.yaml"
+          yq -i '.commonLabels.ci-run = strenv(RUN_KEY)' "$OVERLAY/kustomization.yaml"
+          kubectl kustomize "$OVERLAY" --load-restrictor LoadRestrictionsNone > "$ARTIFACTS/rendered.yaml"
+          yq 'select(.kind != "Job")' "$ARTIFACTS/rendered.yaml" > "$ARTIFACTS/deploy.yaml"
+          yq 'select(.kind == "Job")' "$ARTIFACTS/rendered.yaml" > "$ARTIFACTS/perf.yaml"
+          yq -i '(.spec.template.spec.containers[0].env[] | select(.name == "ARTIFACT_ROOT").value) = "/opt/models/perf/" + strenv(RUN_KEY) |
+            .spec.template.spec.containers[0].envFrom[0].secretRef.name = "hf-" + strenv(RUN_KEY)' "$ARTIFACTS/perf.yaml"
+          git rev-parse HEAD > "$ARTIFACTS/source-sha.txt"
+      - name: Preflight and create run credentials
+        timeout-minutes: 10
+        env:
+          DRY_RUN: ${{ inputs.preflight_only && 'server' || 'none' }}
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          ACR_HOST: ${{ secrets.AZURE_ACR_HOSTNAME }}
+          ACR_USER: ${{ secrets.AZURE_ACR_USER }}
+          ACR_PASSWORD: ${{ secrets.AZURE_ACR_PASSWORD }}
+        run: |
+          k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
+          if [[ "$DRY_RUN" == none ]]; then
+            # Runs are serialized, so remove fixed-name resources left by an interrupted run.
+            "${k[@]}" delete --request-timeout=0 \
+              job/dsv4-bench dgd/dsv4-pro-agg computedomain/dsv4-pro-agg-compute-domain \
+              --ignore-not-found --cascade=foreground --wait=true --timeout=5m
+            "${k[@]}" delete secret -l ci-workflow=deepseek-v4-pro-nightly --ignore-not-found --wait=false
+          fi
+          [[ "$("${k[@]}" get pvc shared-model-cache -o jsonpath='{.status.phase}')" == Bound ]]
+          "${k[@]}" create --dry-run=server -f "$ARTIFACTS/deploy.yaml" > "$ARTIFACTS/deploy-dry-run.txt"
+          "${k[@]}" create --dry-run=server -f "$ARTIFACTS/perf.yaml" > "$ARTIFACTS/perf-dry-run.txt"
+          "${k[@]}" create secret generic "hf-$RUN_KEY" --dry-run=client -o json \
+            --from-literal=HF_TOKEN="${HF_TOKEN:?Missing HF_TOKEN}" |
+            jq '.metadata.labels = {"ci-workflow": "deepseek-v4-pro-nightly"}' |
+            "${k[@]}" create --dry-run="$DRY_RUN" -f - -o name
+          "${k[@]}" create secret docker-registry "acr-$RUN_KEY" --dry-run=client -o json \
+            --docker-server="$ACR_HOST" --docker-username="$ACR_USER" --docker-password="$ACR_PASSWORD" |
+            jq '.metadata.labels = {"ci-workflow": "deepseek-v4-pro-nightly"}' |
+            "${k[@]}" create --dry-run="$DRY_RUN" -f - -o name
+      - name: Deploy and wait for readiness
+        if: ${{ !inputs.preflight_only }}
+        timeout-minutes: 105
+        run: |
+          k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
+          selector=nvidia.com/dynamo-graph-deployment-name=dsv4-pro-agg
+          "${k[@]}" create -f "$ARTIFACTS/deploy.yaml"
+          # Reconnect logs every two minutes to discover new pods and reload tbot certificates.
+          until "${k[@]}" wait dgd/dsv4-pro-agg --for=condition=Ready --timeout=5s; do
+            "${k[@]}" get dgd dsv4-pro-agg
+            "${k[@]}" get pods -l "$selector" -o wide
+            timeout 120s "${k[@]}" logs --request-timeout=0 -f -l "$selector" -c main \
+              --prefix --timestamps --since=1m --tail=20 --ignore-errors || true
+            sleep 5
+          done
+          frontend=$("${k[@]}" get pods -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-agg,nvidia.com/dynamo-component-type=frontend -o jsonpath='{.items[0].metadata.name}')
+          echo "FRONTEND=$frontend" >> "$GITHUB_ENV"
+      - name: Run upstream AIPerf benchmark
+        if: ${{ !inputs.preflight_only }}
+        run: |
+          k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=60s)
+          "${k[@]}" create -f "$ARTIFACTS/perf.yaml"
+          until "${k[@]}" wait job/dsv4-bench --for=condition=Complete --timeout=5s; do
+            "${k[@]}" get job dsv4-bench -o json |
+              jq -e 'all(.status.conditions[]?; .type != "Failed" or .status != "True")' >/dev/null
+            "${k[@]}" get pods -l job-name=dsv4-bench -o wide
+            timeout 120s "${k[@]}" logs --request-timeout=0 -f job/dsv4-bench \
+              --prefix --timestamps --since=1m --tail=20 --ignore-errors || true
+            sleep 5
+          done
+      - name: Collect logs and perf artifacts
+        if: always()
+        timeout-minutes: 5
+        run: |
+          k=(kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=30s)
+          "${k[@]}" get dgd,job,computedomain -l "ci-run=$RUN_KEY" -o yaml > "$ARTIFACTS/resources.yaml" || true
+          "${k[@]}" get events --sort-by=.lastTimestamp > "$ARTIFACTS/events.txt" || true
+          if [[ -n "$("${k[@]}" get dgd -l "ci-run=$RUN_KEY" -o name)" ]]; then
+            "${k[@]}" logs -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-agg \
+              --all-containers --prefix --tail=1000 > "$ARTIFACTS/deployment.log" 2>&1 || true
+          fi
+          if [[ -n "$("${k[@]}" get job -l "ci-run=$RUN_KEY" -o name)" ]]; then
+            "${k[@]}" logs job/dsv4-bench --all-containers > "$ARTIFACTS/perf.log" 2>&1 || true
+          fi
+          if [[ -n "$FRONTEND" ]]; then
+            frontend=$("${k[@]}" get pods \
+              -l nvidia.com/dynamo-graph-deployment-name=dsv4-pro-agg,nvidia.com/dynamo-component-type=frontend -o json |
+              jq -r 'first(.items[] | select(.metadata.deletionTimestamp == null and .status.phase == "Running")) | .metadata.name // empty')
+            if [[ -n "$frontend" ]]; then
+              "${k[@]}" cp -c main "$frontend:/opt/models/perf/$RUN_KEY/." "$ARTIFACTS/perf"
+            fi
+          fi
+      - name: Verify AIPerf results
+        if: ${{ success() && !inputs.preflight_only }}
+        timeout-minutes: 5
+        run: |
+          concurrencies=$(yq '.spec.template.spec.containers[0].env[] | select(.name == "CONCURRENCIES").value' "$ARTIFACTS/perf.yaml")
+          for concurrency in $concurrencies; do
+            jq -e --argjson requests "$((concurrency * 3))" '
+              .request_count.avg == $requests and (.error_request_count.avg // 0) == 0 and
+              .output_sequence_length.avg > 0' "$ARTIFACTS/perf/c$concurrency/profile_export_aiperf.json"
+          done
+      - name: Clean up recipe resources
+        if: always()
+        timeout-minutes: 20
+        run: |
+          trap 'rm -f "$REGISTRY_AUTH_FILE"' EXIT
+          failed=0
+          kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=0 \
+            delete job,dgd,computedomain -l "ci-run=$RUN_KEY" --cascade=foreground --wait=true --timeout=15m || failed=1
+          kubectl --context "$KUBE_CONTEXT" -n "$NAMESPACE" --request-timeout=30s \
+            delete secret "hf-$RUN_KEY" "acr-$RUN_KEY" --ignore-not-found --wait=false || failed=1
+          exit "$failed"
+      - timeout-minutes: 5
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f  # v6.0.0
+        if: always()
+        with:
+          name: deepseek-v4-pro-agg-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ env.ARTIFACTS }}
+          retention-days: 14
+          if-no-files-found: error
@@ -536,6 +536,16 @@ jobs:
       copy_timeout_minutes: 20
     secrets: inherit
 
+  deepseek-v4-pro-agg-perf:
+    name: DeepSeek V4 Pro aggregated perf
+    needs: [compute-release-mode, resolve-source-sha, vllm-build, vllm-copy-to-acr]
+    if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
+    uses: ./.github/workflows/deepseek-v4-pro-nightly.yml
+    with:
+      source_ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
+      image_tag: ${{ needs.vllm-copy-to-acr.outputs.image_tag }}
+    secrets: inherit
+
   sglang-copy-to-acr:
     name: sglang-runtime-nightly
     needs: [compute-release-mode, sglang-build]
@@ -1656,7 +1666,7 @@ jobs:
   ############################## SLACK NOTIFICATION ##############################
   notify-slack:
     if: always()
-    needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, sidecar-build, sidecar-vllm-test, sidecar-sglang-test, deploy-cleanup, rust-tests]
+    needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, sidecar-build, sidecar-vllm-test, sidecar-sglang-test, deploy-cleanup, deepseek-v4-pro-agg-perf, rust-tests]
     permissions:
       contents: read
       actions: read   # grant the reusable notifier read access to list this run's jobs