diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index a82e92f8415a..91f47ed2ac60 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -14,3 +14,4 @@ paths: self-hosted-runner: labels: - prod-builder-v3 + - prod-deploy-tester-v1 diff --git a/.github/ci/deepseek-v4-pro/deploy/kustomization.yaml b/.github/ci/deepseek-v4-pro/deploy/kustomization.yaml new file mode 100644 index 000000000000..56c150b29596 --- /dev/null +++ b/.github/ci/deepseek-v4-pro/deploy/kustomization.yaml @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../../../../recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/deploy.yaml + +# GCP GB200 exposes the shared cache under a cluster-managed PVC, taints its +# Arm nodes, and requires all four serving nodes to stay in one NVLink clique. +patches: + - target: + group: nvidia.com + version: v1alpha1 + kind: DynamoGraphDeployment + name: dsv4-pro-disagg + patch: |- + - op: replace + path: /spec/pvcs/0/name + value: shared-model-cache + - op: replace + path: /spec/services/Frontend/volumeMounts/0/name + value: shared-model-cache + - op: add + path: /spec/services/Frontend/extraPodSpec/priorityClassName + value: ci-nightly-high-priority + - op: replace + path: /spec/services/prefill/volumeMounts/0/name + value: shared-model-cache + - op: add + path: /spec/services/prefill/extraPodSpec/priorityClassName + value: ci-nightly-high-priority + - op: add + path: /spec/services/prefill/extraPodSpec/affinity + value: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: nvidia.com/gpu.clique + labelSelector: + matchExpressions: + - key: nvidia.com/dynamo-component-type + operator: In + values: [worker, prefill, decode] + - op: add + path: /spec/services/prefill/extraPodSpec/tolerations + value: + - key: kubernetes.io/arch + operator: Equal + value: arm64 + effect: NoSchedule + - op: replace + path: /spec/services/decode/volumeMounts/0/name + value: shared-model-cache + - op: add + path: /spec/services/decode/extraPodSpec/priorityClassName + value: ci-nightly-high-priority + - op: add + path: /spec/services/decode/extraPodSpec/affinity + value: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: nvidia.com/gpu.clique + labelSelector: + matchExpressions: + - key: nvidia.com/dynamo-component-type + operator: In + values: [worker, prefill, decode] + - op: add + path: /spec/services/decode/extraPodSpec/tolerations + value: + - key: kubernetes.io/arch + operator: Equal + value: arm64 + effect: NoSchedule diff --git a/.github/ci/deepseek-v4-pro/smoke/kustomization.yaml b/.github/ci/deepseek-v4-pro/smoke/kustomization.yaml new file mode 100644 index 000000000000..e517545c7b37 --- /dev/null +++ b/.github/ci/deepseek-v4-pro/smoke/kustomization.yaml @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../../../../recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/perf.yaml + +# Keep validation intentionally small: one concurrency with short synthetic +# sequences. The upstream exhaustive 8K/1K sweep remains unchanged. +patches: + - target: + group: batch + version: v1 + kind: Job + name: dsv4-pro-disagg-bench + patch: |- + apiVersion: batch/v1 + kind: Job + metadata: + name: dsv4-pro-disagg-bench + spec: + activeDeadlineSeconds: 1800 + template: + spec: + containers: + - name: bench + env: + - name: ISL + value: "128" + - name: OSL + value: "16" + - name: CONCURRENCIES + value: "1" + - name: ARTIFACT_ROOT + value: /tmp/aiperf/deepseek-v4-pro-smoke + volumes: + - name: model-cache + persistentVolumeClaim: + claimName: shared-model-cache diff --git a/.github/filters.yaml b/.github/filters.yaml index 1287320c1f20..59cd64f45dbc 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -152,10 +152,12 @@ ignore: - 'lib/sidecar/**/*.rst' ci: &ci + - '.github/ci/**' - '.github/workflows/**' - '.github/filters.yaml' - '.github/actions/**' - '.github/scripts/parse_buildkit_output.py' + - '.github/scripts/run_deepseek_v4_pro_smoke.sh' - '.github/scripts/route_buildkit.sh' - 'container/Dockerfile.test' diff --git a/.github/scripts/run_deepseek_v4_pro_smoke.sh b/.github/scripts/run_deepseek_v4_pro_smoke.sh new file mode 100755 index 000000000000..023f63f68c0c --- /dev/null +++ b/.github/scripts/run_deepseek_v4_pro_smoke.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -Eeuo pipefail + +REPO_ROOT="$(git rev-parse --show-toplevel)" +readonly REPO_ROOT +readonly NAMESPACE="${NAMESPACE:-dynamo-nightly-dsv4}" +readonly ARTIFACT_DIR="${ARTIFACT_DIR:-${REPO_ROOT}/artifacts/deepseek-v4-pro-smoke}" +readonly DEPLOY_TIMEOUT="${DEPLOY_TIMEOUT:-100m}" +readonly SCHEDULING_TIMEOUT="${SCHEDULING_TIMEOUT:-10m}" +readonly PERF_TIMEOUT="${PERF_TIMEOUT:-30m}" +readonly PVC_TIMEOUT_SECONDS="${PVC_TIMEOUT_SECONDS:-300}" +readonly KEEP_RESOURCES="${KEEP_RESOURCES:-0}" + +KUBECTL=(kubectl) +if [[ -n "${KUBE_CONTEXT:-}" ]]; then + KUBECTL+=(--context "${KUBE_CONTEXT}") +fi + +secret_manifest="" +deploy_manifest="" +smoke_manifest="" +mkdir -p "${ARTIFACT_DIR}" + +collect_diagnostics() { + "${KUBECTL[@]}" -n "${NAMESPACE}" get \ + pods,services,jobs,dynamographdeployments,computedomains \ + -o wide >"${ARTIFACT_DIR}/resources.txt" 2>&1 || true + "${KUBECTL[@]}" -n "${NAMESPACE}" get events \ + --sort-by=.lastTimestamp >"${ARTIFACT_DIR}/events.txt" 2>&1 || true + + while IFS= read -r pod; do + [[ -n "${pod}" ]] || continue + "${KUBECTL[@]}" -n "${NAMESPACE}" describe pod "${pod}" \ + >"${ARTIFACT_DIR}/${pod}-describe.txt" 2>&1 || true + "${KUBECTL[@]}" -n "${NAMESPACE}" logs "${pod}" --all-containers \ + --tail=500 >"${ARTIFACT_DIR}/${pod}.log" 2>&1 || true + done < <("${KUBECTL[@]}" -n "${NAMESPACE}" get pods -o name 2>/dev/null | cut -d/ -f2) +} + +cleanup() { + local status="${1:-$?}" + trap - EXIT INT TERM + + if (( status != 0 )); then + collect_diagnostics + fi + + if [[ "${KEEP_RESOURCES}" != "1" ]]; then + [[ -z "${smoke_manifest}" ]] || \ + "${KUBECTL[@]}" -n "${NAMESPACE}" delete -f "${smoke_manifest}" \ + --ignore-not-found --wait=false --timeout=2m || true + [[ -z "${deploy_manifest}" ]] || \ + "${KUBECTL[@]}" -n "${NAMESPACE}" delete -f "${deploy_manifest}" \ + --ignore-not-found --wait=false --timeout=2m || true + "${KUBECTL[@]}" -n "${NAMESPACE}" delete secret hf-token-secret \ + --ignore-not-found --wait=false --timeout=2m || true + else + echo "Keeping benchmark resources in ${NAMESPACE} for inspection" + fi + + [[ -z "${secret_manifest}" ]] || rm -f "${secret_manifest}" + [[ -z "${deploy_manifest}" ]] || rm -f "${deploy_manifest}" + [[ -z "${smoke_manifest}" ]] || rm -f "${smoke_manifest}" + + exit "${status}" +} +trap 'cleanup $?' EXIT +trap 'exit 130' INT TERM + +for command in git kubectl; do + command -v "${command}" >/dev/null || { + echo "Required command not found: ${command}" >&2 + exit 2 + } +done + +cd "${REPO_ROOT}" +secret_manifest="$(mktemp)" +deploy_manifest="$(mktemp)" +smoke_manifest="$(mktemp)" +chmod 600 "${secret_manifest}" "${deploy_manifest}" "${smoke_manifest}" + +"${KUBECTL[@]}" kustomize .github/ci/deepseek-v4-pro/deploy \ + --load-restrictor=LoadRestrictionsNone >"${deploy_manifest}" +"${KUBECTL[@]}" kustomize .github/ci/deepseek-v4-pro/smoke \ + --load-restrictor=LoadRestrictionsNone >"${smoke_manifest}" + +# The workflow owns these names and its concurrency group serializes runs. +"${KUBECTL[@]}" -n "${NAMESPACE}" delete -f "${smoke_manifest}" \ + --ignore-not-found --wait=true --timeout=5m +"${KUBECTL[@]}" -n "${NAMESPACE}" delete -f "${deploy_manifest}" \ + --ignore-not-found --wait=true --timeout=5m +"${KUBECTL[@]}" -n "${NAMESPACE}" delete secret hf-token-secret \ + --ignore-not-found --wait=true --timeout=2m + +"${KUBECTL[@]}" -n "${NAMESPACE}" create secret generic hf-token-secret \ + --from-literal=HF_TOKEN="${HF_TOKEN:-cache-only-smoke}" \ + --dry-run=client -o yaml >"${secret_manifest}" +"${KUBECTL[@]}" apply -f "${secret_manifest}" + +deadline=$((SECONDS + PVC_TIMEOUT_SECONDS)) +until [[ "$("${KUBECTL[@]}" -n "${NAMESPACE}" get pvc shared-model-cache \ + -o jsonpath='{.status.phase}' 2>/dev/null || true)" == "Bound" ]]; do + if (( SECONDS >= deadline )); then + echo "shared-model-cache did not bind within ${PVC_TIMEOUT_SECONDS}s" >&2 + exit 1 + fi + sleep 5 +done + +"${KUBECTL[@]}" -n "${NAMESPACE}" apply -f "${deploy_manifest}" + +for _ in {1..60}; do + worker_count="$("${KUBECTL[@]}" -n "${NAMESPACE}" get pods \ + -l 'nvidia.com/dynamo-component-type in (prefill,decode)' \ + -o name 2>/dev/null | wc -l | tr -d ' ')" + [[ "${worker_count}" == "4" ]] && break + sleep 5 +done +if [[ "${worker_count}" != "4" ]]; then + echo "Dynamo operator did not create all four worker pods" >&2 + exit 1 +fi +"${KUBECTL[@]}" -n "${NAMESPACE}" wait \ + --for=condition=PodScheduled pod \ + -l 'nvidia.com/dynamo-component-type in (prefill,decode)' \ + --timeout="${SCHEDULING_TIMEOUT}" +"${KUBECTL[@]}" -n "${NAMESPACE}" wait \ + --for=condition=Ready dynamographdeployment/dsv4-pro-disagg \ + --timeout="${DEPLOY_TIMEOUT}" + +"${KUBECTL[@]}" -n "${NAMESPACE}" apply -f "${smoke_manifest}" +"${KUBECTL[@]}" -n "${NAMESPACE}" wait \ + --for=condition=Complete job/dsv4-pro-disagg-bench \ + --timeout="${PERF_TIMEOUT}" + +"${KUBECTL[@]}" -n "${NAMESPACE}" logs job/dsv4-pro-disagg-bench \ + >"${ARTIFACT_DIR}/aiperf.log" +pod="$("${KUBECTL[@]}" -n "${NAMESPACE}" get pods \ + -l job-name=dsv4-pro-disagg-bench -o jsonpath='{.items[0].metadata.name}')" +"${KUBECTL[@]}" -n "${NAMESPACE}" cp \ + "${pod}:/tmp/aiperf/." "${ARTIFACT_DIR}/" || true + +collect_diagnostics +echo "DeepSeek-V4-Pro AIPerf smoke test completed in ${NAMESPACE}" diff --git a/.github/workflows/deepseek-v4-pro-nightly.yml b/.github/workflows/deepseek-v4-pro-nightly.yml new file mode 100644 index 000000000000..3299b11e454a --- /dev/null +++ b/.github/workflows/deepseek-v4-pro-nightly.yml @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: DeepSeek V4 Pro Nightly Smoke + +on: + # Temporary pre-merge validation trigger. Remove after the copy-branch run. + push: + branches: + - pull-request/14893 + paths: + - .github/ci/deepseek-v4-pro/** + - .github/scripts/run_deepseek_v4_pro_smoke.sh + - .github/workflows/deepseek-v4-pro-nightly.yml + - recipes/deepseek-v4/deepseek-v4-pro/vllm/disagg/gb200/** + schedule: + - cron: '0 10 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: deepseek-v4-pro-nightly + cancel-in-progress: false + +jobs: + smoke: + # Keep schedules inert until the nightly benchmark is enabled. Manual + # dispatch and this copy-branch push are for pre-merge bring-up. + if: >- + github.event_name == 'workflow_dispatch' || + github.event_name == 'push' || + vars.DEEPSEEK_V4_PRO_NIGHTLY_ENABLED == 'true' + runs-on: prod-deploy-tester-v1 + timeout-minutes: 150 + env: + NAMESPACE: dynamo-nightly-dsv4 + KUBE_CONTEXT: nv-prd-dgxc.teleport.sh-dynamo-gcp-dev-02 + HF_TOKEN: ${{ secrets.HF_TOKEN }} + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Deploy upstream recipe and run AIPerf smoke + shell: bash + env: + ARTIFACT_DIR: ${{ runner.temp }}/deepseek-v4-pro-smoke + run: bash .github/scripts/run_deepseek_v4_pro_smoke.sh + + - name: Upload smoke artifacts + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: deepseek-v4-pro-smoke-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/deepseek-v4-pro-smoke/ + retention-days: 14 + if-no-files-found: warn