From 4762be39fcb44533a9a637ea60c51cdb91c28d70 Mon Sep 17 00:00:00 2001 From: Kavita Narang Date: Thu, 23 Jul 2026 16:59:36 -0700 Subject: [PATCH 1/4] ci: add Kubernetes deploy test to nightly Signed-off-by: Kavita Narang --- .github/workflows/nightly-ci.yml | 88 +++++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 95e8a6db7008..66e996b6f8c8 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -313,6 +313,90 @@ jobs: image_tag_suffix: '-nightly' secrets: inherit + # ============================================================================ + # NIGHTLY K8S DEPLOYMENT + # Build the operator, copy the nightly SGLang image to the deploy registry, + # then validate a baseline profile on the Kubernetes deploy-test runner. + # ============================================================================ + + operator: + name: Operator + needs: [create-fresh-builder, compute-release-mode, resolve-source-sha] + if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} + runs-on: prod-default-v2 + outputs: + operator_default_tag: ${{ steps.build.outputs.image_tag }} + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ needs.resolve-source-sha.outputs.source_sha }} + - name: Build and push operator + id: build + uses: ./.github/actions/build-deploy-component + with: + component: operator + image_tag: ${{ needs.resolve-source-sha.outputs.source_sha }}-operator + builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }} + aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} + aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} + azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }} + azure_acr_user: ${{ secrets.AZURE_ACR_USER }} + azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }} + extra_tags: | + ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.resolve-source-sha.outputs.source_sha }}-operator + diff_event_context: nightly + diff_current_branch: ${{ github.ref_name }} + gh_token: ${{ secrets.GITHUB_TOKEN }} + + sglang-copy-to-acr: + name: sglang-runtime-nightly + needs: [compute-release-mode, sglang-build] + if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} + uses: ./.github/workflows/shared-copy.yml + with: + target_tag_plain: ${{ format('{0}-nightly', needs.sglang-build.outputs.target_tag_plain) }} + cuda_version: '["13.0"]' + copy_timeout_minutes: 20 + secrets: inherit + + deploy-operator: + needs: [compute-release-mode, operator, resolve-source-sha] + if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} + runs-on: prod-deploy-tester-v1 + outputs: + namespace: ${{ steps.setup.outputs.namespace }} + vcluster_name: ${{ steps.setup.outputs.vcluster_name }} + operator_tag: ${{ steps.setup.outputs.operator_tag }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + ref: ${{ needs.resolve-source-sha.outputs.source_sha }} + - name: Setup vCluster and operator + id: setup + uses: ./.github/actions/setup-dynamo-operator + with: + registry: ${{ secrets.AZURE_ACR_HOSTNAME }} + operator_tag: ${{ needs.operator.outputs.operator_default_tag }} + hf_token: ${{ secrets.HF_TOKEN }} + dockerhub_username: ${{ secrets.DOCKERHUB_LOGIN_USER }} + dockerhub_password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + + deploy-test-sglang: + name: sglang Nightly Deploy Test + needs: [compute-release-mode, deploy-operator, sglang-copy-to-acr] + if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} + uses: ./.github/workflows/shared-deploy-test.yml + with: + framework: sglang + profiles: '["agg"]' + image_suffix: sglang-runtime-nightly + namespace: ${{ needs.deploy-operator.outputs.namespace }} + vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }} + operator_tag: ${{ needs.deploy-operator.outputs.operator_tag }} + secrets: inherit + # ============================================================================ # COMPLIANCE # ============================================================================ @@ -863,7 +947,7 @@ jobs: name: Clean K8s builder if exists runs-on: prod-default-small-v2 if: always() - needs: [vllm-build, sglang-build, trtllm-build, dynamo-pipeline, create-fresh-builder] + needs: [vllm-build, sglang-build, trtllm-build, dynamo-pipeline, operator, create-fresh-builder] permissions: contents: read steps: @@ -884,7 +968,7 @@ jobs: ############################## SLACK NOTIFICATION ############################## notify-slack: if: always() - needs: [vllm-test, vllm-multi-gpu-test, sglang-test, sglang-multi-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, rust-tests] + needs: [vllm-test, vllm-multi-gpu-test, sglang-test, sglang-multi-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, deploy-test-sglang, rust-tests] permissions: contents: read actions: read # grant the reusable notifier read access to list this run's jobs From b467ddaccd5e5667183fbf186727b01f6e129bae Mon Sep 17 00:00:00 2001 From: Kavita Narang Date: Thu, 23 Jul 2026 17:24:47 -0700 Subject: [PATCH 2/4] ci: provision and clean nightly deploy environment Signed-off-by: Kavita Narang --- .github/workflows/nightly-ci.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 66e996b6f8c8..ca554dbdf8ed 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -382,6 +382,10 @@ jobs: hf_token: ${{ secrets.HF_TOKEN }} dockerhub_username: ${{ secrets.DOCKERHUB_LOGIN_USER }} dockerhub_password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + # Provision the shared model cache PV/PVC when the repo provides its + # endpoint; empty vars -> no cache (workers download from HuggingFace). + model_cache_server: ${{ vars.AZURE_MODEL_CACHE_SERVER }} + model_cache_path: ${{ vars.AZURE_MODEL_CACHE_PATH }} deploy-test-sglang: name: sglang Nightly Deploy Test @@ -397,6 +401,22 @@ jobs: operator_tag: ${{ needs.deploy-operator.outputs.operator_tag }} secrets: inherit + deploy-cleanup: + name: Nightly Deploy Cleanup + if: always() + needs: [deploy-operator, deploy-test-sglang] + runs-on: prod-deploy-tester-v1 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Teardown vCluster + if: needs.deploy-operator.outputs.namespace != '' && needs.deploy-operator.outputs.vcluster_name != '' + uses: ./.github/actions/teardown-dynamo-operator + with: + vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }} + vcluster_namespace: ${{ needs.deploy-operator.outputs.namespace }} + # ============================================================================ # COMPLIANCE # ============================================================================ @@ -968,7 +988,7 @@ jobs: ############################## SLACK NOTIFICATION ############################## notify-slack: if: always() - needs: [vllm-test, vllm-multi-gpu-test, sglang-test, sglang-multi-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, deploy-test-sglang, rust-tests] + needs: [vllm-test, vllm-multi-gpu-test, sglang-test, sglang-multi-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, deploy-cleanup, rust-tests] permissions: contents: read actions: read # grant the reusable notifier read access to list this run's jobs From 6cd9a31b39410d28ed9507ab10614dd6ca063c10 Mon Sep 17 00:00:00 2001 From: Kavita Narang Date: Tue, 28 Jul 2026 13:03:24 -0700 Subject: [PATCH 3/4] ci: address nightly deploy review feedback Signed-off-by: Kavita Narang --- .github/workflows/nightly-ci.yml | 65 ++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index ca554dbdf8ed..d1c2316e4f48 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -315,8 +315,8 @@ jobs: # ============================================================================ # NIGHTLY K8S DEPLOYMENT - # Build the operator, copy the nightly SGLang image to the deploy registry, - # then validate a baseline profile on the Kubernetes deploy-test runner. + # Build the operator, copy the nightly runtime images to the deploy registry, + # then validate a baseline profile for each framework on the deploy-test runner. # ============================================================================ operator: @@ -336,7 +336,7 @@ jobs: uses: ./.github/actions/build-deploy-component with: component: operator - image_tag: ${{ needs.resolve-source-sha.outputs.source_sha }}-operator + image_tag: ${{ needs.resolve-source-sha.outputs.source_sha }}-operator-nightly builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }} aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} @@ -344,11 +344,22 @@ jobs: azure_acr_user: ${{ secrets.AZURE_ACR_USER }} azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }} extra_tags: | - ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.resolve-source-sha.outputs.source_sha }}-operator + ${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.resolve-source-sha.outputs.source_sha }}-operator-nightly diff_event_context: nightly diff_current_branch: ${{ github.ref_name }} gh_token: ${{ secrets.GITHUB_TOKEN }} + vllm-copy-to-acr: + name: vllm-runtime-nightly + needs: [compute-release-mode, vllm-build] + if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} + uses: ./.github/workflows/shared-copy.yml + with: + target_tag_plain: ${{ format('{0}-nightly', needs.vllm-build.outputs.target_tag_plain) }} + cuda_version: '["13.0"]' + copy_timeout_minutes: 20 + secrets: inherit + sglang-copy-to-acr: name: sglang-runtime-nightly needs: [compute-release-mode, sglang-build] @@ -360,6 +371,17 @@ jobs: copy_timeout_minutes: 20 secrets: inherit + trtllm-copy-to-acr: + name: trtllm-runtime-nightly + needs: [compute-release-mode, trtllm-build] + if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} + uses: ./.github/workflows/shared-copy.yml + with: + target_tag_plain: ${{ format('{0}-nightly', needs.trtllm-build.outputs.target_tag_plain) }} + cuda_version: '["13.1"]' + copy_timeout_minutes: 20 + secrets: inherit + deploy-operator: needs: [compute-release-mode, operator, resolve-source-sha] if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} @@ -377,6 +399,7 @@ jobs: id: setup uses: ./.github/actions/setup-dynamo-operator with: + vcluster_name: nightly-ci-${{ github.run_id }} registry: ${{ secrets.AZURE_ACR_HOSTNAME }} operator_tag: ${{ needs.operator.outputs.operator_default_tag }} hf_token: ${{ secrets.HF_TOKEN }} @@ -387,15 +410,43 @@ jobs: model_cache_server: ${{ vars.AZURE_MODEL_CACHE_SERVER }} model_cache_path: ${{ vars.AZURE_MODEL_CACHE_PATH }} + deploy-test-vllm: + name: vLLM Nightly Deploy Test + needs: [compute-release-mode, deploy-operator, vllm-build, vllm-copy-to-acr] + if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} + uses: ./.github/workflows/shared-deploy-test.yml + with: + framework: vllm + profiles: '["agg"]' + image_suffix: ${{ format('{0}-nightly', needs.vllm-build.outputs.target_tag_plain) }} + namespace: ${{ needs.deploy-operator.outputs.namespace }} + vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }} + operator_tag: ${{ needs.deploy-operator.outputs.operator_tag }} + secrets: inherit + deploy-test-sglang: name: sglang Nightly Deploy Test - needs: [compute-release-mode, deploy-operator, sglang-copy-to-acr] + needs: [compute-release-mode, deploy-operator, sglang-build, sglang-copy-to-acr] if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} uses: ./.github/workflows/shared-deploy-test.yml with: framework: sglang profiles: '["agg"]' - image_suffix: sglang-runtime-nightly + image_suffix: ${{ format('{0}-nightly', needs.sglang-build.outputs.target_tag_plain) }} + namespace: ${{ needs.deploy-operator.outputs.namespace }} + vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }} + operator_tag: ${{ needs.deploy-operator.outputs.operator_tag }} + secrets: inherit + + deploy-test-trtllm: + name: TensorRT-LLM Nightly Deploy Test + needs: [compute-release-mode, deploy-operator, trtllm-build, trtllm-copy-to-acr] + if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} + uses: ./.github/workflows/shared-deploy-test.yml + with: + framework: trtllm + profiles: '["agg"]' + image_suffix: ${{ format('{0}-nightly', needs.trtllm-build.outputs.target_tag_plain) }} namespace: ${{ needs.deploy-operator.outputs.namespace }} vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }} operator_tag: ${{ needs.deploy-operator.outputs.operator_tag }} @@ -404,7 +455,7 @@ jobs: deploy-cleanup: name: Nightly Deploy Cleanup if: always() - needs: [deploy-operator, deploy-test-sglang] + needs: [deploy-operator, deploy-test-vllm, deploy-test-sglang, deploy-test-trtllm] runs-on: prod-deploy-tester-v1 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 From 06d59f7cc41713fbdc423663244aa71c5d8fde04 Mon Sep 17 00:00:00 2001 From: Kavita Narang Date: Tue, 28 Jul 2026 20:47:46 -0700 Subject: [PATCH 4/4] ci: use valid namespace for nightly deploy Signed-off-by: Kavita Narang --- .github/workflows/nightly-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 7ca0a1384bdc..152e6068e827 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -400,6 +400,7 @@ jobs: uses: ./.github/actions/setup-dynamo-operator with: vcluster_name: nightly-ci-${{ github.run_id }} + vcluster_namespace: nightly-ci-${{ github.run_id }} registry: ${{ secrets.AZURE_ACR_HOSTNAME }} operator_tag: ${{ needs.operator.outputs.operator_default_tag }} hf_token: ${{ secrets.HF_TOKEN }}