Skip to content
Merged
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 106 additions & 2 deletions .github/workflows/nightly-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,110 @@ jobs:
image_tag_suffix: '-nightly'
secrets: inherit

# ============================================================================
# NIGHTLY K8S DEPLOYMENT
# Build the operator, copy the nightly SGLang image to the deploy registry,
# then validate a baseline profile on the Kubernetes deploy-test runner.
# ============================================================================

operator:
name: Operator
needs: [create-fresh-builder, compute-release-mode, resolve-source-sha]
if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
runs-on: prod-default-v2
outputs:
operator_default_tag: ${{ steps.build.outputs.image_tag }}
steps:
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
- name: Build and push operator
id: build
uses: ./.github/actions/build-deploy-component
with:
component: operator
image_tag: ${{ needs.resolve-source-sha.outputs.source_sha }}-operator
builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }}
aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
extra_tags: |
${{ secrets.AZURE_ACR_HOSTNAME }}/ai-dynamo/dynamo:${{ needs.resolve-source-sha.outputs.source_sha }}-operator
Comment thread
knarangN marked this conversation as resolved.
Resolved
diff_event_context: nightly
diff_current_branch: ${{ github.ref_name }}
gh_token: ${{ secrets.GITHUB_TOKEN }}

sglang-copy-to-acr:
name: sglang-runtime-nightly
needs: [compute-release-mode, sglang-build]
if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
uses: ./.github/workflows/shared-copy.yml
with:
target_tag_plain: ${{ format('{0}-nightly', needs.sglang-build.outputs.target_tag_plain) }}
cuda_version: '["13.0"]'
copy_timeout_minutes: 20
secrets: inherit

deploy-operator:
needs: [compute-release-mode, operator, resolve-source-sha]
if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
runs-on: prod-deploy-tester-v1
outputs:
namespace: ${{ steps.setup.outputs.namespace }}
vcluster_name: ${{ steps.setup.outputs.vcluster_name }}
operator_tag: ${{ steps.setup.outputs.operator_tag }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ needs.resolve-source-sha.outputs.source_sha }}
- name: Setup vCluster and operator
id: setup
uses: ./.github/actions/setup-dynamo-operator
with:
registry: ${{ secrets.AZURE_ACR_HOSTNAME }}
operator_tag: ${{ needs.operator.outputs.operator_default_tag }}
hf_token: ${{ secrets.HF_TOKEN }}
dockerhub_username: ${{ secrets.DOCKERHUB_LOGIN_USER }}
dockerhub_password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
Comment thread
knarangN marked this conversation as resolved.
Resolved
# Provision the shared model cache PV/PVC when the repo provides its
# endpoint; empty vars -> no cache (workers download from HuggingFace).
model_cache_server: ${{ vars.AZURE_MODEL_CACHE_SERVER }}
model_cache_path: ${{ vars.AZURE_MODEL_CACHE_PATH }}

deploy-test-sglang:
Comment thread
knarangN marked this conversation as resolved.
Resolved
Comment thread
knarangN marked this conversation as resolved.
Resolved
name: sglang Nightly Deploy Test
needs: [compute-release-mode, deploy-operator, sglang-copy-to-acr]
if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
uses: ./.github/workflows/shared-deploy-test.yml
with:
framework: sglang
profiles: '["agg"]'
Comment thread
knarangN marked this conversation as resolved.
Resolved
image_suffix: sglang-runtime-nightly
Comment thread
knarangN marked this conversation as resolved.
Resolved
namespace: ${{ needs.deploy-operator.outputs.namespace }}
vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }}
operator_tag: ${{ needs.deploy-operator.outputs.operator_tag }}
secrets: inherit

deploy-cleanup:
name: Nightly Deploy Cleanup
if: always()
needs: [deploy-operator, deploy-test-sglang]
runs-on: prod-deploy-tester-v1
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: Teardown vCluster
if: needs.deploy-operator.outputs.namespace != '' && needs.deploy-operator.outputs.vcluster_name != ''
uses: ./.github/actions/teardown-dynamo-operator
with:
vcluster_name: ${{ needs.deploy-operator.outputs.vcluster_name }}
vcluster_namespace: ${{ needs.deploy-operator.outputs.namespace }}

# ============================================================================
# COMPLIANCE
# ============================================================================
Expand Down Expand Up @@ -863,7 +967,7 @@ jobs:
name: Clean K8s builder if exists
runs-on: prod-default-small-v2
if: always()
needs: [vllm-build, sglang-build, trtllm-build, dynamo-pipeline, create-fresh-builder]
needs: [vllm-build, sglang-build, trtllm-build, dynamo-pipeline, operator, create-fresh-builder]
permissions:
contents: read
steps:
Expand All @@ -884,7 +988,7 @@ jobs:
############################## SLACK NOTIFICATION ##############################
notify-slack:
if: always()
needs: [vllm-test, vllm-multi-gpu-test, sglang-test, sglang-multi-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, rust-tests]
needs: [vllm-test, vllm-multi-gpu-test, sglang-test, sglang-multi-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, deploy-cleanup, rust-tests]
permissions:
contents: read
actions: read # grant the reusable notifier read access to list this run's jobs
Expand Down
Loading