diff --git a/.github/FILTERS.md b/.github/FILTERS.md index 4dd89b3b6e27..ba36a34dee29 100644 --- a/.github/FILTERS.md +++ b/.github/FILTERS.md @@ -10,7 +10,8 @@ When you open a PR, CI checks which files changed and runs only relevant jobs: |--------|----------| | `core` | Main test suite (vLLM, SGLang, TRT-LLM containers) | | `operator` | Kubernetes operator tests | -| `snapshot` | Snapshot Agent tests | +| `snapshot` | Snapshot Agent + all-framework DynamoCheckpoint deploy tests | +| `snapshot_vllm` / `snapshot_sglang` / `snapshot_trtllm` | That framework's DynamoCheckpoint deploy suite | | `deploy` | Deploy-specific tests | | `vllm` / `sglang` / `trtllm` | Backend-specific tests | | `benchmarks` | Dynamo runtime pipeline (runs `tests/benchmarks/**` pytest suite) | diff --git a/.github/actions/changed-files/action.yml b/.github/actions/changed-files/action.yml index c0e8a83d704e..b17674fb29f6 100644 --- a/.github/actions/changed-files/action.yml +++ b/.github/actions/changed-files/action.yml @@ -14,8 +14,17 @@ outputs: description: 'Whether operator files changed' value: ${{ steps.filter.outputs.operator_any_modified }} snapshot: - description: 'Whether snapshot-agent files changed' + description: 'Whether shared snapshot/DynamoCheckpoint files changed' value: ${{ steps.filter.outputs.snapshot_any_modified }} + snapshot_vllm: + description: 'Whether vLLM snapshot lifecycle files changed' + value: ${{ steps.filter.outputs.snapshot_vllm_any_modified }} + snapshot_sglang: + description: 'Whether SGLang snapshot lifecycle files changed' + value: ${{ steps.filter.outputs.snapshot_sglang_any_modified }} + snapshot_trtllm: + description: 'Whether TRT-LLM snapshot lifecycle files changed' + value: ${{ steps.filter.outputs.snapshot_trtllm_any_modified }} power_agent: description: 'Whether power-agent files changed' value: ${{ steps.filter.outputs.power_agent_any_modified }} @@ -134,6 +143,9 @@ runs: echo "core: ${{ steps.filter.outputs.core_any_modified }}" echo "operator: ${{ steps.filter.outputs.operator_any_modified }}" echo "snapshot: ${{ steps.filter.outputs.snapshot_any_modified }}" + echo "snapshot_vllm: ${{ steps.filter.outputs.snapshot_vllm_any_modified }}" + echo "snapshot_sglang: ${{ steps.filter.outputs.snapshot_sglang_any_modified }}" + echo "snapshot_trtllm: ${{ steps.filter.outputs.snapshot_trtllm_any_modified }}" echo "power_agent: ${{ steps.filter.outputs.power_agent_any_modified }}" echo "deploy: ${{ steps.filter.outputs.deploy_any_modified }}" echo "planner: ${{ steps.filter.outputs.planner_any_modified }}" @@ -152,6 +164,9 @@ runs: echo "core: ${{ steps.filter.outputs.core_all_modified_files }}" echo "operator: ${{ steps.filter.outputs.operator_all_modified_files }}" echo "snapshot: ${{ steps.filter.outputs.snapshot_all_modified_files }}" + echo "snapshot_vllm: ${{ steps.filter.outputs.snapshot_vllm_all_modified_files }}" + echo "snapshot_sglang: ${{ steps.filter.outputs.snapshot_sglang_all_modified_files }}" + echo "snapshot_trtllm: ${{ steps.filter.outputs.snapshot_trtllm_all_modified_files }}" echo "power_agent: ${{ steps.filter.outputs.power_agent_all_modified_files }}" echo "deploy: ${{ steps.filter.outputs.deploy_all_modified_files }}" echo "planner: ${{ steps.filter.outputs.planner_all_modified_files }}" @@ -168,7 +183,7 @@ runs: shell: bash run: | # Combine all filter-specific files into one list - COVERED_FILES=$(echo "${{ steps.filter.outputs.docs_all_modified_files }} ${{ steps.filter.outputs.examples_all_modified_files }} ${{ steps.filter.outputs.ignore_all_modified_files }} ${{ steps.filter.outputs.ci_all_modified_files }} ${{ steps.filter.outputs.core_all_modified_files }} ${{ steps.filter.outputs.operator_all_modified_files }} ${{ steps.filter.outputs.snapshot_all_modified_files }} ${{ steps.filter.outputs.power_agent_all_modified_files }} ${{ steps.filter.outputs.deploy_all_modified_files }} ${{ steps.filter.outputs.planner_all_modified_files }} ${{ steps.filter.outputs.vllm_all_modified_files }} ${{ steps.filter.outputs.sglang_all_modified_files }} ${{ steps.filter.outputs.trtllm_all_modified_files }} ${{ steps.filter.outputs.sample_all_modified_files }} ${{ steps.filter.outputs.frontend_all_modified_files }} ${{ steps.filter.outputs.benchmarks_all_modified_files }} ${{ steps.filter.outputs.efa_all_modified_files }} ${{ steps.filter.outputs.rust_all_modified_files }}" | tr ' ' '\n' | grep -v '^$' | sort -u) + COVERED_FILES=$(echo "${{ steps.filter.outputs.docs_all_modified_files }} ${{ steps.filter.outputs.examples_all_modified_files }} ${{ steps.filter.outputs.ignore_all_modified_files }} ${{ steps.filter.outputs.ci_all_modified_files }} ${{ steps.filter.outputs.core_all_modified_files }} ${{ steps.filter.outputs.operator_all_modified_files }} ${{ steps.filter.outputs.snapshot_all_modified_files }} ${{ steps.filter.outputs.snapshot_vllm_all_modified_files }} ${{ steps.filter.outputs.snapshot_sglang_all_modified_files }} ${{ steps.filter.outputs.snapshot_trtllm_all_modified_files }} ${{ steps.filter.outputs.power_agent_all_modified_files }} ${{ steps.filter.outputs.deploy_all_modified_files }} ${{ steps.filter.outputs.planner_all_modified_files }} ${{ steps.filter.outputs.vllm_all_modified_files }} ${{ steps.filter.outputs.sglang_all_modified_files }} ${{ steps.filter.outputs.trtllm_all_modified_files }} ${{ steps.filter.outputs.sample_all_modified_files }} ${{ steps.filter.outputs.frontend_all_modified_files }} ${{ steps.filter.outputs.benchmarks_all_modified_files }} ${{ steps.filter.outputs.efa_all_modified_files }} ${{ steps.filter.outputs.rust_all_modified_files }}" | tr ' ' '\n' | grep -v '^$' | sort -u) # Get all modified files ALL_FILES=$(echo "${{ steps.filter.outputs.all_all_modified_files }}" | tr ' ' '\n' | grep -v '^$' | sort -u) diff --git a/.github/filters.yaml b/.github/filters.yaml index 256075249fb3..121e5332bccf 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -3,7 +3,8 @@ # Filters that trigger CI jobs: # core -> dynamo build-test, all backend builds (vllm/sglang/trtllm) # operator -> operator build and test -# snapshot -> snapshot-agent build and test +# snapshot -> snapshot-agent build/test + all-framework DynamoCheckpoint deploy tests +# snapshot_vllm / snapshot_sglang / snapshot_trtllm -> that framework's DynamoCheckpoint suite # power_agent -> power-agent image build # deploy -> deploy tests # planner -> dynamo build-test @@ -152,6 +153,19 @@ snapshot: - 'deploy/operator/internal/checkpoint/**' - 'tests/deploy/test_dynamocheckpoint.py' +# Framework-local snapshot lifecycle files, plus TRT-LLM's unit test. These gate +# that framework's DynamoCheckpoint deploy jobs (and shared snapshot-agent/operator +# deps) without forcing the other backends' checkpoint suites. +snapshot_vllm: + - 'components/src/dynamo/vllm/snapshot.py' + +snapshot_sglang: + - 'components/src/dynamo/sglang/snapshot.py' + +snapshot_trtllm: + - 'components/src/dynamo/trtllm/snapshot.py' + - 'components/src/dynamo/trtllm/tests/test_trtllm_snapshot.py' + power_agent: - *ci - 'deploy/power-agent/**' diff --git a/.github/scripts/test-filters.js b/.github/scripts/test-filters.js index 5f8295d8673c..583b2d143bfc 100755 --- a/.github/scripts/test-filters.js +++ b/.github/scripts/test-filters.js @@ -169,6 +169,56 @@ const testCases = [ expect: { core: false, deploy: true }, desc: 'helm file triggers deploy' }, + + // Framework snapshot lifecycle: backend filter + that framework's DynamoCheckpoint filter + { + file: 'components/src/dynamo/vllm/snapshot.py', + expect: { + vllm: true, + snapshot: false, + snapshot_vllm: true, + snapshot_sglang: false, + snapshot_trtllm: false, + }, + desc: 'vllm snapshot.py gates only vllm DynamoCheckpoint' + }, + { + file: 'components/src/dynamo/sglang/snapshot.py', + expect: { + sglang: true, + snapshot: false, + snapshot_vllm: false, + snapshot_sglang: true, + snapshot_trtllm: false, + }, + desc: 'sglang snapshot.py gates only sglang DynamoCheckpoint' + }, + { + file: 'components/src/dynamo/trtllm/snapshot.py', + expect: { + trtllm: true, + snapshot: false, + snapshot_vllm: false, + snapshot_sglang: false, + snapshot_trtllm: true, + }, + desc: 'trtllm snapshot.py gates only trtllm DynamoCheckpoint' + }, + { + file: 'components/src/dynamo/trtllm/tests/test_trtllm_snapshot.py', + expect: { trtllm: true, snapshot_trtllm: true, snapshot: false }, + desc: 'trtllm snapshot unit test gates only trtllm DynamoCheckpoint' + }, + { + file: 'components/src/dynamo/common/snapshot/lifecycle.py', + expect: { snapshot: true, snapshot_vllm: false, core: true }, + desc: 'shared snapshot lifecycle triggers shared snapshot filter' + }, + { + file: 'tests/deploy/test_dynamocheckpoint.py', + expect: { snapshot: true, deploy: true }, + desc: 'DynamoCheckpoint deploy test triggers shared snapshot' + }, ]; // Print available filters diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d985a22b2603..0ae61bb46e67 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -27,6 +27,9 @@ jobs: planner: ${{ steps.changes.outputs.planner }} operator: ${{ steps.changes.outputs.operator }} snapshot: ${{ steps.changes.outputs.snapshot }} + snapshot_vllm: ${{ steps.changes.outputs.snapshot_vllm }} + snapshot_sglang: ${{ steps.changes.outputs.snapshot_sglang }} + snapshot_trtllm: ${{ steps.changes.outputs.snapshot_trtllm }} power_agent: ${{ steps.changes.outputs.power_agent }} deploy: ${{ steps.changes.outputs.deploy }} vllm: ${{ steps.changes.outputs.vllm }} @@ -146,7 +149,10 @@ jobs: needs.changed-files.outputs.sglang == 'true' || needs.changed-files.outputs.trtllm == 'true' || needs.changed-files.outputs.deploy == 'true' || - needs.changed-files.outputs.snapshot == 'true' + needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_vllm == 'true' || + needs.changed-files.outputs.snapshot_sglang == 'true' || + needs.changed-files.outputs.snapshot_trtllm == 'true' name: Operator runs-on: prod-default-v2 # actions: read lets the compliance step download a prior run's baseline @@ -211,6 +217,9 @@ jobs: needs: changed-files if: | needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_vllm == 'true' || + needs.changed-files.outputs.snapshot_sglang == 'true' || + needs.changed-files.outputs.snapshot_trtllm == 'true' || needs.changed-files.outputs.operator == 'true' || needs.changed-files.outputs.deploy == 'true' name: Snapshot Agent @@ -317,7 +326,8 @@ jobs: needs.changed-files.outputs.deploy == 'true' || needs.changed-files.outputs.sample == 'true' || needs.changed-files.outputs.operator == 'true' || - needs.changed-files.outputs.snapshot == 'true' + needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_vllm == 'true' uses: ./.github/workflows/shared-build-image.yml with: framework: vllm @@ -368,7 +378,8 @@ jobs: needs.changed-files.outputs.sglang == 'true' || needs.changed-files.outputs.deploy == 'true' || needs.changed-files.outputs.operator == 'true' || - needs.changed-files.outputs.snapshot == 'true' + needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_sglang == 'true' uses: ./.github/workflows/shared-build-image.yml with: framework: sglang @@ -419,7 +430,8 @@ jobs: needs.changed-files.outputs.trtllm == 'true' || needs.changed-files.outputs.deploy == 'true' || needs.changed-files.outputs.operator == 'true' || - needs.changed-files.outputs.snapshot == 'true' + needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_trtllm == 'true' uses: ./.github/workflows/shared-build-image.yml with: framework: trtllm @@ -621,6 +633,9 @@ jobs: needs.changed-files.outputs.frontend == 'true' || needs.changed-files.outputs.operator == 'true' || needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_vllm == 'true' || + needs.changed-files.outputs.snapshot_sglang == 'true' || + needs.changed-files.outputs.snapshot_trtllm == 'true' || needs.changed-files.outputs.deploy == 'true' uses: ./.github/workflows/shared-build-image.yml with: @@ -874,6 +889,9 @@ jobs: needs.changed-files.outputs.frontend == 'true' || needs.changed-files.outputs.operator == 'true' || needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_vllm == 'true' || + needs.changed-files.outputs.snapshot_sglang == 'true' || + needs.changed-files.outputs.snapshot_trtllm == 'true' || needs.changed-files.outputs.deploy == 'true' uses: ./.github/workflows/shared-compliance-audit.yml with: @@ -913,7 +931,8 @@ jobs: needs.changed-files.outputs.vllm == 'true' || needs.changed-files.outputs.deploy == 'true' || needs.changed-files.outputs.operator == 'true' || - needs.changed-files.outputs.snapshot == 'true') && + needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_vllm == 'true') && needs.vllm-build.result == 'success' uses: ./.github/workflows/shared-copy.yml with: @@ -928,7 +947,10 @@ jobs: needs: [changed-files, frontend-build] if: | always() && !cancelled() && - (needs.changed-files.outputs.snapshot == 'true') && + (needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_vllm == 'true' || + needs.changed-files.outputs.snapshot_sglang == 'true' || + needs.changed-files.outputs.snapshot_trtllm == 'true') && needs.frontend-build.result == 'success' uses: ./.github/workflows/shared-copy.yml with: @@ -943,7 +965,8 @@ jobs: needs: [changed-files, vllm-copy-to-acr] if: | always() && !cancelled() && - (needs.changed-files.outputs.snapshot == 'true') + (needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_vllm == 'true') runs-on: prod-default-v2 steps: - name: Checkout code @@ -976,7 +999,8 @@ jobs: needs.changed-files.outputs.sglang == 'true' || needs.changed-files.outputs.deploy == 'true' || needs.changed-files.outputs.operator == 'true' || - needs.changed-files.outputs.snapshot == 'true') && + needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_sglang == 'true') && needs.sglang-build.result == 'success' uses: ./.github/workflows/shared-copy.yml with: @@ -991,7 +1015,8 @@ jobs: needs: [changed-files, sglang-copy-to-acr] if: | always() && !cancelled() && - (needs.changed-files.outputs.snapshot == 'true') + (needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_sglang == 'true') runs-on: prod-default-v2 steps: - name: Checkout code @@ -1024,7 +1049,8 @@ jobs: needs.changed-files.outputs.trtllm == 'true' || needs.changed-files.outputs.deploy == 'true' || needs.changed-files.outputs.operator == 'true' || - needs.changed-files.outputs.snapshot == 'true') && + needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_trtllm == 'true') && needs.trtllm-build.result == 'success' uses: ./.github/workflows/shared-copy.yml with: @@ -1039,7 +1065,8 @@ jobs: needs: [changed-files, trtllm-copy-to-acr] if: | always() && !cancelled() && - (needs.changed-files.outputs.snapshot == 'true') + (needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_trtllm == 'true') runs-on: prod-default-v2 steps: - name: Checkout code @@ -1160,7 +1187,8 @@ jobs: name: vLLM DynamoCheckpoint Operator Setup if: | always() && !cancelled() && - (needs.changed-files.outputs.snapshot == 'true') && + (needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_vllm == 'true') && needs.operator.result == 'success' needs: [changed-files, operator] runs-on: prod-deploy-tester-v1 @@ -1190,7 +1218,8 @@ jobs: name: SGLang DynamoCheckpoint Operator Setup if: | always() && !cancelled() && - (needs.changed-files.outputs.snapshot == 'true') && + (needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_sglang == 'true') && needs.operator.result == 'success' needs: [changed-files, operator] runs-on: prod-deploy-tester-v1 @@ -1220,7 +1249,8 @@ jobs: name: TRTLLM DynamoCheckpoint Operator Setup if: | always() && !cancelled() && - (needs.changed-files.outputs.snapshot == 'true') && + (needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_trtllm == 'true') && needs.operator.result == 'success' needs: [changed-files, operator] runs-on: prod-deploy-tester-v1 @@ -1251,7 +1281,8 @@ jobs: needs: [changed-files, deploy-operator-checkpoint-vllm, snapshot-agent] if: | always() && !cancelled() && - (needs.changed-files.outputs.snapshot == 'true') && + (needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_vllm == 'true') && needs.deploy-operator-checkpoint-vllm.result == 'success' && needs.snapshot-agent.result == 'success' runs-on: prod-deploy-tester-v1 @@ -1298,7 +1329,8 @@ jobs: needs: [changed-files, deploy-operator-checkpoint-sglang, snapshot-agent] if: | always() && !cancelled() && - (needs.changed-files.outputs.snapshot == 'true') && + (needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_sglang == 'true') && needs.deploy-operator-checkpoint-sglang.result == 'success' && needs.snapshot-agent.result == 'success' runs-on: prod-deploy-tester-v1 @@ -1345,7 +1377,8 @@ jobs: needs: [changed-files, deploy-operator-checkpoint-trtllm, snapshot-agent] if: | always() && !cancelled() && - (needs.changed-files.outputs.snapshot == 'true') && + (needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_trtllm == 'true') && needs.deploy-operator-checkpoint-trtllm.result == 'success' && needs.snapshot-agent.result == 'success' runs-on: prod-deploy-tester-v1 @@ -1392,7 +1425,8 @@ jobs: needs: [changed-files, deploy-operator-checkpoint-vllm, deploy-snapshot-agent-checkpoint-vllm, snapshot-placeholder-vllm, frontend-copy-to-acr] if: | always() && !cancelled() && - (needs.changed-files.outputs.snapshot == 'true') && + (needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_vllm == 'true') && needs.deploy-snapshot-agent-checkpoint-vllm.result == 'success' && needs.snapshot-placeholder-vllm.result == 'success' && needs.frontend-copy-to-acr.result == 'success' @@ -1457,7 +1491,8 @@ jobs: needs: [changed-files, deploy-operator-checkpoint-sglang, deploy-snapshot-agent-checkpoint-sglang, snapshot-placeholder-sglang, frontend-copy-to-acr] if: | always() && !cancelled() && - (needs.changed-files.outputs.snapshot == 'true') && + (needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_sglang == 'true') && needs.deploy-snapshot-agent-checkpoint-sglang.result == 'success' && needs.snapshot-placeholder-sglang.result == 'success' && needs.frontend-copy-to-acr.result == 'success' @@ -1522,7 +1557,8 @@ jobs: needs: [changed-files, deploy-operator-checkpoint-trtllm, deploy-snapshot-agent-checkpoint-trtllm, snapshot-placeholder-trtllm, frontend-copy-to-acr] if: | always() && !cancelled() && - (needs.changed-files.outputs.snapshot == 'true') && + (needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_trtllm == 'true') && needs.deploy-snapshot-agent-checkpoint-trtllm.result == 'success' && needs.snapshot-placeholder-trtllm.result == 'success' && needs.frontend-copy-to-acr.result == 'success' @@ -1601,7 +1637,8 @@ jobs: name: vLLM DynamoCheckpoint Cleanup if: | always() && - (needs.changed-files.outputs.snapshot == 'true') + (needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_vllm == 'true') needs: [changed-files, deploy-operator-checkpoint-vllm, deploy-snapshot-agent-checkpoint-vllm, deploy-test-checkpoint-vllm] runs-on: prod-deploy-tester-v1 steps: @@ -1616,7 +1653,8 @@ jobs: name: SGLang DynamoCheckpoint Cleanup if: | always() && - (needs.changed-files.outputs.snapshot == 'true') + (needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_sglang == 'true') needs: [changed-files, deploy-operator-checkpoint-sglang, deploy-snapshot-agent-checkpoint-sglang, deploy-test-checkpoint-sglang] runs-on: prod-deploy-tester-v1 steps: @@ -1631,7 +1669,8 @@ jobs: name: TRTLLM DynamoCheckpoint Cleanup if: | always() && - (needs.changed-files.outputs.snapshot == 'true') + (needs.changed-files.outputs.snapshot == 'true' || + needs.changed-files.outputs.snapshot_trtllm == 'true') needs: [changed-files, deploy-operator-checkpoint-trtllm, deploy-snapshot-agent-checkpoint-trtllm, deploy-test-checkpoint-trtllm] runs-on: prod-deploy-tester-v1 steps: