diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index 7875c5c511c2..5b27be0c38ef 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -4,6 +4,7 @@ depends_on: steps: - label: ":nvidia: (L4) Distributed Comm Ops" key: distributed-comm-ops + automatic_shard: true timeout_in_minutes: 25 working_dir: "/vllm-workspace/tests" device: l4 @@ -21,6 +22,7 @@ steps: - pytest -v -s distributed/test_shm_storage.py mirror: amd: + automatic_shard: true label: ':amd: (MI300) Distributed Comm Ops' dind: false device: mi300_2 @@ -40,6 +42,7 @@ steps: - label: ":nvidia: (L4) Sharded RDT Weight Transfer" key: sharded-rdt-weight-transfer + automatic_shard: true device: l4 timeout_in_minutes: 20 working_dir: "/vllm-workspace/tests" @@ -55,6 +58,7 @@ steps: - pytest -v -s distributed/test_sharded_rdt_trainer.py mirror: amd: + automatic_shard: true label: ":amd: (MI355 DPX) Sharded RDT Weight Transfer" dind: false device: mi355_dpx @@ -130,6 +134,7 @@ steps: - label: ":nvidia: (L4) Distributed Compile + RPC" key: distributed-compile-rpc-tests-2-gpus + automatic_shard: true timeout_in_minutes: 65 working_dir: "/vllm-workspace/tests" device: l4 @@ -152,6 +157,7 @@ steps: - pytest -v -s ./compile/test_wrapper.py mirror: amd: + automatic_shard: true label: ":amd: (MI355) Distributed Compile + RPC" dind: false device: mi355_2 @@ -176,6 +182,7 @@ steps: - label: ":nvidia: (L4) Distributed Torchrun + Shutdown" key: distributed-torchrun-shutdown-tests-2-gpus + automatic_shard: true timeout_in_minutes: 30 working_dir: "/vllm-workspace/tests" device: l4 @@ -200,6 +207,7 @@ steps: - pytest -v -s v1/worker/test_worker_memory_snapshot.py mirror: amd: + automatic_shard: true label: ':amd: (MI300) Distributed Torchrun + Shutdown' dind: false device: mi300_2 @@ -369,6 +377,7 @@ steps: - label: ":nvidia: (L4) [Nightly] Distributed Compile + Comm" key: distributed-compile-comm-4-gpus + automatic_shard: true # Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs. optional: true timeout_in_minutes: 70 @@ -392,6 +401,7 @@ steps: - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node mirror: amd: + automatic_shard: true # Run on PRs even when the L4 step is optional (nightly only). optional: false label: ':amd: (MI355) Distributed Compile + Comm' @@ -416,6 +426,7 @@ steps: - label: ":nvidia: (H100) [Nightly] Distributed DP + EP" key: distributed-tests-8xh100 + automatic_shard: true timeout_in_minutes: 20 optional: true device: h100 @@ -441,6 +452,7 @@ steps: - pytest -v -s distributed/test_eplb_execute.py -k symm_mem mirror: amd: + automatic_shard: true label: ":amd: (MI355) Distributed DP + EP" dind: false device: mi355_8 @@ -577,6 +589,7 @@ steps: - label: ":nvidia: (B200) [Nightly] Distributed" key: distributed-tests-2xb200 + automatic_shard: true device: b200-k8s optional: true working_dir: "/vllm-workspace/" @@ -589,6 +602,7 @@ steps: - pytest -v -s tests/distributed/test_mnnvl_alltoall.py mirror: amd: + automatic_shard: true label: ":amd: (MI355) Distributed AgRs All2All" dind: false device: mi355_2 @@ -611,6 +625,7 @@ steps: - label: ":nvidia: (B200) GEMM-RS/AR" key: gemm-rs-ar-2xb200 + automatic_shard: true timeout_in_minutes: 20 device: b200-k8s working_dir: "/vllm-workspace/" @@ -623,6 +638,7 @@ steps: - pytest -v -s tests/kernels/test_gemm_rs_ar.py mirror: amd: + automatic_shard: true label: ":amd: (MI355) GEMM-RS-AR" dind: false device: mi355_4 @@ -779,6 +795,7 @@ steps: - label: ":nvidia: (L4) Distributed Preload" key: preload-2-gpus + automatic_shard: true timeout_in_minutes: 15 working_dir: "/vllm-workspace/tests" device: l4 @@ -791,6 +808,7 @@ steps: - pytest -v -s model_executor/model_loader/test_weight_cache.py -m 'distributed(num_gpus=2)' mirror: amd: + automatic_shard: true label: ":amd: (MI355) Distributed Preload" dind: false device: mi355_2 diff --git a/.buildkite/test_areas/expert_parallelism.yaml b/.buildkite/test_areas/expert_parallelism.yaml index 1ca3eacd48cb..e9c13dad6348 100644 --- a/.buildkite/test_areas/expert_parallelism.yaml +++ b/.buildkite/test_areas/expert_parallelism.yaml @@ -4,6 +4,7 @@ depends_on: steps: - label: ":nvidia: (H200 MIG 18GB) EPLB Algorithm" key: eplb-algorithm + automatic_shard: true timeout_in_minutes: 20 device: h200_18gb working_dir: "/vllm-workspace/tests" @@ -18,6 +19,7 @@ steps: - pytest -v -s distributed/test_eplb_utils.py mirror: amd: + automatic_shard: true label: ":amd: (MI355 DPX) EPLB Algorithm" dind: false device: mi355_dpx @@ -32,6 +34,7 @@ steps: - label: ":nvidia: (L4) [Nightly] EPLB Execution" key: eplb-execution + automatic_shard: true # Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs. optional: true timeout_in_minutes: 25 @@ -46,6 +49,7 @@ steps: - pytest -v -s distributed/test_eplb_spec_decode.py mirror: amd: + automatic_shard: true # Run on PRs even when the L4 step is optional (nightly only). optional: false label: ":amd: (MI355) EPLB Execution" diff --git a/.buildkite/test_areas/fault_tolerance.yaml b/.buildkite/test_areas/fault_tolerance.yaml index f9439880d454..b68d9b0fb1e4 100644 --- a/.buildkite/test_areas/fault_tolerance.yaml +++ b/.buildkite/test_areas/fault_tolerance.yaml @@ -62,6 +62,7 @@ steps: # Keep the AMD job: its commands do not run this installation. - label: ':amd: (MI355) MoRI EP Healthy Baseline' key: amd-fault-tolerance-e2e-2xh100 + automatic_shard: true timeout_in_minutes: 10 device: mi355_2 num_devices: 2 diff --git a/.buildkite/test_areas/models_distributed.yaml b/.buildkite/test_areas/models_distributed.yaml index bdaef9ec5a1b..22e38ab1aa24 100644 --- a/.buildkite/test_areas/models_distributed.yaml +++ b/.buildkite/test_areas/models_distributed.yaml @@ -4,6 +4,7 @@ depends_on: steps: - label: ":nvidia: (L4) [Nightly] Qwen4 Exp SP" key: qwen4-exp-sp-tests-4-gpus + automatic_shard: true # Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs. optional: true timeout_in_minutes: 30 @@ -22,6 +23,7 @@ steps: - pytest -v -s models/qwen4_exp/test_tpsp.py mirror: amd: + automatic_shard: true # Run on PRs even when the L4 step is optional (nightly only). optional: false label: ":amd: (MI355) Qwen4 Exp SP"