[ROCm][CI] Expand MI355 mirrors and route MIG-sized jobs to DPX

.buildkite/test-amd.yaml

@@ -3251,6 +3251,85 @@ steps:
   - pytest -v -s tests/compile/passes/test_fusion_attn.py tests/compile/passes/test_silu_mul_quant_fusion.py tests/compile/passes/test_silu_mul_quant_manual_fusion.py
   - pytest -v -s tests/compile/correctness_e2e/test_attn_quant.py
 
+- label: ":amd: (MI355) Sequence Parallel Correctness"
+  timeout_in_minutes: 180
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_2
+  num_gpus: 2
+  optional: true
+  working_dir: "/vllm-workspace/"
+  source_file_dependencies:
+  - .buildkite/test_areas/compile.yaml
+  - vllm/compilation/
+  - vllm/config/
+  - vllm/distributed/
+  - vllm/model_executor/kernels/linear/
+  - vllm/model_executor/layers/
+  - vllm/model_executor/models/llama.py
+  - vllm/v1/attention/
+  - vllm/v1/worker/
+  - vllm/v1/cudagraph_dispatcher.py
+  - vllm/platforms/rocm.py
+  - tests/compile/correctness_e2e/test_sequence_parallel.py
+  - tests/models/registry.py
+  - tests/utils.py
+  commands:
+  - export VLLM_TEST_CLEAN_GPU_MEMORY=1
+  - pytest -v -s tests/compile/correctness_e2e/test_sequence_parallel.py
+
+- label: ":amd: (MI355) AsyncTP Correctness"
+  timeout_in_minutes: 60
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_2
+  num_gpus: 2
+  optional: true
+  working_dir: "/vllm-workspace/"
+  source_file_dependencies:
+  - .buildkite/test_areas/compile.yaml
+  - vllm/compilation/
+  - vllm/config/
+  - vllm/distributed/
+  - vllm/model_executor/layers/
+  - vllm/model_executor/kernels/linear/
+  - vllm/model_executor/models/llama.py
+  - vllm/model_executor/models/qwen3.py
+  - vllm/v1/attention/
+  - vllm/_aiter_ops.py
+  - vllm/platforms/rocm.py
+  - vllm/transformers_utils/repo_utils.py
+  - tests/compile/correctness_e2e/test_async_tp.py
+  - tests/compile/fusions_e2e/common.py
+  - tests/models/registry.py
+  - tests/utils.py
+  commands:
+  - export VLLM_TEST_CLEAN_GPU_MEMORY=1
+  - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py::test_rocm_async_tp_bf16_output_correctness
+  - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py::test_async_tp_pass_correctness -k RedHatAI
+  - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py::test_async_tp_pass_nvfp4_correctness
+
+- label: ":amd: (MI355 DPX) Fusion E2E FP8 Config Sweep"
+  timeout_in_minutes: 45
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_dpx
+  num_gpus: 1
+  optional: true
+  working_dir: "/vllm-workspace/"
+  source_file_dependencies:
+  - .buildkite/test_areas/compile.yaml
+  - csrc/quantization/
+  - vllm/compilation/
+  - vllm/model_executor/
+  - vllm/v1/attention/
+  - vllm/_aiter_ops.py
+  - vllm/platforms/rocm.py
+  - tests/compile/fusions_e2e/
+  commands:
+  - rocm-smi
+  - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "test_tp1_fp8_fusions and inductor_partition and not +rms_norm and not True and ((Llama-3 and TRITON_ATTN) or (Qwen3-30B and +quant_fp8 and TRITON_ATTN) or (DeepSeek-Coder and +quant_fp8 and TRITON_MLA))"
+
 #--------------------------------------------------------  mi355 · distributed  --------------------------------------------------------#
 
 - label: ":amd: (MI355 DPX) EPLB Algorithm" # TBD
@@ -3522,6 +3601,26 @@ steps:
   - pytest -v -s tests/distributed/test_rocm_quick_reduce.py
   - pytest -v -s tests/distributed/test_quick_all_reduce.py
 
+- label: ":amd: (MI355) Distributed AgRs All2All"
+  timeout_in_minutes: 30
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_2
+  num_gpus: 2
+  optional: true
+  working_dir: "/vllm-workspace/"
+  source_file_dependencies:
+  - .buildkite/test_areas/distributed.yaml
+  - vllm/distributed/
+  - vllm/config/parallel.py
+  - vllm/config/vllm.py
+  - vllm/forward_context.py
+  - vllm/platforms/rocm.py
+  - tests/distributed/test_mnnvl_alltoall.py
+  - tests/utils.py
+  commands:
+  - pytest -v -s tests/distributed/test_mnnvl_alltoall.py::test_args_dispatch_combine
+
 - label: ":amd: (MI355) Fusion E2E TP2 Quick" # TBD
   timeout_in_minutes: 60
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3903,6 +4002,30 @@ steps:
   commands:
     - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
 
+- label: ":amd: (MI355 DPX) LM Eval Small Models"
+  timeout_in_minutes: 45
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_dpx
+  num_gpus: 1
+  optional: true
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - .buildkite/test_areas/lm_eval.yaml
+  - csrc/
+  - tests/evals/gsm8k/
+  - vllm/model_executor/layers/quantization
+  - vllm/model_executor/kernels/linear/
+  - vllm/model_executor/layers/fused_moe/
+  - vllm/model_executor/models/
+  - vllm/model_executor/model_loader/
+  - vllm/v1/attention/backends/
+  - vllm/v1/attention/selector.py
+  - vllm/_aiter_ops.py
+  - vllm/platforms/rocm.py
+  commands:
+  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-gfx950-small.txt
+
 - label: ":amd: (MI355 DPX) LM Eval Watermarking"
   timeout_in_minutes: 45
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3949,7 +4072,7 @@ steps:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark-dspark-tp2.txt
 
 - label: ":amd: (MI355) LM Eval Qwen3-5 Models" # TBD
-  timeout_in_minutes: 40
+  timeout_in_minutes: 75
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
   dind: false
   agent_pool: mi355_2
@@ -3968,13 +4091,59 @@ steps:
   - vllm/distributed/
   - vllm/model_executor/layers/fused_moe/
   - vllm/third_party/flash_linear_attention/ops/
+  - vllm/model_executor/layers/quantization/modelopt.py
+  - vllm/model_executor/kernels/linear/nvfp4/
+  - vllm/v1/attention/
+  - vllm/v1/worker/
   - tests/evals/gsm8k/
   - tests/utils.py
   - vllm/_aiter_ops.py
   - vllm/platforms/rocm.py
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt
 
+- label: ":amd: (MI355) LM Eval Large Models EP"
+  timeout_in_minutes: 120
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_2
+  num_gpus: 2
+  optional: true
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - .buildkite/test_areas/lm_eval.yaml
+  - tests/evals/gsm8k/
+  - tests/utils.py
+  - tests/conftest.py
+  - csrc/
+  - vllm/config/
+  - vllm/distributed/
+  - vllm/model_executor/models/config.py
+  - vllm/model_executor/models/registry.py
+  - vllm/model_executor/models/utils.py
+  - vllm/model_executor/models/qwen3_next.py
+  - vllm/model_executor/models/qwen3_next_mtp.py
+  - vllm/model_executor/models/nemotron_h.py
+  - vllm/model_executor/models/nemotron_h_mtp.py
+  - vllm/transformers_utils/configs/qwen3_next.py
+  - vllm/transformers_utils/configs/nemotron_h.py
+  - vllm/model_executor/model_loader/
+  - vllm/model_executor/layers/quantization/
+  - vllm/model_executor/layers/fused_moe/
+  - vllm/model_executor/layers/mamba/
+  - vllm/model_executor/layers/layernorm.py
+  - vllm/model_executor/layers/linear.py
+  - vllm/model_executor/kernels/linear/
+  - vllm/third_party/flash_linear_attention/ops/
+  - vllm/v1/attention/
+  - vllm/v1/spec_decode/
+  - vllm/v1/worker/
+  - vllm/_aiter_ops.py
+  - vllm/platforms/rocm.py
+  commands:
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell-ep-mi355.txt
+
 - label: ":amd: (MI355) Qwen3-8-Flash-Next-FP8 Accuracy Eval"
   timeout_in_minutes: 105
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -3996,11 +4165,11 @@ steps:
   - uv pip install --system 'evalscope==1.10.0'
   - pytest -s -v evals/qwen4_exp/test_accuracy.py --config-list-file=configs/models-rocm.txt
 
-- label: ":amd: (MI355) LM Eval TurboQuant k8v4"
+- label: ":amd: (MI355 DPX) LM Eval TurboQuant k8v4"
   timeout_in_minutes: 30
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
   dind: false
-  agent_pool: mi355_1
+  agent_pool: mi355_dpx
   num_gpus: 1
   optional: true
   working_dir: "/vllm-workspace/tests"
@@ -4022,11 +4191,11 @@ steps:
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k8v4-rocm.txt
 
-- label: ":amd: (MI355) LM Eval TurboQuant t4nc"
+- label: ":amd: (MI355 DPX) LM Eval TurboQuant t4nc"
   timeout_in_minutes: 30
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
   dind: false
-  agent_pool: mi355_1
+  agent_pool: mi355_dpx
   num_gpus: 1
   optional: true
   working_dir: "/vllm-workspace/tests"
@@ -4048,11 +4217,11 @@ steps:
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t4nc-rocm.txt
 
-- label: ":amd: (MI355) LM Eval TurboQuant k3v4nc"
+- label: ":amd: (MI355 DPX) LM Eval TurboQuant k3v4nc"
   timeout_in_minutes: 30
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
   dind: false
-  agent_pool: mi355_1
+  agent_pool: mi355_dpx
   num_gpus: 1
   optional: true
   working_dir: "/vllm-workspace/tests"
@@ -4074,11 +4243,11 @@ steps:
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k3v4nc-rocm.txt
 
-- label: ":amd: (MI355) LM Eval TurboQuant t3nc"
+- label: ":amd: (MI355 DPX) LM Eval TurboQuant t3nc"
   timeout_in_minutes: 30
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
   dind: false
-  agent_pool: mi355_1
+  agent_pool: mi355_dpx
   num_gpus: 1
   optional: true
   working_dir: "/vllm-workspace/tests"
@@ -4285,41 +4454,6 @@ steps:
   - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
   - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
 
-- label: ":amd: (MI355) Examples" # TBD
-  timeout_in_minutes: 75
-  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
-  dind: false
-  agent_pool: mi355_1
-  working_dir: "/vllm-workspace/examples"
-  source_file_dependencies:
-  - vllm/entrypoints
-  - vllm/multimodal
-  - examples/
-  - vllm/platforms/rocm.py
-  optional: true
-  commands:
-  # Basic
-  # Multi-modal models
-  # Pooling models
-  # Features demo
-  - pip install --no-deps tensorizer
-  - python3 basic/offline_inference/chat.py
-  - python3 basic/offline_inference/generate.py --model facebook/opt-125m
-  - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10
-  - python3 basic/offline_inference/classify.py
-  - python3 basic/offline_inference/embed.py
-  - python3 basic/offline_inference/score.py
-  - python3 generate/multimodal/audio_language_offline.py --seed 0
-  - python3 generate/multimodal/vision_language_offline.py --seed 0
-  - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
-  - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0
-  - python3 pooling/embed/vision_embedding_offline.py --seed 0
-  - python3 features/automatic_prefix_caching/prefix_caching_offline.py
-  - python3 deployment/llm_engine_example.py
-  - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
-  - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
-  - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
-
 #----------------------------------------------------------  mi355 · kernels  ----------------------------------------------------------#
 
 - label: ":amd: (MI355 DPX) Attention DiffKV Kernels"
@@ -5005,12 +5139,12 @@ steps:
 
 #-------------------------------------------------------  mi355 · quantization  --------------------------------------------------------#
 
-- label: ":amd: (MI355) Quantization Shard %N"
+- label: ":amd: (MI355 DPX) Quantization Shard %N"
   timeout_in_minutes: 115
   parallelism: 4
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
   dind: false
-  agent_pool: mi355_1
+  agent_pool: mi355_dpx
   working_dir: "/vllm-workspace/tests"
   source_file_dependencies:
   - csrc/
@@ -5080,6 +5214,34 @@ steps:
 
 #------------------------------------------------------------  mi355 · v1  -------------------------------------------------------------#
 
+- label: ":amd: (MI355) Spec Decode AL DFlash2 Nightly"
+  timeout_in_minutes: 60
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
+  dind: false
+  agent_pool: mi355_1
+  num_gpus: 1
+  optional: true
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - .buildkite/test_areas/spec_decode.yaml
+  - tests/v1/e2e/spec_decode/
+  - tests/evals/gsm8k/
+  - tests/utils.py
+  - vllm/v1/spec_decode/
+  - vllm/v1/worker/gpu/spec_decode/
+  - vllm/v1/worker/gpu/sample/
+  - vllm/v1/worker/gpu/model_runner.py
+  - vllm/v1/attention/
+  - vllm/model_executor/models/qwen3_dflash.py
+  - vllm/model_executor/models/qwen3_dflash2.py
+  - vllm/model_executor/models/qwen3_5.py
+  - vllm/model_executor/model_loader/
+  - vllm/model_executor/layers/quantization/
+  - vllm/model_executor/kernels/linear/
+  - vllm/platforms/rocm.py
+  commands:
+  - pytest -v -s v1/e2e/spec_decode/acceptance_rates/dflash/ -k "dflash2"
+
 - label: ":amd: (MI355 DPX) Spec Decode Draft Model" # TBD
   timeout_in_minutes: 60
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
@@ -5797,11 +5959,11 @@ steps:
   - pytest -v -s v1/test_outputs.py
 
 
-- label: ":amd: (MI355) V1 Spec Decode" # TBD
+- label: ":amd: (MI355 DPX) V1 Spec Decode" # TBD
   timeout_in_minutes: 50
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
   dind: false
-  agent_pool: mi355_1
+  agent_pool: mi355_dpx
   working_dir: "/vllm-workspace/tests"
   source_file_dependencies:
   - vllm/
@@ -5969,11 +6131,11 @@ steps:
   - amdmi355
   optional: true
 
-- label: ":amd: (MI355) MRV2 Sampler JIT Warmup"
+- label: ":amd: (MI355 DPX) MRV2 Sampler JIT Warmup"
   timeout_in_minutes: 10
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
   dind: false
-  agent_pool: mi355_1
+  agent_pool: mi355_dpx
   num_gpus: 1
   optional: true
   working_dir: "/vllm-workspace/"