From 93bc99d97fe5ca498d440c557a58d0ec4c801204 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sat, 11 Jul 2026 20:01:44 -0500 Subject: [PATCH 01/16] [CI][AMD] Enable native in-pod test execution Signed-off-by: Andreas Karatzas --- .../scripts/hardware_ci/run-amd-test.sh | 124 ++++++++++- .buildkite/test-amd.yaml | 201 +++++++++++++++++- .buildkite/test_areas/attention.yaml | 1 + .buildkite/test_areas/basic_correctness.yaml | 1 + .buildkite/test_areas/benchmarks.yaml | 1 + .buildkite/test_areas/disaggregated.yaml | 5 + .buildkite/test_areas/distributed.yaml | 1 + .buildkite/test_areas/engine.yaml | 5 + .buildkite/test_areas/entrypoints.yaml | 6 + .buildkite/test_areas/expert_parallelism.yaml | 1 + .buildkite/test_areas/kernels.yaml | 3 + .buildkite/test_areas/lm_eval.yaml | 2 + .buildkite/test_areas/lora.yaml | 1 + .buildkite/test_areas/misc.yaml | 7 + .buildkite/test_areas/model_executor.yaml | 1 + .buildkite/test_areas/models_basic.yaml | 1 + .buildkite/test_areas/models_language.yaml | 4 + .buildkite/test_areas/models_multimodal.yaml | 7 + .buildkite/test_areas/pytorch.yaml | 2 + .buildkite/test_areas/samplers.yaml | 1 + .buildkite/test_areas/spec_decode.yaml | 4 + .buildkite/test_areas/weight_loading.yaml | 1 + 22 files changed, 377 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index fee9ab04f4bd..711fb56d04f9 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -1,7 +1,7 @@ #!/bin/bash -# This script runs tests inside the corresponding ROCm docker container. -# It handles both single-node and multi-node test configurations. +# This script runs ROCm tests either directly in a native CI pod or inside the +# corresponding Docker container. Multi-node tests continue to use Docker. # # Multi-node detection: Instead of matching on fragile group names, we detect # multi-node jobs structurally by looking for the bracket command syntax @@ -168,6 +168,78 @@ EOF return 0 } +is_native_runtime() { + [[ "${AMD_CI_RUNTIME:-}" == "native" || "${NATIVE_CI:-}" == "true" ]] +} + +prepare_native_workspace() { + if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" != "1" ]]; then + echo "Native CI requires VLLM_CI_USE_ARTIFACTS=1" + return 1 + fi + if ! command -v buildkite-agent >/dev/null 2>&1; then + echo "buildkite-agent not found; cannot download ROCm wheel artifact" + return 1 + fi + + local artifact_glob="${VLLM_CI_ARTIFACT_GLOB:-artifacts/vllm-rocm-install/vllm-rocm-install.tar.gz}" + local archive="" + local wheel_dir="" + + artifact_work_dir=$(mktemp -d -t vllm-rocm-artifact.XXXXXX) + wheel_dir="${artifact_work_dir}/wheels" + mkdir -p "${wheel_dir}" + + echo "--- Downloading ROCm wheel artifact (native in-pod)" + if ! buildkite-agent artifact download "${artifact_glob}" "${artifact_work_dir}"; then + echo "Failed to download ${artifact_glob}" + return 1 + fi + + archive=$(find "${artifact_work_dir}" -name "vllm-rocm-install.tar.gz" -type f | head -1) + if [[ -z "${archive}" || ! -f "${archive}" ]]; then + echo "ROCm wheel artifact archive was not found" + return 1 + fi + + tar -xzf "${archive}" -C "${wheel_dir}" || return 1 + if ! ls "${wheel_dir}"/*.whl >/dev/null 2>&1; then + echo "ROCm wheel artifact did not contain a wheel" + return 1 + fi + if [[ ! -d "${wheel_dir}/tests" ]]; then + echo "ROCm wheel artifact did not contain the test workspace" + return 1 + fi + + echo "--- Installing ROCm wheel into pod environment" + python3 -m pip install --no-deps --force-reinstall "${wheel_dir}"/*.whl || return 1 + + echo "--- Preparing /vllm-workspace from artifact" + rm -rf /vllm-workspace || return 1 + mkdir -p /vllm-workspace || return 1 + tar -C "${wheel_dir}" --exclude='*.whl' -cf - . \ + | tar -C /vllm-workspace -xf - || return 1 + + return 0 +} + +run_native_preflight() { + : "${TMPDIR:=/tmp}" + : "${TORCHINDUCTOR_CACHE_DIR:=/tmp/vllm-native/torchinductor}" + : "${TRITON_CACHE_DIR:=/tmp/vllm-native/triton}" + : "${VLLM_CACHE_ROOT:=/home/buildkite-agent/huggingface/vllm-cache}" + : "${XDG_CACHE_HOME:=/tmp/vllm-native/xdg}" + export TMPDIR TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT XDG_CACHE_HOME + + mkdir -p "${TMPDIR}" \ + "${TORCHINDUCTOR_CACHE_DIR}" \ + "${TRITON_CACHE_DIR}" \ + "${VLLM_CACHE_ROOT}" \ + "${XDG_CACHE_HOME}" || return 1 + python3 -c "import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing" +} + is_multi_node() { local cmds="$1" # Primary signal: NUM_NODES environment variable set by the pipeline @@ -354,6 +426,54 @@ re_quote_pytest_markers() { echo "--- ROCm info" rocminfo +if is_native_runtime; then + echo "--- Native in-pod ROCm CI (AMD_CI_RUNTIME=${AMD_CI_RUNTIME:-unset}, NATIVE_CI=${NATIVE_CI:-unset})" + artifact_work_dir="" + + cleanup_native_workspace() { + if [[ -n "${artifact_work_dir}" ]]; then + rm -rf "${artifact_work_dir}" + fi + } + trap cleanup_native_workspace EXIT + + if ! prepare_native_workspace; then + echo "Failed to prepare native test workspace" + exit 1 + fi + + if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then + commands="${VLLM_TEST_COMMANDS}" + commands_source="env" + else + commands="$*" + commands_source="argv" + if [[ -z "$commands" ]]; then + echo "Error: No test commands provided for native CI." >&2 + exit 1 + fi + fi + + if [[ "$commands_source" == "argv" ]]; then + commands=$(re_quote_pytest_markers "$commands") + fi + + if is_multi_node "$commands"; then + echo "Native CI does not support multi-node jobs yet." + exit 1 + fi + + export PYTHONPATH="/vllm-workspace" + : "${HF_HUB_DOWNLOAD_TIMEOUT:=300}" + : "${HF_HUB_ETAG_TIMEOUT:=60}" + export HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT + + echo "Native test commands: $commands" + run_native_preflight || exit 1 + /bin/bash -c "${commands}" + handle_pytest_exit "$?" +fi + # --- Docker status --- report_docker_usage diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index ba7823e420a4..f2d6462871e6 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -14,7 +14,8 @@ # command(str): the single command to run for tests. incompatible with commands. # commands(list): the list of commands to run for the test. incompatible with command. # mirror_hardwares(list): the list of hardware to run the test on as well. currently only supports [amdexperimental] -# num_gpus(int): override the number of GPUs for the test. defaults to 1 GPU. currently supports 2,4. +# native_ci(bool): run directly in the AMD Kubernetes pod instead of Docker-in-Docker. +# num_gpus(int): override the number of GPUs for the test. defaults to 1 GPU. currently supports 2,4,8. # num_nodes(int): whether to simulate multi-node setup by launching multiple containers on one host, # in this case, commands must be specified. the first command runs on the first host, the second # command runs on the second host. @@ -116,6 +117,7 @@ steps: - label: PyTorch Fullgraph Smoke Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -135,6 +137,7 @@ steps: - label: Pipeline + Context Parallelism (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -158,6 +161,7 @@ steps: - label: Kernels Helion Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -172,6 +176,7 @@ steps: - label: Kernels Mamba Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -188,6 +193,7 @@ steps: - label: Basic Models Test (Other CPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 no_gpu: true optional: true @@ -204,6 +210,7 @@ steps: - label: Language Models Test (MTEB) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -215,6 +222,7 @@ steps: - label: Language Models Test (PPL) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -228,6 +236,7 @@ steps: - label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -243,6 +252,7 @@ steps: - label: Batch Invariance (H100-MI250) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -261,6 +271,7 @@ steps: - label: Cudagraph # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -277,6 +288,7 @@ steps: - label: e2e Core (1 GPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -290,6 +302,7 @@ steps: - label: e2e Scheduling (1 GPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -303,6 +316,7 @@ steps: - label: Engine (1 GPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -317,6 +331,7 @@ steps: - label: Spec Decode Draft Model # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -334,6 +349,7 @@ steps: - label: Spec Decode Speculators + MTP # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -352,6 +368,7 @@ steps: - label: V1 attention (H100-MI250) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -370,6 +387,7 @@ steps: - label: Async Engine, Inputs, Utils, Worker, Config (CPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 optional: true no_gpu: true @@ -411,6 +429,7 @@ steps: - label: Rust Frontend Cargo Style + Clippy # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 no_gpu: true working_dir: "/vllm-workspace" @@ -425,6 +444,7 @@ steps: - label: Rust Frontend Cargo Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 no_gpu: true working_dir: "/vllm-workspace" @@ -441,6 +461,7 @@ steps: - label: Docker Build Metadata (ROCm) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + native_ci: true agent_pool: mi250_1 no_gpu: true working_dir: "/vllm-workspace/tests" @@ -469,6 +490,7 @@ steps: - label: Basic Correctness # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -486,6 +508,7 @@ steps: - label: Distributed Model Tests (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 num_gpus: 2 optional: true @@ -515,6 +538,7 @@ steps: - label: Benchmarks CLI Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -529,6 +553,7 @@ steps: - label: PyTorch Compilation Unit Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -548,6 +573,7 @@ steps: - label: Fusion E2E Config Sweep (H100-MI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 num_gpus: 1 working_dir: "/vllm-workspace/" @@ -568,6 +594,7 @@ steps: - label: Fusion E2E Quick (H100-MI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 num_gpus: 1 working_dir: "/vllm-workspace/" @@ -589,6 +616,7 @@ steps: - label: PyTorch Compilation Passes Unit Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -600,6 +628,7 @@ steps: - label: PyTorch Fullgraph # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -617,6 +646,7 @@ steps: - label: Pytorch Nightly Dependency Override Check # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true soft_fail: true @@ -630,6 +660,7 @@ steps: - label: Distributed Compile Unit Tests (2xH100-2xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 num_gpus: 2 working_dir: "/vllm-workspace/" @@ -650,6 +681,7 @@ steps: - label: Platform Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -664,6 +696,7 @@ steps: - label: Async Engine, Inputs, Utils, Worker # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -681,6 +714,7 @@ steps: - label: Distributed Comm Ops # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -697,6 +731,7 @@ steps: - label: EPLB Algorithm # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -711,6 +746,7 @@ steps: - label: EPLB Execution # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -726,6 +762,7 @@ steps: - label: Distributed Tests (2xH100-2xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 num_gpus: 2 working_dir: "/vllm-workspace/" @@ -752,6 +789,7 @@ steps: - label: Distributed Tests (4xA100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 optional: true @@ -767,6 +805,7 @@ steps: - label: Distributed Torchrun + Examples (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -792,6 +831,7 @@ steps: - label: Elastic EP Scaling Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 optional: true @@ -809,6 +849,7 @@ steps: - label: RayExecutorV2 (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -831,6 +872,7 @@ steps: - label: Distributed Tests (8xH100-8xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_8 num_gpus: 8 optional: true @@ -851,6 +893,7 @@ steps: - label: Entrypoints Unit Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -866,6 +909,7 @@ steps: - label: Entrypoints Integration (LLM) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true fast_check: true @@ -882,6 +926,7 @@ steps: - label: Entrypoints Integration (API Server) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true fast_check: true @@ -899,6 +944,7 @@ steps: - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 fast_check: true optional: true @@ -914,6 +960,7 @@ steps: - label: Entrypoints Integration (API Server OpenAI - Part 2) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 fast_check: true optional: true @@ -930,6 +977,7 @@ steps: - label: Entrypoints Integration (API Server Generate) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true fast_check: true @@ -950,6 +998,7 @@ steps: - label: Entrypoints Integration (Responses API) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -963,6 +1012,7 @@ steps: - label: Entrypoints Integration (Speech to Text) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -976,6 +1026,7 @@ steps: - label: Entrypoints Integration (Multimodal) timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -989,6 +1040,7 @@ steps: - label: Entrypoints Integration (Pooling) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -1002,6 +1054,7 @@ steps: - label: OpenAI API correctness # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1023,6 +1076,7 @@ steps: - label: DeepSeek V2-Lite Prefetch Offload Accuracy (H100-MI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 num_gpus: 1 working_dir: "/vllm-workspace" @@ -1043,6 +1097,7 @@ steps: - label: LM Eval Small Models # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1060,6 +1115,7 @@ steps: - label: MRCR Eval Small Models # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1077,6 +1133,7 @@ steps: - label: LM Eval Small Models (MI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/.buildkite/lm-eval-harness" @@ -1095,6 +1152,7 @@ steps: - label: Multi-Modal Accuracy Eval (Small Models) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/.buildkite/lm-eval-harness" @@ -1110,6 +1168,7 @@ steps: - label: GPQA Eval (GPT-OSS) (2xH100-2xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 num_gpus: 2 optional: true @@ -1132,7 +1191,9 @@ steps: - label: LM Eval Small Models (2xB200-2xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 + num_gpus: 2 working_dir: "/vllm-workspace/tests" source_file_dependencies: - csrc/ @@ -1149,6 +1210,7 @@ steps: - label: DeepSeek V2-Lite Sync EPLB Accuracy (4xH100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 optional: true @@ -1171,6 +1233,7 @@ steps: - label: LM Eval Large Models (4xA100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 optional: true @@ -1191,7 +1254,9 @@ steps: - label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (4xH100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 + num_gpus: 4 optional: true working_dir: "/vllm-workspace" source_file_dependencies: @@ -1211,6 +1276,7 @@ steps: - label: Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy (4xH100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 optional: true @@ -1232,6 +1298,7 @@ steps: - label: Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 optional: true @@ -1254,6 +1321,7 @@ steps: - label: LM Eval Large Models (8xH200-8xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_8 optional: true num_gpus: 8 @@ -1276,6 +1344,7 @@ steps: - label: ROCm LM Eval Large Models (8 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_8 optional: true num_gpus: 8 @@ -1299,6 +1368,7 @@ steps: - label: Examples # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/examples" @@ -1335,6 +1405,7 @@ steps: - label: vLLM IR Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/" @@ -1350,6 +1421,7 @@ steps: - label: Kernels Attention Test %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true parallelism: 2 @@ -1368,6 +1440,7 @@ steps: - label: Kernels Core Operation Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1384,6 +1457,7 @@ steps: - label: Kernels KDA Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1399,6 +1473,7 @@ steps: - label: Kernels MoE Test %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true parallelism: 5 @@ -1420,6 +1495,7 @@ steps: - label: Kernels Quantization Test %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true parallelism: 2 @@ -1438,7 +1514,9 @@ steps: - label: Kernels FP8 MoE Test (2xH100-2xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 + num_gpus: 2 optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1457,6 +1535,7 @@ steps: - label: LoRA %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 parallelism: 4 optional: true @@ -1471,6 +1550,7 @@ steps: - label: LoRA TP (Distributed) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -1491,6 +1571,7 @@ steps: - label: Model Executor # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1513,6 +1594,7 @@ steps: - label: Model Runner V2 Core Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1537,6 +1619,7 @@ steps: - label: Model Runner V2 Examples # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/examples" @@ -1571,6 +1654,7 @@ steps: - label: Model Runner V2 Distributed (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 num_gpus: 2 optional: true @@ -1592,6 +1676,7 @@ steps: - label: Model Runner V2 Pipeline Parallelism (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 optional: true @@ -1611,6 +1696,7 @@ steps: - label: Model Runner V2 Spec Decode # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1635,6 +1721,7 @@ steps: - label: Basic Models Tests (Extra Initialization) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 parallelism: 6 working_dir: "/vllm-workspace/tests" @@ -1651,6 +1738,7 @@ steps: - label: Basic Models Tests (Initialization) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1663,6 +1751,7 @@ steps: - label: Basic Models Tests (Other) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1679,6 +1768,7 @@ steps: - label: Language Models Test (Extended Pooling) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1691,6 +1781,7 @@ steps: - label: Language Models Tests (Standard) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1704,6 +1795,7 @@ steps: - label: Language Models Tests (Extra Standard) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true parallelism: 2 @@ -1728,6 +1820,7 @@ steps: - label: Multi-Modal Models (Extended Generation 1) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1742,6 +1835,7 @@ steps: - label: Multi-Modal Models (Extended Generation 2) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1755,6 +1849,7 @@ steps: - label: Multi-Modal Models (Extended Generation 3) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1767,6 +1862,7 @@ steps: - label: "Multi-Modal Models (Standard) 1: qwen2" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1780,6 +1876,7 @@ steps: - label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1794,6 +1891,7 @@ steps: - label: "Multi-Modal Models (Standard) 4: other + whisper" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1808,6 +1906,7 @@ steps: - label: Multi-Modal Processor # 1h 42m timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1821,6 +1920,7 @@ steps: - label: Multi-Modal Processor (CPU) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 parallelism: 4 no_gpu: true @@ -1838,6 +1938,7 @@ steps: - label: Quantized Models Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1854,6 +1955,7 @@ steps: - label: Transformers Nightly Models (Shardable) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 parallelism: 4 optional: true @@ -1876,6 +1978,7 @@ steps: - label: Transformers Nightly Models (Single) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/" @@ -1903,6 +2006,7 @@ steps: - label: Plugin Tests (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 num_gpus: 2 optional: true @@ -1954,6 +2058,7 @@ steps: - label: GGUF Plugin # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true soft_fail: true @@ -1972,6 +2077,7 @@ steps: - label: Rust Frontend OpenAI Coverage # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2004,6 +2110,7 @@ steps: - label: Rust Frontend Serve Admin Coverage # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2031,6 +2138,7 @@ steps: - label: Rust Frontend Core Correctness # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2048,6 +2156,7 @@ steps: - label: Rust Frontend Tool Use # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2066,6 +2175,7 @@ steps: - label: Rust Frontend Distributed # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 optional: true @@ -2094,6 +2204,7 @@ steps: - label: Quantization # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2120,6 +2231,7 @@ steps: - label: ROCm AITER Ops Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2137,6 +2249,7 @@ steps: - label: Samplers Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2154,6 +2267,8 @@ steps: #------------------------------------------------------------ mi300 · misc ------------------------------------------------------------# +# Native CI intentionally remains disabled: this test requires the full source +# tree, while the native wheel artifact contains only the test workspace. - label: Python-only Installation # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -2170,6 +2285,7 @@ steps: - label: Regression # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2184,6 +2300,7 @@ steps: - label: Ray Dependency Compatibility Check # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 working_dir: "/" source_file_dependencies: @@ -2198,6 +2315,7 @@ steps: - label: Acceptance Length Test (Large Models) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2213,6 +2331,7 @@ steps: - label: e2e Scheduling (1 GPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2226,6 +2345,7 @@ steps: - label: Engine (1 GPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2240,6 +2360,7 @@ steps: - label: Spec Decode Draft Model # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2257,6 +2378,7 @@ steps: - label: Spec Decode Eagle # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2274,6 +2396,7 @@ steps: - label: Spec Decode Ngram + Suffix # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2291,6 +2414,7 @@ steps: - label: Spec Decode Speculators + MTP # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2309,6 +2433,7 @@ steps: - label: Speculators Correctness # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2340,6 +2465,7 @@ steps: - label: Extract Hidden States Integration # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -2371,6 +2497,7 @@ steps: - label: V1 attention (H100-MI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2388,6 +2515,7 @@ steps: - label: V1 Core + KV + Metrics # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2414,6 +2542,7 @@ steps: - label: V1 others (CPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 no_gpu: true optional: true @@ -2431,6 +2560,7 @@ steps: - label: V1 Sample + Logits # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2451,6 +2581,7 @@ steps: - label: Distributed DP Tests (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 optional: true num_gpus: 2 @@ -2474,6 +2605,7 @@ steps: - label: Metrics, Tracing (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 optional: true num_gpus: 2 @@ -2492,7 +2624,9 @@ steps: - label: V1 e2e (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 + num_gpus: 2 optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2504,6 +2638,7 @@ steps: - label: NixlConnector PD + Spec Decode acceptance (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 num_gpus: 2 optional: true @@ -2520,6 +2655,7 @@ steps: - label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 optional: true @@ -2535,6 +2671,7 @@ steps: - label: Distributed DP Tests (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -2556,6 +2693,7 @@ steps: - label: Distributed NixlConnector PD accuracy (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 optional: true @@ -2571,6 +2709,7 @@ steps: - label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 optional: true @@ -2586,6 +2725,7 @@ steps: - label: Hybrid SSM NixlConnector PD accuracy tests (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 num_gpus: 4 optional: true @@ -2601,6 +2741,7 @@ steps: - label: Hybrid SSM NixlConnector PD prefix cache test (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 num_gpus: 2 optional: true @@ -2618,6 +2759,7 @@ steps: - label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 num_gpus: 2 optional: true @@ -2636,6 +2778,7 @@ steps: - label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 num_gpus: 2 optional: true @@ -2654,7 +2797,9 @@ steps: - label: V1 e2e (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 + num_gpus: 4 optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2666,7 +2811,9 @@ steps: - label: V1 e2e (4xH100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_4 + num_gpus: 4 optional: true source_file_dependencies: - vllm/v1/attention/backends/utils.py @@ -2680,6 +2827,7 @@ steps: - label: Weight Loading Multiple GPU # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -2692,6 +2840,7 @@ steps: - label: Weight Loading Multiple GPU - Large Models # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true agent_pool: mi300_2 num_gpus: 2 optional: true @@ -2713,6 +2862,7 @@ steps: - label: Distributed Compile + RPC Tests (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] + native_ci: true agent_pool: mi325_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -2738,6 +2888,7 @@ steps: - label: Distributed Torchrun + Shutdown Tests (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] + native_ci: true agent_pool: mi325_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -2761,6 +2912,7 @@ steps: - label: Distributed Compile + Comm (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] + native_ci: true agent_pool: mi325_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -2784,6 +2936,7 @@ steps: - label: Engine # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] + native_ci: true agent_pool: mi325_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2802,6 +2955,7 @@ steps: - label: LM Eval Large Models (4xH100-4xMI325) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] + native_ci: true agent_pool: mi325_4 num_gpus: 4 optional: true @@ -2824,6 +2978,7 @@ steps: - label: Language Models Test (Extended Generation) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] + native_ci: true agent_pool: mi325_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2837,6 +2992,7 @@ steps: - label: Language Models Tests (Hybrid) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] + native_ci: true agent_pool: mi325_1 parallelism: 2 optional: true @@ -2854,6 +3010,7 @@ steps: - label: Multi-Modal Models (Extended Pooling) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] + native_ci: true agent_pool: mi325_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2866,6 +3023,7 @@ steps: - label: "Multi-Modal Models (Standard) 2: qwen3 + gemma" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] + native_ci: true agent_pool: mi325_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2881,6 +3039,7 @@ steps: - label: V1 Spec Decode # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] + native_ci: true agent_pool: mi325_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2900,6 +3059,7 @@ steps: - label: Attention Benchmarks Smoke Test (B200-MI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_2 num_gpus: 2 working_dir: "/vllm-workspace/" @@ -2916,6 +3076,7 @@ steps: - label: Distributed Tests (2xH100-2xMI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_2 num_gpus: 2 optional: true @@ -2960,6 +3121,7 @@ steps: - label: Entrypoints Integration (API Server) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 optional: true fast_check: true @@ -2977,6 +3139,7 @@ steps: - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 fast_check: true optional: true @@ -2992,6 +3155,7 @@ steps: - label: Entrypoints Integration (API Server OpenAI - Part 2) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 fast_check: true optional: true @@ -3008,6 +3172,7 @@ steps: - label: Entrypoints Integration (API Server Generate) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 fast_check: true optional: true @@ -3028,6 +3193,7 @@ steps: - label: Entrypoints Integration (Speech to Text) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355] + native_ci: true agent_pool: mi355_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -3041,6 +3207,7 @@ steps: - label: Entrypoints Integration (Multimodal) timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355] + native_ci: true agent_pool: mi355_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -3054,6 +3221,7 @@ steps: - label: Entrypoints Integration (Pooling) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -3069,6 +3237,7 @@ steps: - label: GPQA Eval (GPT-OSS) (2xB200-2xMI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_2 num_gpus: 2 optional: true @@ -3091,6 +3260,7 @@ steps: - label: LM Eval Qwen3-5 Models (B200-MI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_2 num_gpus: 2 optional: true @@ -3113,7 +3283,9 @@ steps: - label: LM Eval Small Models (2xB200-2xMI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_2 + num_gpus: 2 optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3131,6 +3303,7 @@ steps: - label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (B200-MI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_2 num_gpus: 2 working_dir: "/vllm-workspace" @@ -3151,6 +3324,7 @@ steps: - label: LM Eval Large Models (4xH100-4xMI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_4 num_gpus: 4 optional: true @@ -3173,6 +3347,7 @@ steps: - label: Examples # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/examples" source_file_dependencies: @@ -3208,6 +3383,7 @@ steps: - label: Kernels (B200-MI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/" source_file_dependencies: @@ -3233,6 +3409,7 @@ steps: - label: Kernels Attention Test %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 parallelism: 2 working_dir: "/vllm-workspace/tests" @@ -3250,6 +3427,7 @@ steps: - label: Kernels MoE Test %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 parallelism: 5 working_dir: "/vllm-workspace/tests" @@ -3270,6 +3448,7 @@ steps: - label: Kernels Quantization Test %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 parallelism: 2 working_dir: "/vllm-workspace/tests" @@ -3287,7 +3466,9 @@ steps: - label: Kernels FP8 MoE Test (2xH100-2xMI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_2 + num_gpus: 2 working_dir: "/vllm-workspace/tests" source_file_dependencies: - csrc/moe/ @@ -3305,6 +3486,7 @@ steps: - label: Language Models Test (Extended Generation) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3318,6 +3500,7 @@ steps: - label: Language Models Test (Extended Pooling) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3330,6 +3513,7 @@ steps: - label: Language Models Test (PPL) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3357,6 +3541,7 @@ steps: - label: Language Models Tests (Standard) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3371,6 +3556,7 @@ steps: - label: Multi-Modal Models (Extended Generation 1) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3385,6 +3571,7 @@ steps: - label: Multi-Modal Models (Extended Generation 3) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3397,6 +3584,7 @@ steps: - label: Multi-Modal Models (Extended Pooling) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3409,6 +3597,7 @@ steps: - label: "Multi-Modal Models (Standard) 1: qwen2" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3422,6 +3611,7 @@ steps: - label: "Multi-Modal Models (Standard) 4: other + whisper" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3438,6 +3628,7 @@ steps: - label: Quantized Models Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3454,6 +3645,7 @@ steps: - label: Quantization # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3498,6 +3690,7 @@ steps: - label: V1 attention (B200-MI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3514,6 +3707,7 @@ steps: - label: V1 Core + KV + Metrics # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3540,6 +3734,7 @@ steps: - label: V1 Sample + Logits # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3560,6 +3755,7 @@ steps: - label: V1 Spec Decode # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3573,6 +3769,7 @@ steps: - label: Weight Loading Multiple GPU # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -3585,6 +3782,7 @@ steps: - label: Weight Loading Multiple GPU - Large Models # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_2 working_dir: "/vllm-workspace/tests" num_gpus: 2 @@ -3600,6 +3798,7 @@ steps: - label: Regression # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" diff --git a/.buildkite/test_areas/attention.yaml b/.buildkite/test_areas/attention.yaml index aadea2908c68..29b94de98bf7 100644 --- a/.buildkite/test_areas/attention.yaml +++ b/.buildkite/test_areas/attention.yaml @@ -16,6 +16,7 @@ steps: parallelism: 2 mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 95 depends_on: diff --git a/.buildkite/test_areas/basic_correctness.yaml b/.buildkite/test_areas/basic_correctness.yaml index 1b92babb3ce7..74c42425d319 100644 --- a/.buildkite/test_areas/basic_correctness.yaml +++ b/.buildkite/test_areas/basic_correctness.yaml @@ -18,6 +18,7 @@ steps: - pytest -v -s basic_correctness/test_cpu_offload.py mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 70 depends_on: diff --git a/.buildkite/test_areas/benchmarks.yaml b/.buildkite/test_areas/benchmarks.yaml index 7c26a8d6e982..e8ef928569e7 100644 --- a/.buildkite/test_areas/benchmarks.yaml +++ b/.buildkite/test_areas/benchmarks.yaml @@ -13,6 +13,7 @@ steps: - pytest -v -s benchmarks/ mirror: amd: + native_ci: true device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index 38020d91ad00..fff7f502d164 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -15,6 +15,7 @@ steps: - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: + native_ci: true device: mi300_4 timeout_in_minutes: 85 depends_on: @@ -65,6 +66,7 @@ steps: - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: + native_ci: true device: mi300_4 timeout_in_minutes: 60 depends_on: @@ -90,6 +92,7 @@ steps: - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: + native_ci: true device: mi300_4 timeout_in_minutes: 85 depends_on: @@ -115,6 +118,7 @@ steps: - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: + native_ci: true device: mi300_4 timeout_in_minutes: 80 depends_on: @@ -171,6 +175,7 @@ steps: - bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh mirror: amd: + native_ci: true device: mi300_2 timeout_in_minutes: 70 depends_on: diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index b519132c6f1f..72f8ab402819 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -39,6 +39,7 @@ steps: - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py mirror: amd: + native_ci: true device: mi300_2 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index 1c50ab7a1132..aa6ddd1f21a7 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -28,6 +28,7 @@ steps: - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 50 depends_on: @@ -44,6 +45,7 @@ steps: - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 55 depends_on: @@ -60,6 +62,7 @@ steps: - pytest -v -s v1/e2e/general/test_async_scheduling.py mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 70 depends_on: @@ -76,6 +79,7 @@ steps: - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 60 depends_on: @@ -114,6 +118,7 @@ steps: - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism" mirror: amd: + native_ci: true device: mi300_2 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index 2db4c5ad5a21..01149150e247 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -28,6 +28,7 @@ steps: - pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests mirror: amd: + native_ci: true device: mi325_1 # TODO(akaratza): Test after Torch >= 2.12 bump soft_fail: true @@ -50,6 +51,7 @@ steps: - pytest -v -s entrypoints/scale_out mirror: amd: + native_ci: true device: mi325_1 depends_on: - image-build-amd @@ -67,6 +69,7 @@ steps: - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 65 depends_on: @@ -86,6 +89,7 @@ steps: - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 80 depends_on: @@ -108,6 +112,7 @@ steps: - pytest -v -s entrypoints/anthropic mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 65 depends_on: @@ -169,6 +174,7 @@ steps: - pytest -s entrypoints/openai/correctness/ mirror: amd: + native_ci: true device: mi325_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/expert_parallelism.yaml b/.buildkite/test_areas/expert_parallelism.yaml index d02ffeb77485..03ac500c844c 100644 --- a/.buildkite/test_areas/expert_parallelism.yaml +++ b/.buildkite/test_areas/expert_parallelism.yaml @@ -16,6 +16,7 @@ steps: - pytest -v -s distributed/test_eplb_utils.py mirror: amd: + native_ci: true device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 894cda341193..02f2bde0ff15 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -79,6 +79,7 @@ steps: parallelism: 2 mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 90 depends_on: @@ -117,6 +118,7 @@ steps: parallelism: 2 mirror: amd: + native_ci: true device: mi325_1 source_file_dependencies: - csrc/quantization/ @@ -147,6 +149,7 @@ steps: parallelism: 5 mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 65 source_file_dependencies: diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 1f7d27875367..7703e895da2b 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -14,6 +14,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 55 depends_on: @@ -117,6 +118,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200.txt mirror: amd: + native_ci: true device: mi300_8 timeout_in_minutes: 60 depends_on: diff --git a/.buildkite/test_areas/lora.yaml b/.buildkite/test_areas/lora.yaml index 46a3710ea808..98f73eb42a4c 100644 --- a/.buildkite/test_areas/lora.yaml +++ b/.buildkite/test_areas/lora.yaml @@ -14,6 +14,7 @@ steps: parallelism: 4 mirror: amd: + native_ci: true device: mi325_1 working_dir: "/vllm-workspace/tests" timeout_in_minutes: 65 diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 3365d09c7b85..f5de7d82034c 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -23,6 +23,7 @@ steps: - pytest -v -s -m 'not slow_test' v1/spec_decode mirror: amd: + native_ci: true device: mi300_1 timeout_in_minutes: 75 depends_on: @@ -58,6 +59,7 @@ steps: - pytest -v -s v1/test_outputs.py mirror: amd: + native_ci: true device: mi325_1 depends_on: - image-build-amd @@ -109,6 +111,7 @@ steps: - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 75 depends_on: @@ -228,6 +231,7 @@ steps: - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 mirror: amd: + native_ci: true device: mi325_1 source_file_dependencies: - vllm/entrypoints @@ -264,6 +268,7 @@ steps: - pytest -v -s v1/tracing mirror: amd: + native_ci: true device: mi325_2 depends_on: - image-build-amd @@ -281,6 +286,8 @@ steps: commands: - bash standalone_tests/python_only_compile.sh mirror: + # Native CI intentionally remains disabled because this test requires the + # full source tree rather than the thin ROCm wheel/test artifact. amd: device: mi325_1 timeout_in_minutes: 45 diff --git a/.buildkite/test_areas/model_executor.yaml b/.buildkite/test_areas/model_executor.yaml index 4280e600df4b..30ca2f038ec3 100644 --- a/.buildkite/test_areas/model_executor.yaml +++ b/.buildkite/test_areas/model_executor.yaml @@ -25,6 +25,7 @@ steps: - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread mirror: amd: + native_ci: true device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index 95827a894588..f6a410cf285a 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -42,6 +42,7 @@ steps: - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py mirror: amd: + native_ci: true device: mi325_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index d89d3fabaecf..9a9d6344ca63 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -15,6 +15,7 @@ steps: - pytest -v -s models/language -m 'core_model and (not slow_test)' mirror: amd: + native_ci: true device: mi300_1 depends_on: - image-build-amd @@ -35,6 +36,7 @@ steps: parallelism: 2 mirror: amd: + native_ci: true device: mi300_1 depends_on: - image-build-amd @@ -66,6 +68,7 @@ steps: parallelism: 2 mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 70 depends_on: @@ -113,6 +116,7 @@ steps: - pytest -v -s models/language/pooling -m 'not core_model' mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 100 depends_on: diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index 473192f03f9a..f0ca7d5af403 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -14,6 +14,7 @@ steps: - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model mirror: amd: + native_ci: true device: mi325_1 depends_on: - image-build-amd @@ -31,6 +32,7 @@ steps: - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model mirror: amd: + native_ci: true device: mi325_1 depends_on: - image-build-amd @@ -47,6 +49,7 @@ steps: - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model mirror: amd: + native_ci: true device: mi325_1 depends_on: - image-build-amd @@ -65,6 +68,7 @@ steps: - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work mirror: amd: + native_ci: true device: mi325_1 depends_on: - image-build-amd @@ -107,6 +111,7 @@ steps: - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1 mirror: amd: + native_ci: true device: mi300_1 depends_on: - image-build-amd @@ -129,6 +134,7 @@ steps: - pytest -v -s models/multimodal/test_mapping.py mirror: amd: + native_ci: true device: mi325_1 depends_on: - image-build-amd @@ -164,6 +170,7 @@ steps: - pytest -v -s models/multimodal/pooling -m 'not core_model' mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 75 depends_on: diff --git a/.buildkite/test_areas/pytorch.yaml b/.buildkite/test_areas/pytorch.yaml index 72cb00696644..6915eb962a40 100644 --- a/.buildkite/test_areas/pytorch.yaml +++ b/.buildkite/test_areas/pytorch.yaml @@ -109,6 +109,7 @@ steps: - pytest -s -v compile/passes --ignore compile/passes/distributed mirror: amd: + native_ci: true device: mi300_1 timeout_in_minutes: 65 depends_on: @@ -197,6 +198,7 @@ steps: - bash standalone_tests/pytorch_nightly_dependency.sh mirror: amd: + native_ci: true device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/samplers.yaml b/.buildkite/test_areas/samplers.yaml index 2e7cd4a623e6..ce698caee992 100644 --- a/.buildkite/test_areas/samplers.yaml +++ b/.buildkite/test_areas/samplers.yaml @@ -19,6 +19,7 @@ steps: - VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers mirror: amd: + native_ci: true device: mi325_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index 096c324bb8e2..0a639701c17b 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -14,6 +14,7 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "eagle_correctness" mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 60 depends_on: @@ -53,6 +54,7 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness" mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 65 depends_on: @@ -92,6 +94,7 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "ngram or suffix" mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 55 # TODO(akaratza): Test after Torch >= 2.12 bump @@ -119,6 +122,7 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference" mirror: amd: + native_ci: true device: mi325_1 timeout_in_minutes: 55 depends_on: diff --git a/.buildkite/test_areas/weight_loading.yaml b/.buildkite/test_areas/weight_loading.yaml index eeb24a49d8d3..0538902d50d0 100644 --- a/.buildkite/test_areas/weight_loading.yaml +++ b/.buildkite/test_areas/weight_loading.yaml @@ -15,6 +15,7 @@ steps: - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt mirror: amd: + native_ci: true device: mi300_2 depends_on: - image-build-amd From 36e27d3c1edd83e01639a413be6900d18e612f51 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sat, 11 Jul 2026 23:56:53 -0500 Subject: [PATCH 02/16] [CI][AMD] Limit native execution to MI300 Signed-off-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 103 ++++--------------- .buildkite/test_areas/attention.yaml | 1 - .buildkite/test_areas/basic_correctness.yaml | 1 - .buildkite/test_areas/engine.yaml | 4 - .buildkite/test_areas/entrypoints.yaml | 6 -- .buildkite/test_areas/kernels.yaml | 3 - .buildkite/test_areas/lm_eval.yaml | 1 - .buildkite/test_areas/lora.yaml | 1 - .buildkite/test_areas/misc.yaml | 8 +- .buildkite/test_areas/models_basic.yaml | 1 - .buildkite/test_areas/models_language.yaml | 2 - .buildkite/test_areas/models_multimodal.yaml | 6 -- .buildkite/test_areas/samplers.yaml | 1 - .buildkite/test_areas/spec_decode.yaml | 4 - 14 files changed, 21 insertions(+), 121 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index f2d6462871e6..a8fc9fc2caf3 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -14,7 +14,8 @@ # command(str): the single command to run for tests. incompatible with commands. # commands(list): the list of commands to run for the test. incompatible with command. # mirror_hardwares(list): the list of hardware to run the test on as well. currently only supports [amdexperimental] -# native_ci(bool): run directly in the AMD Kubernetes pod instead of Docker-in-Docker. +# native_ci(bool): run MI300 jobs directly in the AMD Kubernetes pod instead of Docker-in-Docker. +# Omit this for MI250, MI325, and MI355 jobs, which still use the legacy Docker path. # num_gpus(int): override the number of GPUs for the test. defaults to 1 GPU. currently supports 2,4,8. # num_nodes(int): whether to simulate multi-node setup by launching multiple containers on one host, # in this case, commands must be specified. the first command runs on the first host, the second @@ -117,7 +118,6 @@ steps: - label: PyTorch Fullgraph Smoke Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -137,7 +137,6 @@ steps: - label: Pipeline + Context Parallelism (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -161,7 +160,6 @@ steps: - label: Kernels Helion Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -176,7 +174,6 @@ steps: - label: Kernels Mamba Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -193,7 +190,6 @@ steps: - label: Basic Models Test (Other CPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 no_gpu: true optional: true @@ -210,7 +206,6 @@ steps: - label: Language Models Test (MTEB) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -222,7 +217,6 @@ steps: - label: Language Models Test (PPL) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -236,7 +230,6 @@ steps: - label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -252,7 +245,6 @@ steps: - label: Batch Invariance (H100-MI250) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -271,7 +263,6 @@ steps: - label: Cudagraph # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -288,7 +279,6 @@ steps: - label: e2e Core (1 GPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -302,7 +292,6 @@ steps: - label: e2e Scheduling (1 GPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -316,7 +305,6 @@ steps: - label: Engine (1 GPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -331,7 +319,6 @@ steps: - label: Spec Decode Draft Model # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -349,7 +336,6 @@ steps: - label: Spec Decode Speculators + MTP # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -368,7 +354,6 @@ steps: - label: V1 attention (H100-MI250) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -387,7 +372,6 @@ steps: - label: Async Engine, Inputs, Utils, Worker, Config (CPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 optional: true no_gpu: true @@ -429,7 +413,6 @@ steps: - label: Rust Frontend Cargo Style + Clippy # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 no_gpu: true working_dir: "/vllm-workspace" @@ -444,7 +427,6 @@ steps: - label: Rust Frontend Cargo Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 no_gpu: true working_dir: "/vllm-workspace" @@ -461,7 +443,6 @@ steps: - label: Docker Build Metadata (ROCm) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - native_ci: true agent_pool: mi250_1 no_gpu: true working_dir: "/vllm-workspace/tests" @@ -2267,21 +2248,6 @@ steps: #------------------------------------------------------------ mi300 · misc ------------------------------------------------------------# -# Native CI intentionally remains disabled: this test requires the full source -# tree, while the native wheel artifact contains only the test workspace. -- label: Python-only Installation # TBD - timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - tests/standalone_tests/python_only_compile.sh - - setup.py - - vllm/platforms/rocm.py - commands: - - bash standalone_tests/python_only_compile.sh - - label: Regression # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -2862,7 +2828,6 @@ steps: - label: Distributed Compile + RPC Tests (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - native_ci: true agent_pool: mi325_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -2888,7 +2853,6 @@ steps: - label: Distributed Torchrun + Shutdown Tests (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - native_ci: true agent_pool: mi325_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -2912,7 +2876,6 @@ steps: - label: Distributed Compile + Comm (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - native_ci: true agent_pool: mi325_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -2936,7 +2899,6 @@ steps: - label: Engine # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - native_ci: true agent_pool: mi325_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2955,7 +2917,6 @@ steps: - label: LM Eval Large Models (4xH100-4xMI325) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - native_ci: true agent_pool: mi325_4 num_gpus: 4 optional: true @@ -2978,7 +2939,6 @@ steps: - label: Language Models Test (Extended Generation) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - native_ci: true agent_pool: mi325_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2992,7 +2952,6 @@ steps: - label: Language Models Tests (Hybrid) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - native_ci: true agent_pool: mi325_1 parallelism: 2 optional: true @@ -3010,7 +2969,6 @@ steps: - label: Multi-Modal Models (Extended Pooling) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - native_ci: true agent_pool: mi325_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3023,7 +2981,6 @@ steps: - label: "Multi-Modal Models (Standard) 2: qwen3 + gemma" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - native_ci: true agent_pool: mi325_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3039,7 +2996,6 @@ steps: - label: V1 Spec Decode # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - native_ci: true agent_pool: mi325_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3048,6 +3004,23 @@ steps: commands: - pytest -v -s -m 'not slow_test' v1/spec_decode +#----------------------------------------------------------- mi325 · misc ------------------------------------------------------------# + +# This test keeps the legacy Docker path because it requires the full source +# tree rather than the thin native artifact. +- label: Python-only Installation # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] + agent_pool: mi325_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - tests/standalone_tests/python_only_compile.sh + - setup.py + - vllm/platforms/rocm.py + commands: + - bash standalone_tests/python_only_compile.sh + ######################################################################################################################################### # # # MI355 (gfx950) tests # @@ -3059,7 +3032,6 @@ steps: - label: Attention Benchmarks Smoke Test (B200-MI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_2 num_gpus: 2 working_dir: "/vllm-workspace/" @@ -3076,7 +3048,6 @@ steps: - label: Distributed Tests (2xH100-2xMI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_2 num_gpus: 2 optional: true @@ -3121,7 +3092,6 @@ steps: - label: Entrypoints Integration (API Server) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 optional: true fast_check: true @@ -3139,7 +3109,6 @@ steps: - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 fast_check: true optional: true @@ -3155,7 +3124,6 @@ steps: - label: Entrypoints Integration (API Server OpenAI - Part 2) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 fast_check: true optional: true @@ -3172,7 +3140,6 @@ steps: - label: Entrypoints Integration (API Server Generate) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 fast_check: true optional: true @@ -3193,7 +3160,6 @@ steps: - label: Entrypoints Integration (Speech to Text) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355] - native_ci: true agent_pool: mi355_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -3207,7 +3173,6 @@ steps: - label: Entrypoints Integration (Multimodal) timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355] - native_ci: true agent_pool: mi355_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -3221,7 +3186,6 @@ steps: - label: Entrypoints Integration (Pooling) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -3237,7 +3201,6 @@ steps: - label: GPQA Eval (GPT-OSS) (2xB200-2xMI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_2 num_gpus: 2 optional: true @@ -3260,7 +3223,6 @@ steps: - label: LM Eval Qwen3-5 Models (B200-MI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_2 num_gpus: 2 optional: true @@ -3283,7 +3245,6 @@ steps: - label: LM Eval Small Models (2xB200-2xMI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_2 num_gpus: 2 optional: true @@ -3303,7 +3264,6 @@ steps: - label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (B200-MI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_2 num_gpus: 2 working_dir: "/vllm-workspace" @@ -3324,7 +3284,6 @@ steps: - label: LM Eval Large Models (4xH100-4xMI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_4 num_gpus: 4 optional: true @@ -3347,7 +3306,6 @@ steps: - label: Examples # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/examples" source_file_dependencies: @@ -3383,7 +3341,6 @@ steps: - label: Kernels (B200-MI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/" source_file_dependencies: @@ -3409,7 +3366,6 @@ steps: - label: Kernels Attention Test %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 parallelism: 2 working_dir: "/vllm-workspace/tests" @@ -3427,7 +3383,6 @@ steps: - label: Kernels MoE Test %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 parallelism: 5 working_dir: "/vllm-workspace/tests" @@ -3448,7 +3403,6 @@ steps: - label: Kernels Quantization Test %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 parallelism: 2 working_dir: "/vllm-workspace/tests" @@ -3466,7 +3420,6 @@ steps: - label: Kernels FP8 MoE Test (2xH100-2xMI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -3486,7 +3439,6 @@ steps: - label: Language Models Test (Extended Generation) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3500,7 +3452,6 @@ steps: - label: Language Models Test (Extended Pooling) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3513,7 +3464,6 @@ steps: - label: Language Models Test (PPL) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3541,7 +3491,6 @@ steps: - label: Language Models Tests (Standard) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3556,7 +3505,6 @@ steps: - label: Multi-Modal Models (Extended Generation 1) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3571,7 +3519,6 @@ steps: - label: Multi-Modal Models (Extended Generation 3) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3584,7 +3531,6 @@ steps: - label: Multi-Modal Models (Extended Pooling) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3597,7 +3543,6 @@ steps: - label: "Multi-Modal Models (Standard) 1: qwen2" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3611,7 +3556,6 @@ steps: - label: "Multi-Modal Models (Standard) 4: other + whisper" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3628,7 +3572,6 @@ steps: - label: Quantized Models Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3645,7 +3588,6 @@ steps: - label: Quantization # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3690,7 +3632,6 @@ steps: - label: V1 attention (B200-MI355) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3707,7 +3648,6 @@ steps: - label: V1 Core + KV + Metrics # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3734,7 +3674,6 @@ steps: - label: V1 Sample + Logits # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" @@ -3755,7 +3694,6 @@ steps: - label: V1 Spec Decode # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3769,7 +3707,6 @@ steps: - label: Weight Loading Multiple GPU # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -3782,7 +3719,6 @@ steps: - label: Weight Loading Multiple GPU - Large Models # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_2 working_dir: "/vllm-workspace/tests" num_gpus: 2 @@ -3798,7 +3734,6 @@ steps: - label: Regression # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - native_ci: true agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" diff --git a/.buildkite/test_areas/attention.yaml b/.buildkite/test_areas/attention.yaml index 29b94de98bf7..aadea2908c68 100644 --- a/.buildkite/test_areas/attention.yaml +++ b/.buildkite/test_areas/attention.yaml @@ -16,7 +16,6 @@ steps: parallelism: 2 mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 95 depends_on: diff --git a/.buildkite/test_areas/basic_correctness.yaml b/.buildkite/test_areas/basic_correctness.yaml index 74c42425d319..1b92babb3ce7 100644 --- a/.buildkite/test_areas/basic_correctness.yaml +++ b/.buildkite/test_areas/basic_correctness.yaml @@ -18,7 +18,6 @@ steps: - pytest -v -s basic_correctness/test_cpu_offload.py mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 70 depends_on: diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index aa6ddd1f21a7..b32f3ae9c4cb 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -28,7 +28,6 @@ steps: - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 50 depends_on: @@ -45,7 +44,6 @@ steps: - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 55 depends_on: @@ -62,7 +60,6 @@ steps: - pytest -v -s v1/e2e/general/test_async_scheduling.py mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 70 depends_on: @@ -79,7 +76,6 @@ steps: - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 60 depends_on: diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index 01149150e247..2db4c5ad5a21 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -28,7 +28,6 @@ steps: - pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests mirror: amd: - native_ci: true device: mi325_1 # TODO(akaratza): Test after Torch >= 2.12 bump soft_fail: true @@ -51,7 +50,6 @@ steps: - pytest -v -s entrypoints/scale_out mirror: amd: - native_ci: true device: mi325_1 depends_on: - image-build-amd @@ -69,7 +67,6 @@ steps: - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 65 depends_on: @@ -89,7 +86,6 @@ steps: - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 80 depends_on: @@ -112,7 +108,6 @@ steps: - pytest -v -s entrypoints/anthropic mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 65 depends_on: @@ -174,7 +169,6 @@ steps: - pytest -s entrypoints/openai/correctness/ mirror: amd: - native_ci: true device: mi325_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 02f2bde0ff15..894cda341193 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -79,7 +79,6 @@ steps: parallelism: 2 mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 90 depends_on: @@ -118,7 +117,6 @@ steps: parallelism: 2 mirror: amd: - native_ci: true device: mi325_1 source_file_dependencies: - csrc/quantization/ @@ -149,7 +147,6 @@ steps: parallelism: 5 mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 65 source_file_dependencies: diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 7703e895da2b..a701f967337d 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -14,7 +14,6 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 55 depends_on: diff --git a/.buildkite/test_areas/lora.yaml b/.buildkite/test_areas/lora.yaml index 98f73eb42a4c..46a3710ea808 100644 --- a/.buildkite/test_areas/lora.yaml +++ b/.buildkite/test_areas/lora.yaml @@ -14,7 +14,6 @@ steps: parallelism: 4 mirror: amd: - native_ci: true device: mi325_1 working_dir: "/vllm-workspace/tests" timeout_in_minutes: 65 diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index f5de7d82034c..9dd83a040982 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -59,7 +59,6 @@ steps: - pytest -v -s v1/test_outputs.py mirror: amd: - native_ci: true device: mi325_1 depends_on: - image-build-amd @@ -111,7 +110,6 @@ steps: - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 75 depends_on: @@ -231,7 +229,6 @@ steps: - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 mirror: amd: - native_ci: true device: mi325_1 source_file_dependencies: - vllm/entrypoints @@ -268,7 +265,6 @@ steps: - pytest -v -s v1/tracing mirror: amd: - native_ci: true device: mi325_2 depends_on: - image-build-amd @@ -286,8 +282,8 @@ steps: commands: - bash standalone_tests/python_only_compile.sh mirror: - # Native CI intentionally remains disabled because this test requires the - # full source tree rather than the thin ROCm wheel/test artifact. + # This MI325 mirror intentionally keeps the legacy Docker path because the + # test requires the full source tree rather than the thin native artifact. amd: device: mi325_1 timeout_in_minutes: 45 diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index f6a410cf285a..95827a894588 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -42,7 +42,6 @@ steps: - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py mirror: amd: - native_ci: true device: mi325_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index 9a9d6344ca63..654648d43502 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -68,7 +68,6 @@ steps: parallelism: 2 mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 70 depends_on: @@ -116,7 +115,6 @@ steps: - pytest -v -s models/language/pooling -m 'not core_model' mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 100 depends_on: diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index f0ca7d5af403..18c57f08966d 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -14,7 +14,6 @@ steps: - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model mirror: amd: - native_ci: true device: mi325_1 depends_on: - image-build-amd @@ -32,7 +31,6 @@ steps: - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model mirror: amd: - native_ci: true device: mi325_1 depends_on: - image-build-amd @@ -49,7 +47,6 @@ steps: - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model mirror: amd: - native_ci: true device: mi325_1 depends_on: - image-build-amd @@ -68,7 +65,6 @@ steps: - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work mirror: amd: - native_ci: true device: mi325_1 depends_on: - image-build-amd @@ -134,7 +130,6 @@ steps: - pytest -v -s models/multimodal/test_mapping.py mirror: amd: - native_ci: true device: mi325_1 depends_on: - image-build-amd @@ -170,7 +165,6 @@ steps: - pytest -v -s models/multimodal/pooling -m 'not core_model' mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 75 depends_on: diff --git a/.buildkite/test_areas/samplers.yaml b/.buildkite/test_areas/samplers.yaml index ce698caee992..2e7cd4a623e6 100644 --- a/.buildkite/test_areas/samplers.yaml +++ b/.buildkite/test_areas/samplers.yaml @@ -19,7 +19,6 @@ steps: - VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers mirror: amd: - native_ci: true device: mi325_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index 0a639701c17b..096c324bb8e2 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -14,7 +14,6 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "eagle_correctness" mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 60 depends_on: @@ -54,7 +53,6 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness" mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 65 depends_on: @@ -94,7 +92,6 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "ngram or suffix" mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 55 # TODO(akaratza): Test after Torch >= 2.12 bump @@ -122,7 +119,6 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference" mirror: amd: - native_ci: true device: mi325_1 timeout_in_minutes: 55 depends_on: From 99ad09c1560e2f62bb0cdf26f2d9ddbf18e8a748 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sun, 12 Jul 2026 01:03:07 -0500 Subject: [PATCH 03/16] [CI][AMD] Harden native MI300 test execution Signed-off-by: Andreas Karatzas --- .buildkite/scripts/ci-bake-rocm.sh | 62 ++- .../scripts/hardware_ci/run-amd-test.sh | 269 +++++++++-- .buildkite/test-amd.yaml | 19 +- .buildkite/test_areas/expert_parallelism.yaml | 11 + .pre-commit-config.yaml | 7 + docker/Dockerfile.rocm | 1 + tests/tools/test_amd_native_ci_scripts.py | 427 ++++++++++++++++++ tests/tools/test_validate_amd_native_ci.py | 238 ++++++++++ tools/pre_commit/validate_amd_native_ci.py | 168 +++++++ 9 files changed, 1141 insertions(+), 61 deletions(-) create mode 100644 tests/tools/test_amd_native_ci_scripts.py create mode 100644 tests/tools/test_validate_amd_native_ci.py create mode 100644 tools/pre_commit/validate_amd_native_ci.py diff --git a/.buildkite/scripts/ci-bake-rocm.sh b/.buildkite/scripts/ci-bake-rocm.sh index 22391291b449..48cc58d4f6e6 100644 --- a/.buildkite/scripts/ci-bake-rocm.sh +++ b/.buildkite/scripts/ci-bake-rocm.sh @@ -1898,8 +1898,8 @@ confirm_remote_image_push() { fi if [[ -z "${remote_revision}" \ - && ${IMAGE_EXISTED_BEFORE_BUILD} -eq 0 \ - && image_tag_is_commit_scoped ]]; then + && ${IMAGE_EXISTED_BEFORE_BUILD} -eq 0 ]] \ + && image_tag_is_commit_scoped; then echo "Remote image exists under a commit-scoped tag; accepting push despite missing revision label." return 0 fi @@ -2029,36 +2029,57 @@ upload_wheel_artifacts_if_present() { local wheel_dir="./wheel-export" local artifact_dir="artifacts/vllm-rocm-install" local archive_name="vllm-rocm-install.tar.gz" + local metadata_dir="${wheel_dir}/.vllm-ci-artifact" + local native_base_image="" local whl="" local whl_name="" + local -a wheels=() if ! should_upload_wheel_artifacts; then return 0 fi - if [[ ! -d "${wheel_dir}" ]] || ! ls "${wheel_dir}"/*.whl >/dev/null 2>&1; then - echo "No ROCm wheel artifacts found in ${wheel_dir}" - return 0 + if [[ -d "${wheel_dir}" ]]; then + mapfile -t wheels < <(find "${wheel_dir}" -maxdepth 1 -type f -name '*.whl' -print) + fi + if [[ ${#wheels[@]} -ne 1 ]]; then + echo "Expected exactly one ROCm wheel in ${wheel_dir}; found ${#wheels[@]}" >&2 + return 1 + fi + whl="${wheels[0]}" + whl_name=$(basename "${whl}") + native_base_image="${CI_BASE_IMAGE_TAG_COMMIT_REF:-${CI_BASE_IMAGE:-}}" + if [[ -z "${native_base_image}" ]]; then + echo "Native ROCm artifact requires a ci_base image reference" >&2 + return 1 fi echo "--- :package: Uploading ROCm vLLM install artifact" - mkdir -p "${artifact_dir}" + rm -rf "${artifact_dir}" "${metadata_dir}" + mkdir -p "${artifact_dir}" "${metadata_dir}" + + printf '%s\n' "${BUILDKITE_COMMIT:-local}" > "${metadata_dir}/commit.txt" + printf '%s\n' "${native_base_image}" > "${metadata_dir}/native-base-image.txt" + printf '%s\n' "${CI_BASE_IMAGE:-}" > "${metadata_dir}/ci-base-image.txt" + printf '%s\n' "${IMAGE_TAG:-}" > "${metadata_dir}/fallback-image.txt" + printf '%s\n' "${whl_name}" > "${metadata_dir}/wheel-filename.txt" tar -C "${wheel_dir}" -czf "${artifact_dir}/${archive_name}" . + ( + cd "${artifact_dir}" + sha256sum "${archive_name}" > "${archive_name}.sha256" + ) echo "Created ${archive_name}: $(du -sh "${artifact_dir}/${archive_name}" | cut -f1)" - printf '%s\n' "${CI_BASE_IMAGE:-}" > "${artifact_dir}/ci-base-image.txt" - printf '%s\n' "${IMAGE_TAG:-}" > "${artifact_dir}/fallback-image.txt" - - for whl in "${wheel_dir}"/*.whl; do - [[ -f "${whl}" ]] || continue - whl_name=$(basename "${whl}") - cp "${whl}" "${artifact_dir}/${whl_name}" - echo "Copied ${whl_name}: $(du -sh "${artifact_dir}/${whl_name}" | cut -f1)" - done + cp "${metadata_dir}"/*.txt "${artifact_dir}/" + cp "${whl}" "${artifact_dir}/${whl_name}" + echo "Copied ${whl_name}: $(du -sh "${artifact_dir}/${whl_name}" | cut -f1)" if command -v buildkite-agent >/dev/null 2>&1; then - buildkite-agent artifact upload "${artifact_dir}/*" + buildkite-agent artifact upload "${artifact_dir}/*" || return 1 echo "ROCm vLLM install artifacts uploaded to ${artifact_dir}/" + elif [[ "${BUILDKITE:-false}" == "true" ]]; then + echo "buildkite-agent not found; cannot upload required ROCm artifacts" >&2 + return 1 else echo "Not in Buildkite, skipping artifact upload" fi @@ -2090,9 +2111,18 @@ main() { echo "BAKE_PRINT_ONLY=1 set; skipping build" return 0 fi + if should_upload_wheel_artifacts; then + # wheel-export is an output directory, not a BuildKit cache. Starting + # clean prevents a failed/retried export from packaging a stale wheel. + rm -rf ./wheel-export + fi seed_dependency_caches_if_needed run_bake upload_wheel_artifacts_if_present } +if [[ "${VLLM_CI_BAKE_ROCM_LIB_ONLY:-0}" == "1" ]]; then + return 0 2>/dev/null || exit 0 +fi + main "$@" diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 711fb56d04f9..0aa394d6609c 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -145,7 +145,11 @@ prepare_artifact_image() { fi cp "${wheel_dir}"/*.whl "${context_dir}/wheels/" || return 1 - tar -C "${wheel_dir}" --exclude='*.whl' -cf - . \ + tar -C "${wheel_dir}" \ + --exclude='*.whl' \ + --exclude='.vllm-ci-artifact' \ + --exclude='./.vllm-ci-artifact' \ + -cf - . \ | tar -C "${workspace_dir}" -xf - || return 1 cat > "${context_dir}/Dockerfile" <<'EOF' ARG BASE_IMAGE @@ -172,6 +176,36 @@ is_native_runtime() { [[ "${AMD_CI_RUNTIME:-}" == "native" || "${NATIVE_CI:-}" == "true" ]] } +validate_native_workspace() { + local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}" + local workspace_real="" + local checkout_real="" + local workspace_mount="" + + mkdir -p "${workspace_dir}" || return 1 + workspace_real=$(readlink -m "${workspace_dir}") || return 1 + if [[ -n "${BUILDKITE_BUILD_CHECKOUT_PATH:-}" ]]; then + checkout_real=$(readlink -m "${BUILDKITE_BUILD_CHECKOUT_PATH}") || return 1 + if [[ "${checkout_real}" == "${workspace_real}" \ + || "${checkout_real}" == "${workspace_real}/"* \ + || "${workspace_real}" == "${checkout_real}/"* ]]; then + echo "Refusing to replace ${workspace_real}; it overlaps the Buildkite checkout ${checkout_real}" >&2 + return 1 + fi + fi + if [[ "${VLLM_CI_REQUIRE_WORKSPACE_MOUNT:-1}" == "1" ]]; then + if ! command -v findmnt >/dev/null 2>&1; then + echo "findmnt is required to verify the native workspace mount" >&2 + return 1 + fi + workspace_mount=$(findmnt -n -T "${workspace_real}" -o TARGET 2>/dev/null || true) + if [[ "$(readlink -m "${workspace_mount:-/}")" != "${workspace_real}" ]]; then + echo "Native CI requires a dedicated volume mounted at ${workspace_real}" >&2 + return 1 + fi + fi +} + prepare_native_workspace() { if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" != "1" ]]; then echo "Native CI requires VLLM_CI_USE_ARTIFACTS=1" @@ -181,63 +215,207 @@ prepare_native_workspace() { echo "buildkite-agent not found; cannot download ROCm wheel artifact" return 1 fi + validate_native_workspace || return 1 local artifact_glob="${VLLM_CI_ARTIFACT_GLOB:-artifacts/vllm-rocm-install/vllm-rocm-install.tar.gz}" + local artifact_checksum_glob="${VLLM_CI_ARTIFACT_CHECKSUM_GLOB:-${artifact_glob}.sha256}" + local artifact_step="${VLLM_CI_ARTIFACT_STEP:-image-build-amd}" local archive="" + local checksum="" + local download_dir="" + local metadata_dir="" + local recorded_base="" + local recorded_commit="" + local recorded_wheel="" + local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}" local wheel_dir="" + local attempt=0 + local attempt_dir="" + local -a archives=() + local -a checksums=() + local -a wheels=() - artifact_work_dir=$(mktemp -d -t vllm-rocm-artifact.XXXXXX) + artifact_work_dir=$(mktemp -d -t vllm-rocm-artifact.XXXXXX) || return 1 wheel_dir="${artifact_work_dir}/wheels" - mkdir -p "${wheel_dir}" - - echo "--- Downloading ROCm wheel artifact (native in-pod)" - if ! buildkite-agent artifact download "${artifact_glob}" "${artifact_work_dir}"; then - echo "Failed to download ${artifact_glob}" + mkdir -p "${wheel_dir}" || return 1 + + echo "--- Downloading ROCm wheel artifact from ${artifact_step} (native in-pod)" + for attempt in 1 2 3; do + attempt_dir="${artifact_work_dir}/download-${attempt}" + rm -rf "${attempt_dir}" || return 1 + mkdir -p "${attempt_dir}" || return 1 + if buildkite-agent artifact download \ + "${artifact_glob}" "${attempt_dir}" --step "${artifact_step}" \ + && buildkite-agent artifact download \ + "${artifact_checksum_glob}" "${attempt_dir}" --step "${artifact_step}"; then + download_dir="${attempt_dir}" + break + fi + echo "Artifact download attempt ${attempt}/3 failed" + if [[ "${attempt}" -lt 3 ]]; then + sleep $((attempt * 2)) + fi + done + if [[ -z "${download_dir}" ]]; then + echo "Failed to download ${artifact_glob} and ${artifact_checksum_glob} from ${artifact_step}" return 1 fi - archive=$(find "${artifact_work_dir}" -name "vllm-rocm-install.tar.gz" -type f | head -1) - if [[ -z "${archive}" || ! -f "${archive}" ]]; then - echo "ROCm wheel artifact archive was not found" + mapfile -t archives < <( + find "${download_dir}" -name "vllm-rocm-install.tar.gz" -type f -print + ) + mapfile -t checksums < <( + find "${download_dir}" -name "vllm-rocm-install.tar.gz.sha256" -type f -print + ) + if [[ ${#archives[@]} -ne 1 || ${#checksums[@]} -ne 1 ]]; then + echo "Expected exactly one ROCm archive and checksum; found ${#archives[@]} archive(s) and ${#checksums[@]} checksum(s)" >&2 + return 1 + fi + archive="${archives[0]}" + checksum="${checksums[0]}" + if [[ "$(dirname "${archive}")" != "$(dirname "${checksum}")" ]]; then + echo "ROCm archive and checksum were downloaded to different directories" >&2 return 1 fi + ( + cd "$(dirname "${archive}")" + sha256sum -c "$(basename "${checksum}")" + ) || return 1 + + tar --no-same-owner -xzf "${archive}" -C "${wheel_dir}" || return 1 + mapfile -t wheels < <( + find "${wheel_dir}" -maxdepth 1 -type f -name '*.whl' -print + ) + if [[ ${#wheels[@]} -ne 1 ]]; then + echo "ROCm artifact must contain exactly one top-level wheel; found ${#wheels[@]}" >&2 + return 1 + fi + metadata_dir="${wheel_dir}/.vllm-ci-artifact" + for metadata_file in commit.txt native-base-image.txt wheel-filename.txt; do + if [[ ! -s "${metadata_dir}/${metadata_file}" ]]; then + echo "ROCm artifact metadata is missing ${metadata_file}" >&2 + return 1 + fi + done + for metadata_file in ci-base-image.txt fallback-image.txt; do + if [[ ! -f "${metadata_dir}/${metadata_file}" ]]; then + echo "ROCm artifact metadata is missing ${metadata_file}" >&2 + return 1 + fi + done - tar -xzf "${archive}" -C "${wheel_dir}" || return 1 - if ! ls "${wheel_dir}"/*.whl >/dev/null 2>&1; then - echo "ROCm wheel artifact did not contain a wheel" + recorded_commit=$(tr -d '\r\n' < "${metadata_dir}/commit.txt") + recorded_base=$(tr -d '\r\n' < "${metadata_dir}/native-base-image.txt") + recorded_wheel=$(tr -d '\r\n' < "${metadata_dir}/wheel-filename.txt") + if [[ -z "${BUILDKITE_COMMIT:-}" || "${recorded_commit}" != "${BUILDKITE_COMMIT}" ]]; then + echo "ROCm artifact commit ${recorded_commit} does not match ${BUILDKITE_COMMIT:-unset}" >&2 return 1 fi - if [[ ! -d "${wheel_dir}/tests" ]]; then - echo "ROCm wheel artifact did not contain the test workspace" + if [[ -z "${VLLM_CI_BASE_IMAGE:-}" || "${recorded_base}" != "${VLLM_CI_BASE_IMAGE}" ]]; then + echo "ROCm artifact base ${recorded_base} does not match ${VLLM_CI_BASE_IMAGE:-unset}" >&2 + return 1 + fi + if [[ "${recorded_wheel}" != "$(basename "${wheels[0]}")" ]]; then + echo "ROCm artifact wheel manifest ${recorded_wheel} does not match $(basename "${wheels[0]}")" >&2 return 1 fi + for required_dir in tests .buildkite requirements; do + if [[ ! -d "${wheel_dir}/${required_dir}" ]]; then + echo "ROCm wheel artifact did not contain ${required_dir}/" >&2 + return 1 + fi + done echo "--- Installing ROCm wheel into pod environment" - python3 -m pip install --no-deps --force-reinstall "${wheel_dir}"/*.whl || return 1 - - echo "--- Preparing /vllm-workspace from artifact" - rm -rf /vllm-workspace || return 1 - mkdir -p /vllm-workspace || return 1 - tar -C "${wheel_dir}" --exclude='*.whl' -cf - . \ - | tar -C /vllm-workspace -xf - || return 1 + python3 -m pip install --no-deps --force-reinstall "${wheels[0]}" || return 1 + + echo "--- Preparing ${workspace_dir} from artifact" + find "${workspace_dir}" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + || return 1 + tar -C "${wheel_dir}" \ + --exclude='*.whl' \ + --exclude='.vllm-ci-artifact' \ + --exclude='./.vllm-ci-artifact' \ + -cf - . | tar --no-same-owner -C "${workspace_dir}" -xf - || return 1 + if [[ ! -d "${workspace_dir}/tests" ]]; then + echo "Failed to stage the native test workspace" >&2 + return 1 + fi return 0 } -run_native_preflight() { - : "${TMPDIR:=/tmp}" - : "${TORCHINDUCTOR_CACHE_DIR:=/tmp/vllm-native/torchinductor}" - : "${TRITON_CACHE_DIR:=/tmp/vllm-native/triton}" - : "${VLLM_CACHE_ROOT:=/home/buildkite-agent/huggingface/vllm-cache}" - : "${XDG_CACHE_HOME:=/tmp/vllm-native/xdg}" +initialize_native_environment() { + local job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-local}}" + local native_root="" + local hf_mount="" + + if [[ "$(id -u)" -ne 0 ]]; then + echo "Native ROCm CI currently requires the ci_base container to run as root" >&2 + return 1 + fi + + job_id="${job_id//[^A-Za-z0-9_.-]/_}" + native_root="/tmp/vllm-native-${job_id}" + : "${TMPDIR:=${native_root}/tmp}" + : "${TORCHINDUCTOR_CACHE_DIR:=${native_root}/cache/torchinductor}" + : "${TRITON_CACHE_DIR:=${native_root}/cache/triton}" + : "${VLLM_CACHE_ROOT:=${native_root}/cache/vllm}" + : "${XDG_CACHE_HOME:=${native_root}/cache/xdg}" + : "${HF_HOME:=/home/buildkite-agent/huggingface}" + : "${HF_HUB_DOWNLOAD_TIMEOUT:=300}" + : "${HF_HUB_ETAG_TIMEOUT:=60}" export TMPDIR TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT XDG_CACHE_HOME + export HF_HOME HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT + export PYTORCH_ROCM_ARCH="" mkdir -p "${TMPDIR}" \ "${TORCHINDUCTOR_CACHE_DIR}" \ "${TRITON_CACHE_DIR}" \ "${VLLM_CACHE_ROOT}" \ - "${XDG_CACHE_HOME}" || return 1 - python3 -c "import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing" + "${XDG_CACHE_HOME}" \ + "${HF_HOME}" || return 1 + + if [[ "${VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE:-0}" == "1" ]]; then + if ! command -v findmnt >/dev/null 2>&1; then + echo "findmnt is required to verify the native Hugging Face cache mount" >&2 + return 1 + fi + hf_mount=$(findmnt -n -T "${HF_HOME}" -o TARGET 2>/dev/null || true) + if [[ -z "${hf_mount}" || "${hf_mount}" == "/" ]]; then + echo "Native CI requires a persistent volume mounted at or above ${HF_HOME}" >&2 + return 1 + fi + fi +} + +run_native_preflight() { + local expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}" + + if [[ ! "${expected_gpus}" =~ ^[0-9]+$ ]]; then + echo "Invalid VLLM_CI_EXPECTED_GPU_COUNT=${expected_gpus}" >&2 + return 1 + fi + + python3 -c "import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing" || return 1 + + if [[ "${expected_gpus}" == "0" ]]; then + echo "Native CPU-only AMD job: skipping ROCm device validation" + return 0 + fi + + echo "--- ROCm info" + rocminfo || return 1 + VLLM_CI_EXPECTED_GPU_COUNT="${expected_gpus}" python3 - <<'PY' +import os + +import torch + +expected = int(os.environ["VLLM_CI_EXPECTED_GPU_COUNT"]) +assert torch.version.hip, "PyTorch is not a ROCm build" +assert torch.cuda.is_available(), "ROCm GPU is not available to PyTorch" +actual = torch.cuda.device_count() +assert actual == expected, f"Expected {expected} ROCm GPU(s), found {actual}" +PY } is_multi_node() { @@ -422,9 +600,9 @@ re_quote_pytest_markers() { # Main ############################################################################### -# --- GPU initialization --- -echo "--- ROCm info" -rocminfo +if [[ "${VLLM_CI_RUN_AMD_TEST_LIB_ONLY:-0}" == "1" ]]; then + return 0 2>/dev/null || exit 0 +fi if is_native_runtime; then echo "--- Native in-pod ROCm CI (AMD_CI_RUNTIME=${AMD_CI_RUNTIME:-unset}, NATIVE_CI=${NATIVE_CI:-unset})" @@ -437,11 +615,6 @@ if is_native_runtime; then } trap cleanup_native_workspace EXIT - if ! prepare_native_workspace; then - echo "Failed to prepare native test workspace" - exit 1 - fi - if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then commands="${VLLM_TEST_COMMANDS}" commands_source="env" @@ -463,17 +636,27 @@ if is_native_runtime; then exit 1 fi - export PYTHONPATH="/vllm-workspace" - : "${HF_HUB_DOWNLOAD_TIMEOUT:=300}" - : "${HF_HUB_ETAG_TIMEOUT:=60}" - export HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT + if ! initialize_native_environment; then + echo "Failed to initialize the native test environment" + exit 1 + fi + if ! prepare_native_workspace; then + echo "Failed to prepare native test workspace" + exit 1 + fi + + export PYTHONPATH="${VLLM_CI_WORKSPACE:-/vllm-workspace}" echo "Native test commands: $commands" run_native_preflight || exit 1 - /bin/bash -c "${commands}" + /bin/bash -o pipefail -c "${commands}" handle_pytest_exit "$?" fi +# --- GPU initialization for legacy Docker execution --- +echo "--- ROCm info" +rocminfo + # --- Docker status --- report_docker_usage diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index a8fc9fc2caf3..93f85dca67e4 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -476,6 +476,18 @@ steps: fast_check: true working_dir: "/vllm-workspace/tests" source_file_dependencies: + - .buildkite/test-amd.yaml + - .buildkite/hardware_tests/amd.yaml + - .buildkite/scripts/ci-bake-rocm.sh + - .buildkite/scripts/hardware_ci/run-amd-test.sh + - .buildkite/scripts/rocm/build-ci-base.sh + - .buildkite/scripts/rocm/build-test-image.sh + - docker/Dockerfile.rocm + - docker/Dockerfile.rocm_base + - docker/ci-rocm.hcl + - docker/docker-bake-rocm.hcl + - requirements/rocm.txt + - requirements/test/rocm.txt - vllm/ - tests/basic_correctness/test_basic_correctness - tests/basic_correctness/test_cpu_offload @@ -719,6 +731,7 @@ steps: source_file_dependencies: - vllm/distributed/eplb - tests/distributed/test_eplb_algo.py + - tests/distributed/test_eplb_utils.py - vllm/platforms/rocm.py commands: - pytest -v -s distributed/test_eplb_algo.py @@ -2563,6 +2576,7 @@ steps: - tests/entrypoints/openai/test_multi_api_servers.py - vllm/platforms/rocm.py commands: + - export NCCL_CUMEM_HOST_ENABLE=0 - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py @@ -2627,7 +2641,7 @@ steps: optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ - tests/v1/kv_connector/nixl_integration/ - vllm/platforms/rocm.py commands: @@ -2648,6 +2662,7 @@ steps: - tests/distributed/test_utils - vllm/platforms/rocm.py commands: + - export NCCL_CUMEM_HOST_ENABLE=0 - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py @@ -2697,7 +2712,7 @@ steps: optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ - tests/v1/kv_connector/nixl_integration/ - vllm/platforms/rocm.py commands: diff --git a/.buildkite/test_areas/expert_parallelism.yaml b/.buildkite/test_areas/expert_parallelism.yaml index 03ac500c844c..910c24dcdacc 100644 --- a/.buildkite/test_areas/expert_parallelism.yaml +++ b/.buildkite/test_areas/expert_parallelism.yaml @@ -21,6 +21,17 @@ steps: depends_on: - image-build-amd source_file_dependencies: + - .buildkite/hardware_tests/amd.yaml + - .buildkite/scripts/ci-bake-rocm.sh + - .buildkite/scripts/hardware_ci/run-amd-test.sh + - .buildkite/scripts/rocm/build-ci-base.sh + - .buildkite/scripts/rocm/build-test-image.sh + - docker/Dockerfile.rocm + - docker/Dockerfile.rocm_base + - docker/ci-rocm.hcl + - docker/docker-bake-rocm.hcl + - requirements/rocm.txt + - requirements/test/rocm.txt - vllm/distributed/eplb - tests/distributed/test_eplb_algo.py - tests/distributed/test_eplb_utils.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ad767c823bef..699f39cad1f8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -189,6 +189,13 @@ repos: entry: tools/pre_commit/shellcheck.sh language: script types: [shell] + - id: validate-amd-native-ci + name: Validate AMD native CI device policy + entry: python tools/pre_commit/validate_amd_native_ci.py + language: python + additional_dependencies: [pyyaml] + files: ^(\.buildkite/(hardware_tests/amd\.yaml|test-amd\.yaml|test_areas/.*\.yaml|scripts/(ci-bake-rocm\.sh|hardware_ci/run-amd-test\.sh|rocm/build-(ci-base|test-image)\.sh))|docker/(Dockerfile\.rocm(_base)?|ci-rocm\.hcl|docker-bake-rocm\.hcl)|requirements/(rocm|test/rocm)\.txt|tools/pre_commit/validate_amd_native_ci\.py)$ + pass_filenames: false - id: png-lint name: Lint PNG exports from excalidraw entry: tools/pre_commit/png-lint.sh diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index 93daf9d150d1..a7cc8c0870bc 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -674,6 +674,7 @@ RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \ libibverbs1 \ ibverbs-providers \ ibverbs-utils \ + util-linux \ unzip \ pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \ libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \ diff --git a/tests/tools/test_amd_native_ci_scripts.py b/tests/tools/test_amd_native_ci_scripts.py new file mode 100644 index 000000000000..5695706c16f8 --- /dev/null +++ b/tests/tools/test_amd_native_ci_scripts.py @@ -0,0 +1,427 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import hashlib +import os +import subprocess +import tarfile +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +RUNNER = REPO_ROOT / ".buildkite" / "scripts" / "hardware_ci" / "run-amd-test.sh" +CI_BAKE = REPO_ROOT / ".buildkite" / "scripts" / "ci-bake-rocm.sh" + +EXPECTED_COMMIT = "0123456789abcdef" +EXPECTED_BASE_IMAGE = "rocm/vllm-dev:ci_base-test" +WHEEL_NAME = "vllm-0.0.0-cp38-abi3-manylinux_2_28_x86_64.whl" + + +def _write_executable(path: Path, contents: str) -> None: + path.write_text(contents) + path.chmod(0o755) + + +def _create_artifact( + tmp_path: Path, + *, + recorded_commit: str = EXPECTED_COMMIT, + recorded_base: str = EXPECTED_BASE_IMAGE, + wheel_names: tuple[str, ...] = (WHEEL_NAME,), +) -> tuple[Path, Path]: + payload = tmp_path / "payload" + metadata = payload / ".vllm-ci-artifact" + metadata.mkdir(parents=True) + + (metadata / "commit.txt").write_text(f"{recorded_commit}\n") + (metadata / "native-base-image.txt").write_text(f"{recorded_base}\n") + (metadata / "wheel-filename.txt").write_text(f"{wheel_names[0]}\n") + (metadata / "ci-base-image.txt").write_text("ci-base\n") + (metadata / "fallback-image.txt").write_text("fallback\n") + + for required_dir in ("tests", ".buildkite", "requirements"): + directory = payload / required_dir + directory.mkdir() + (directory / "artifact-marker.txt").write_text(required_dir) + + for wheel_name in wheel_names: + (payload / wheel_name).write_bytes(b"not-a-real-wheel") + + source = tmp_path / "artifact-source" + source.mkdir() + archive = source / "vllm-rocm-install.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + for child in payload.iterdir(): + tar.add(child, arcname=child.name) + + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + checksum = source / "vllm-rocm-install.tar.gz.sha256" + checksum.write_text(f"{digest} {archive.name}\n") + return archive, checksum + + +def _create_fake_bin(tmp_path: Path) -> Path: + fake_bin = tmp_path / "fake-bin" + fake_bin.mkdir() + + _write_executable( + fake_bin / "buildkite-agent", + """#!/bin/bash +set -u +printf '%s\n' "$*" >> "${FAKE_BUILDKITE_LOG}" +if [[ "${1:-}" != "artifact" || "${2:-}" != "download" ]]; then + exit 2 +fi +query="${3}" +destination="${4}" +output_dir="${destination}/artifacts/vllm-rocm-install" +mkdir -p "${output_dir}" + +case "${query}" in + *.sha256) + attempts=0 + if [[ -f "${FAKE_CHECKSUM_STATE}" ]]; then + read -r attempts < "${FAKE_CHECKSUM_STATE}" + fi + attempts=$((attempts + 1)) + printf '%s\n' "${attempts}" > "${FAKE_CHECKSUM_STATE}" + if (( attempts <= FAKE_CHECKSUM_FAILURES )); then + exit 1 + fi + cp "${FAKE_CHECKSUM}" \ + "${output_dir}/vllm-rocm-install.tar.gz.sha256" + ;; + *) + cp "${FAKE_ARCHIVE}" "${output_dir}/vllm-rocm-install.tar.gz" + ;; +esac +""", + ) + _write_executable( + fake_bin / "python3", + """#!/bin/bash +printf '%s\n' "$*" >> "${FAKE_PYTHON_LOG}" +exit 0 +""", + ) + _write_executable( + fake_bin / "id", + """#!/bin/bash +if [[ "${1:-}" == "-u" ]]; then + printf '0\n' + exit 0 +fi +exec /usr/bin/id "$@" +""", + ) + _write_executable(fake_bin / "sleep", "#!/bin/bash\nexit 0\n") + return fake_bin + + +def _native_env( + tmp_path: Path, + archive: Path, + checksum: Path, + *, + checksum_failures: int = 0, + workspace: Path | None = None, + checkout: Path | None = None, +) -> dict[str, str]: + fake_bin = _create_fake_bin(tmp_path) + workspace = workspace or tmp_path / "workspace" + checkout = checkout or tmp_path / "checkout" + temp_dir = tmp_path / "tmp" + hf_home = tmp_path / "huggingface" + temp_dir.mkdir() + + env = os.environ.copy() + for name in ("AMD_CI_RUNTIME", "NATIVE_CI", "NUM_NODES", "VLLM_TEST_COMMANDS"): + env.pop(name, None) + env.update( + { + "PATH": f"{fake_bin}:{env['PATH']}", + "VLLM_CI_RUN_AMD_TEST_LIB_ONLY": "1", + "VLLM_CI_USE_ARTIFACTS": "1", + "VLLM_CI_ARTIFACT_GLOB": ( + "artifacts/vllm-rocm-install/vllm-rocm-install.tar.gz" + ), + "VLLM_CI_ARTIFACT_STEP": "image-build-amd", + "VLLM_CI_REQUIRE_WORKSPACE_MOUNT": "0", + "VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE": "0", + "VLLM_CI_WORKSPACE": str(workspace), + "VLLM_CI_BASE_IMAGE": EXPECTED_BASE_IMAGE, + "BUILDKITE_BUILD_CHECKOUT_PATH": str(checkout), + "BUILDKITE_COMMIT": EXPECTED_COMMIT, + "BUILDKITE_JOB_ID": "native-runner-test", + "TMPDIR": str(temp_dir), + "HF_HOME": str(hf_home), + "FAKE_ARCHIVE": str(archive), + "FAKE_CHECKSUM": str(checksum), + "FAKE_CHECKSUM_FAILURES": str(checksum_failures), + "FAKE_CHECKSUM_STATE": str(tmp_path / "checksum-attempts"), + "FAKE_BUILDKITE_LOG": str(tmp_path / "buildkite-agent.log"), + "FAKE_PYTHON_LOG": str(tmp_path / "python3.log"), + } + ) + return env + + +def _run_prepare(env: dict[str, str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + "bash", + "-c", + 'source "$1"\nartifact_work_dir=""\nprepare_native_workspace', + "run-amd-test", + str(RUNNER), + ], + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + timeout=30, + ) + + +def _producer_env(tmp_path: Path, *, upload_failure: bool = False) -> dict[str, str]: + fake_bin = tmp_path / "producer-bin" + fake_bin.mkdir() + _write_executable( + fake_bin / "buildkite-agent", + """#!/bin/bash +printf '%s\n' "$*" >> "${FAKE_BUILDKITE_LOG}" +if [[ "${FAKE_UPLOAD_FAILURE}" == "1" ]]; then + exit 1 +fi +exit 0 +""", + ) + env = os.environ.copy() + env.update( + { + "PATH": f"{fake_bin}:{env['PATH']}", + "VLLM_CI_BAKE_ROCM_LIB_ONLY": "1", + "UPLOAD_ROCM_WHEEL_ARTIFACTS": "1", + "BUILDKITE": "true", + "BUILDKITE_COMMIT": EXPECTED_COMMIT, + "CI_BASE_IMAGE_TAG_COMMIT_REF": EXPECTED_BASE_IMAGE, + "CI_BASE_IMAGE": "rocm/vllm-dev:ci_base-content", + "IMAGE_TAG": "rocm/vllm-ci:test", + "FAKE_UPLOAD_FAILURE": "1" if upload_failure else "0", + "FAKE_BUILDKITE_LOG": str(tmp_path / "producer-agent.log"), + } + ) + return env + + +def _run_producer( + tmp_path: Path, + *, + wheel_names: tuple[str, ...], + upload_failure: bool = False, +) -> subprocess.CompletedProcess[str]: + wheel_export = tmp_path / "wheel-export" + wheel_export.mkdir() + for directory in ("tests", ".buildkite", "requirements"): + (wheel_export / directory).mkdir() + for wheel_name in wheel_names: + (wheel_export / wheel_name).write_bytes(b"wheel") + return subprocess.run( + [ + "bash", + "-c", + 'source "$1"\nupload_wheel_artifacts_if_present', + "ci-bake-rocm", + str(CI_BAKE), + ], + cwd=tmp_path, + env=_producer_env(tmp_path, upload_failure=upload_failure), + capture_output=True, + text=True, + timeout=30, + ) + + +def test_rocm_artifact_producer_packages_manifest_and_checksum(tmp_path: Path) -> None: + result = _run_producer(tmp_path, wheel_names=(WHEEL_NAME,)) + + assert result.returncode == 0, result.stdout + result.stderr + artifact_dir = tmp_path / "artifacts" / "vllm-rocm-install" + archive = artifact_dir / "vllm-rocm-install.tar.gz" + checksum = artifact_dir / "vllm-rocm-install.tar.gz.sha256" + expected_digest = checksum.read_text().split()[0] + assert hashlib.sha256(archive.read_bytes()).hexdigest() == expected_digest + with tarfile.open(archive, "r:gz") as tar: + members = {name.removeprefix("./") for name in tar.getnames()} + assert WHEEL_NAME in members + assert ".vllm-ci-artifact/commit.txt" in members + assert ".vllm-ci-artifact/native-base-image.txt" in members + assert ".vllm-ci-artifact/wheel-filename.txt" in members + assert (artifact_dir / "commit.txt").read_text().strip() == EXPECTED_COMMIT + assert (artifact_dir / "native-base-image.txt").read_text().strip() == ( + EXPECTED_BASE_IMAGE + ) + assert (artifact_dir / "wheel-filename.txt").read_text().strip() == WHEEL_NAME + assert ( + "artifact upload artifacts/vllm-rocm-install/*" + in (tmp_path / "producer-agent.log").read_text() + ) + assert not (tmp_path / "wheel-export").exists() + + +@pytest.mark.parametrize("wheel_names", [(), (WHEEL_NAME, "second.whl")]) +def test_rocm_artifact_producer_rejects_wrong_wheel_count( + tmp_path: Path, + wheel_names: tuple[str, ...], +) -> None: + result = _run_producer(tmp_path, wheel_names=wheel_names) + + assert result.returncode != 0 + assert f"found {len(wheel_names)}" in result.stderr + assert not (tmp_path / "producer-agent.log").exists() + + +def test_rocm_artifact_producer_propagates_upload_failure(tmp_path: Path) -> None: + result = _run_producer( + tmp_path, + wheel_names=(WHEEL_NAME,), + upload_failure=True, + ) + + assert result.returncode != 0 + assert ( + "artifact upload artifacts/vllm-rocm-install/*" + in (tmp_path / "producer-agent.log").read_text() + ) + assert (tmp_path / "wheel-export").exists() + + +def test_native_artifact_retries_when_checksum_is_initially_missing( + tmp_path: Path, +) -> None: + archive, checksum = _create_artifact(tmp_path) + env = _native_env(tmp_path, archive, checksum, checksum_failures=1) + + result = _run_prepare(env) + + assert result.returncode == 0, result.stdout + result.stderr + assert (tmp_path / "checksum-attempts").read_text().strip() == "2" + assert (tmp_path / "workspace" / "tests" / "artifact-marker.txt").is_file() + agent_log = (tmp_path / "buildkite-agent.log").read_text() + assert agent_log.count("vllm-rocm-install.tar.gz.sha256") == 2 + assert "--step image-build-amd" in agent_log + + +@pytest.mark.parametrize( + ("recorded_commit", "recorded_base", "expected_message"), + [ + ( + "wrong-commit", + EXPECTED_BASE_IMAGE, + "ROCm artifact commit wrong-commit does not match " + EXPECTED_COMMIT, + ), + ( + EXPECTED_COMMIT, + "rocm/wrong-base:latest", + "ROCm artifact base rocm/wrong-base:latest does not match " + + EXPECTED_BASE_IMAGE, + ), + ], +) +def test_native_artifact_rejects_wrong_provenance( + tmp_path: Path, + recorded_commit: str, + recorded_base: str, + expected_message: str, +) -> None: + archive, checksum = _create_artifact( + tmp_path, + recorded_commit=recorded_commit, + recorded_base=recorded_base, + ) + env = _native_env(tmp_path, archive, checksum) + + result = _run_prepare(env) + + assert result.returncode != 0 + assert expected_message in result.stderr + + +def test_native_artifact_rejects_duplicate_top_level_wheels(tmp_path: Path) -> None: + archive, checksum = _create_artifact( + tmp_path, + wheel_names=(WHEEL_NAME, "second-wheel-0.0.0-py3-none-any.whl"), + ) + env = _native_env(tmp_path, archive, checksum) + + result = _run_prepare(env) + + assert result.returncode != 0 + assert "must contain exactly one top-level wheel; found 2" in result.stderr + + +@pytest.mark.parametrize( + "relationship", + ["same", "checkout-parent", "workspace-parent"], +) +def test_native_workspace_refuses_to_overlap_checkout( + tmp_path: Path, + relationship: str, +) -> None: + paths = tmp_path / "paths" + if relationship == "same": + workspace = checkout = paths / "workspace" + elif relationship == "checkout-parent": + checkout = paths / "checkout" + workspace = checkout / "workspace" + else: + workspace = paths / "workspace" + checkout = workspace / "checkout" + + workspace.mkdir(parents=True) + checkout.mkdir(parents=True, exist_ok=True) + sentinel = workspace / "keep-me" + sentinel.write_text("legitimate checkout content") + archive, checksum = _create_artifact(tmp_path) + env = _native_env( + tmp_path, + archive, + checksum, + workspace=workspace, + checkout=checkout, + ) + + result = _run_prepare(env) + + assert result.returncode != 0 + assert "Refusing to replace" in result.stderr + assert sentinel.read_text() == "legitimate checkout content" + assert not (tmp_path / "python3.log").exists() + assert not (tmp_path / "buildkite-agent.log").exists() + + +def test_native_child_shell_propagates_pipeline_failure(tmp_path: Path) -> None: + archive, checksum = _create_artifact(tmp_path) + env = _native_env(tmp_path, archive, checksum) + env.update( + { + "AMD_CI_RUNTIME": "native", + "VLLM_CI_RUN_AMD_TEST_LIB_ONLY": "0", + "VLLM_CI_EXPECTED_GPU_COUNT": "0", + "VLLM_TEST_COMMANDS": "false | true", + } + ) + + result = subprocess.run( + ["bash", str(RUNNER)], + cwd=REPO_ROOT, + env=env, + capture_output=True, + text=True, + timeout=30, + ) + + assert result.returncode == 1, result.stdout + result.stderr + assert "Native test commands: false | true" in result.stdout + assert "Native CPU-only AMD job" in result.stdout diff --git a/tests/tools/test_validate_amd_native_ci.py b/tests/tools/test_validate_amd_native_ci.py new file mode 100644 index 000000000000..037dc0b63a36 --- /dev/null +++ b/tests/tools/test_validate_amd_native_ci.py @@ -0,0 +1,238 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from tools.pre_commit import validate_amd_native_ci as validator + + +@pytest.fixture +def config_paths(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + test_amd = tmp_path / "test-amd.yaml" + test_areas = tmp_path / "test_areas" + test_areas.mkdir() + monkeypatch.setattr(validator, "TEST_AMD", test_amd) + monkeypatch.setattr(validator, "TEST_AREAS", test_areas) + return test_amd, test_areas + + +def _write_yaml(path: Path, steps: list[dict[str, Any]]) -> None: + path.write_text(yaml.safe_dump({"steps": steps}, sort_keys=False)) + + +def _legacy_step( + device: str, + *, + native_ci: Any = None, + **overrides: Any, +) -> dict[str, Any]: + step: dict[str, Any] = { + "label": f"Legacy {device}", + "mirror_hardwares": ["amdproduction"], + "agent_pool": device, + } + if native_ci is not None: + step["native_ci"] = native_ci + step.update(overrides) + return step + + +def test_legacy_accepts_native_mi300_and_legacy_families(config_paths) -> None: + test_amd, _ = config_paths + steps = [ + _legacy_step( + f"mi300_{gpu_count}", + native_ci=True, + num_gpus=gpu_count, + ) + for gpu_count in (1, 2, 4, 8) + ] + steps.extend(_legacy_step(f"{family}_1") for family in ("mi250", "mi325", "mi355")) + _write_yaml(test_amd, steps) + + errors, count = validator.validate_legacy_config() + + assert errors == [] + assert count == 7 + + +@pytest.mark.parametrize( + ("step", "message"), + [ + (_legacy_step("mi300_1"), "mi300_1 must use native"), + ( + _legacy_step("mi325_1", native_ci=True), + "mi325_1 must use legacy DinD", + ), + ( + _legacy_step("mi300_1", native_ci="true"), + "native_ci must be a boolean", + ), + ], +) +def test_legacy_rejects_missing_or_wrong_native_flag( + config_paths, + step: dict[str, Any], + message: str, +) -> None: + test_amd, _ = config_paths + _write_yaml(test_amd, [step]) + + errors, _ = validator.validate_legacy_config() + + assert any(message in error for error in errors) + + +@pytest.mark.parametrize( + "step", + [ + _legacy_step("mi300_4", native_ci=True), + _legacy_step("mi300_4", native_ci=True, num_gpus=2), + ], +) +def test_legacy_rejects_gpu_count_that_does_not_match_pool_suffix( + config_paths, + step: dict[str, Any], +) -> None: + test_amd, _ = config_paths + _write_yaml(test_amd, [step]) + + errors, _ = validator.validate_legacy_config() + + assert any("does not match mi300_4 (4)" in error for error in errors) + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"num_nodes": 2}, "cannot be multi-node"), + ({"no_plugin": True}, "cannot use no_plugin"), + ], +) +def test_legacy_rejects_unsupported_native_modes( + config_paths, + overrides: dict[str, Any], + message: str, +) -> None: + test_amd, _ = config_paths + step = _legacy_step("mi300_1", native_ci=True, **overrides) + _write_yaml(test_amd, [step]) + + errors, _ = validator.validate_legacy_config() + + assert any(message in error for error in errors) + + +def test_test_areas_accept_direct_and_nested_amd_devices(config_paths) -> None: + _, test_areas = config_paths + _write_yaml( + test_areas / "valid.yaml", + [ + { + "label": "Direct MI300", + "device": "mi300_2", + "native_ci": True, + "num_devices": 2, + }, + { + "label": "Direct legacy MI355", + "device": "mi355_1", + }, + { + "label": "Label says H100-MI300 but mirror is MI325", + "device": "h100", + "mirror": {"amd": {"device": "mi325_1"}}, + }, + { + "label": "Nested MI300 mirror", + "device": "h100", + "num_devices": 4, + "mirror": { + "amd": { + "device": "mi300_4", + "native_ci": True, + } + }, + }, + ], + ) + + errors, count = validator.validate_test_areas() + + assert errors == [] + assert count == 4 + + +def test_test_areas_reject_native_flag_on_non_amd_direct_step(config_paths) -> None: + _, test_areas = config_paths + _write_yaml( + test_areas / "invalid.yaml", + [{"label": "Not actually AMD", "device": "h100", "native_ci": True}], + ) + + errors, count = validator.validate_test_areas() + + assert count == 0 + assert any("native_ci is set on a non-AMD direct step" in error for error in errors) + + +@pytest.mark.parametrize( + ("amd", "message"), + [ + ({"device": "mi300_1"}, "mi300_1 must use native"), + ( + {"device": "mi325_1", "native_ci": True}, + "mi325_1 must use legacy DinD", + ), + ( + {"device": "mi300_1", "native_ci": True, "num_nodes": 2}, + "cannot be multi-node", + ), + ( + {"device": "mi300_1", "native_ci": True, "no_plugin": True}, + "cannot use no_plugin", + ), + ], +) +def test_test_areas_reject_invalid_nested_amd_policy( + config_paths, + amd: dict[str, Any], + message: str, +) -> None: + _, test_areas = config_paths + _write_yaml( + test_areas / "invalid.yaml", + [{"label": "Nested mirror", "device": "h100", "mirror": {"amd": amd}}], + ) + + errors, _ = validator.validate_test_areas() + + assert any(message in error for error in errors) + + +def test_mirror_gpu_count_inherits_top_level_count_for_validation(config_paths) -> None: + _, test_areas = config_paths + _write_yaml( + test_areas / "invalid.yaml", + [ + { + "label": "Mismatched mirror GPU count", + "device": "h100", + "num_devices": 8, + "mirror": { + "amd": { + "device": "mi300_4", + "native_ci": True, + } + }, + } + ], + ) + + errors, _ = validator.validate_test_areas() + + assert any("GPU count 8 does not match mi300_4 (4)" in error for error in errors) diff --git a/tools/pre_commit/validate_amd_native_ci.py b/tools/pre_commit/validate_amd_native_ci.py new file mode 100644 index 000000000000..b330be11a3c6 --- /dev/null +++ b/tools/pre_commit/validate_amd_native_ci.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from pathlib import Path +from typing import Any + +import yaml + +AMD_MODELS = {"250", "300", "325", "355"} +AMD_GPU_COUNTS = {"1", "2", "4", "8"} +TEST_AMD = Path(".buildkite/test-amd.yaml") +TEST_AREAS = Path(".buildkite/test_areas") + + +def _parse_amd_device(device: Any) -> tuple[str, int] | None: + model, separator, gpu_count = str(device or "").rpartition("_") + if ( + separator + and model.startswith("mi") + and model.removeprefix("mi") in AMD_MODELS + and gpu_count in AMD_GPU_COUNTS + ): + return model.removeprefix("mi"), int(gpu_count) + return None + + +def _first_configured(*values: Any) -> Any: + return next((value for value in values if value is not None), None) + + +def _validate_runtime( + *, + label: str, + device: Any, + native: Any, + num_gpus: Any, + num_nodes: Any, + no_plugin: Any, + require_explicit_gpu_count: bool, +) -> list[str]: + errors: list[str] = [] + parsed_device = _parse_amd_device(device) + if parsed_device is None: + return [f"{label}: invalid AMD device or agent pool {device!r}"] + + model, expected_gpus = parsed_device + expected_native = model == "300" + if not isinstance(native, bool): + errors.append(f"{label}: native_ci must be a boolean") + elif native != expected_native: + expected = "native" if expected_native else "legacy DinD" + errors.append(f"{label}: {device} must use {expected}") + + if native and (num_nodes or 1) > 1: + errors.append(f"{label}: native AMD jobs cannot be multi-node") + if native and no_plugin: + errors.append(f"{label}: native AMD jobs cannot use no_plugin") + + configured_gpus = 1 if require_explicit_gpu_count and num_gpus is None else num_gpus + if configured_gpus is not None and configured_gpus != expected_gpus: + errors.append( + f"{label}: GPU count {configured_gpus} does not match " + f"{device} ({expected_gpus})" + ) + + return errors + + +def _load(path: Path) -> dict[str, Any]: + data = yaml.safe_load(path.read_text()) + if not isinstance(data, dict): + raise ValueError(f"{path} must contain a YAML mapping") + return data + + +def validate_legacy_config() -> tuple[list[str], int]: + errors: list[str] = [] + validated = 0 + for index, step in enumerate(_load(TEST_AMD).get("steps", []), start=1): + if not isinstance(step, dict) or "mirror_hardwares" not in step: + continue + label = f"{TEST_AMD}:{step.get('label', f'step {index}')}" + errors.extend( + _validate_runtime( + label=label, + device=step.get("agent_pool"), + native=step.get("native_ci", False), + num_gpus=step.get("num_gpus"), + num_nodes=step.get("num_nodes"), + no_plugin=step.get("no_plugin", False), + require_explicit_gpu_count=True, + ) + ) + validated += 1 + return errors, validated + + +def validate_test_areas() -> tuple[list[str], int]: + errors: list[str] = [] + validated = 0 + for path in sorted(TEST_AREAS.glob("*.yaml")): + for index, step in enumerate(_load(path).get("steps", []), start=1): + if not isinstance(step, dict): + continue + label = f"{path}:{step.get('label', f'step {index}')}" + direct_device = step.get("device") + direct_is_amd = _parse_amd_device(direct_device) is not None + if direct_is_amd: + errors.extend( + _validate_runtime( + label=label, + device=direct_device, + native=step.get("native_ci", False), + num_gpus=_first_configured( + step.get("num_devices"), step.get("num_gpus") + ), + num_nodes=step.get("num_nodes"), + no_plugin=step.get("no_plugin", False), + require_explicit_gpu_count=False, + ) + ) + validated += 1 + elif "native_ci" in step: + errors.append(f"{label}: native_ci is set on a non-AMD direct step") + + mirror = step.get("mirror") + amd = mirror.get("amd") if isinstance(mirror, dict) else None + if not isinstance(amd, dict): + continue + errors.extend( + _validate_runtime( + label=f"{label} [AMD mirror]", + device=amd.get("device"), + native=amd.get("native_ci", False), + num_gpus=_first_configured( + amd.get("num_devices"), + amd.get("num_gpus"), + step.get("num_devices"), + step.get("num_gpus"), + ), + num_nodes=amd.get("num_nodes", step.get("num_nodes")), + no_plugin=amd.get("no_plugin", step.get("no_plugin", False)), + require_explicit_gpu_count=False, + ) + ) + validated += 1 + return errors, validated + + +def main() -> int: + legacy_errors, legacy_count = validate_legacy_config() + area_errors, area_count = validate_test_areas() + errors = legacy_errors + area_errors + if errors: + raise SystemExit( + "Invalid AMD native CI configuration:\n- " + "\n- ".join(errors) + ) + + print( + "AMD native CI policy validation passed " + f"({legacy_count} legacy steps, {area_count} test-area steps)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 85ad5d8f3bd2cc71f02ccf4f54b02f902643ab26 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sun, 12 Jul 2026 01:27:01 -0500 Subject: [PATCH 04/16] [CI][AMD] Generalize native runtime validation Signed-off-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 4 +- .buildkite/test_areas/misc.yaml | 4 +- .pre-commit-config.yaml | 2 +- tests/tools/test_validate_amd_native_ci.py | 80 +++++++++------------- tools/pre_commit/validate_amd_native_ci.py | 29 ++++---- 5 files changed, 51 insertions(+), 68 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 93f85dca67e4..43b6a13de2c6 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -14,8 +14,8 @@ # command(str): the single command to run for tests. incompatible with commands. # commands(list): the list of commands to run for the test. incompatible with command. # mirror_hardwares(list): the list of hardware to run the test on as well. currently only supports [amdexperimental] -# native_ci(bool): run MI300 jobs directly in the AMD Kubernetes pod instead of Docker-in-Docker. -# Omit this for MI250, MI325, and MI355 jobs, which still use the legacy Docker path. +# native_ci(bool): when true, run the job directly in the AMD Kubernetes pod. +# When false or omitted, use the legacy Docker-in-Docker path. # num_gpus(int): override the number of GPUs for the test. defaults to 1 GPU. currently supports 2,4,8. # num_nodes(int): whether to simulate multi-node setup by launching multiple containers on one host, # in this case, commands must be specified. the first command runs on the first host, the second diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 9dd83a040982..4fe82790a3ce 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -282,8 +282,8 @@ steps: commands: - bash standalone_tests/python_only_compile.sh mirror: - # This MI325 mirror intentionally keeps the legacy Docker path because the - # test requires the full source tree rather than the thin native artifact. + # This mirror intentionally leaves native_ci disabled because the test + # requires the full source tree rather than the thin native artifact. amd: device: mi325_1 timeout_in_minutes: 45 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 699f39cad1f8..f28bd577b8a5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -190,7 +190,7 @@ repos: language: script types: [shell] - id: validate-amd-native-ci - name: Validate AMD native CI device policy + name: Validate AMD native CI configuration entry: python tools/pre_commit/validate_amd_native_ci.py language: python additional_dependencies: [pyyaml] diff --git a/tests/tools/test_validate_amd_native_ci.py b/tests/tools/test_validate_amd_native_ci.py index 037dc0b63a36..154fdac7d599 100644 --- a/tests/tools/test_validate_amd_native_ci.py +++ b/tests/tools/test_validate_amd_native_ci.py @@ -41,57 +41,49 @@ def _legacy_step( return step -def test_legacy_accepts_native_mi300_and_legacy_families(config_paths) -> None: +def test_legacy_accepts_explicit_modes_for_current_and_future_devices( + config_paths, +) -> None: test_amd, _ = config_paths steps = [ - _legacy_step( - f"mi300_{gpu_count}", - native_ci=True, - num_gpus=gpu_count, - ) - for gpu_count in (1, 2, 4, 8) + _legacy_step("mi300_1", native_ci=True), + _legacy_step("mi300_1", native_ci=False), + _legacy_step("mi325_1", native_ci=True), + _legacy_step("mi325_1"), + _legacy_step("mi400_16", native_ci=True, num_gpus=16), ] - steps.extend(_legacy_step(f"{family}_1") for family in ("mi250", "mi325", "mi355")) _write_yaml(test_amd, steps) - errors, count = validator.validate_legacy_config() + errors, count = validator.validate_test_amd_config() assert errors == [] - assert count == 7 + assert count == 5 @pytest.mark.parametrize( - ("step", "message"), + "step", [ - (_legacy_step("mi300_1"), "mi300_1 must use native"), - ( - _legacy_step("mi325_1", native_ci=True), - "mi325_1 must use legacy DinD", - ), - ( - _legacy_step("mi300_1", native_ci="true"), - "native_ci must be a boolean", - ), + _legacy_step("mi300_1", native_ci="true"), + _legacy_step("mi400_16", native_ci=1, num_gpus=16), ], ) -def test_legacy_rejects_missing_or_wrong_native_flag( +def test_legacy_rejects_non_boolean_native_flag( config_paths, step: dict[str, Any], - message: str, ) -> None: test_amd, _ = config_paths _write_yaml(test_amd, [step]) - errors, _ = validator.validate_legacy_config() + errors, _ = validator.validate_test_amd_config() - assert any(message in error for error in errors) + assert any("native_ci must be a boolean" in error for error in errors) @pytest.mark.parametrize( "step", [ - _legacy_step("mi300_4", native_ci=True), - _legacy_step("mi300_4", native_ci=True, num_gpus=2), + _legacy_step("mi400_16", native_ci=True), + _legacy_step("mi400_16", native_ci=True, num_gpus=8), ], ) def test_legacy_rejects_gpu_count_that_does_not_match_pool_suffix( @@ -101,9 +93,9 @@ def test_legacy_rejects_gpu_count_that_does_not_match_pool_suffix( test_amd, _ = config_paths _write_yaml(test_amd, [step]) - errors, _ = validator.validate_legacy_config() + errors, _ = validator.validate_test_amd_config() - assert any("does not match mi300_4 (4)" in error for error in errors) + assert any("does not match mi400_16 (16)" in error for error in errors) @pytest.mark.parametrize( @@ -119,10 +111,10 @@ def test_legacy_rejects_unsupported_native_modes( message: str, ) -> None: test_amd, _ = config_paths - step = _legacy_step("mi300_1", native_ci=True, **overrides) + step = _legacy_step("mi400_16", native_ci=True, num_gpus=16, **overrides) _write_yaml(test_amd, [step]) - errors, _ = validator.validate_legacy_config() + errors, _ = validator.validate_test_amd_config() assert any(message in error for error in errors) @@ -133,27 +125,27 @@ def test_test_areas_accept_direct_and_nested_amd_devices(config_paths) -> None: test_areas / "valid.yaml", [ { - "label": "Direct MI300", - "device": "mi300_2", + "label": "Direct native future device", + "device": "mi400_2", "native_ci": True, "num_devices": 2, }, { - "label": "Direct legacy MI355", - "device": "mi355_1", + "label": "Direct legacy current device", + "device": "mi300_1", }, { - "label": "Label says H100-MI300 but mirror is MI325", + "label": "Label does not determine runtime mode", "device": "h100", - "mirror": {"amd": {"device": "mi325_1"}}, + "mirror": {"amd": {"device": "mi300_1"}}, }, { - "label": "Nested MI300 mirror", + "label": "Nested native mirror", "device": "h100", - "num_devices": 4, + "num_devices": 16, "mirror": { "amd": { - "device": "mi300_4", + "device": "mi400_16", "native_ci": True, } }, @@ -183,17 +175,13 @@ def test_test_areas_reject_native_flag_on_non_amd_direct_step(config_paths) -> N @pytest.mark.parametrize( ("amd", "message"), [ - ({"device": "mi300_1"}, "mi300_1 must use native"), - ( - {"device": "mi325_1", "native_ci": True}, - "mi325_1 must use legacy DinD", - ), + ({"device": "mi400_1", "native_ci": "true"}, "must be a boolean"), ( - {"device": "mi300_1", "native_ci": True, "num_nodes": 2}, + {"device": "mi400_1", "native_ci": True, "num_nodes": 2}, "cannot be multi-node", ), ( - {"device": "mi300_1", "native_ci": True, "no_plugin": True}, + {"device": "mi400_1", "native_ci": True, "no_plugin": True}, "cannot use no_plugin", ), ], diff --git a/tools/pre_commit/validate_amd_native_ci.py b/tools/pre_commit/validate_amd_native_ci.py index b330be11a3c6..ca3bbdd79e19 100644 --- a/tools/pre_commit/validate_amd_native_ci.py +++ b/tools/pre_commit/validate_amd_native_ci.py @@ -7,8 +7,6 @@ import yaml -AMD_MODELS = {"250", "300", "325", "355"} -AMD_GPU_COUNTS = {"1", "2", "4", "8"} TEST_AMD = Path(".buildkite/test-amd.yaml") TEST_AREAS = Path(".buildkite/test_areas") @@ -18,10 +16,11 @@ def _parse_amd_device(device: Any) -> tuple[str, int] | None: if ( separator and model.startswith("mi") - and model.removeprefix("mi") in AMD_MODELS - and gpu_count in AMD_GPU_COUNTS + and model.removeprefix("mi").isdigit() + and gpu_count.isdigit() + and int(gpu_count) > 0 ): - return model.removeprefix("mi"), int(gpu_count) + return model, int(gpu_count) return None @@ -44,13 +43,9 @@ def _validate_runtime( if parsed_device is None: return [f"{label}: invalid AMD device or agent pool {device!r}"] - model, expected_gpus = parsed_device - expected_native = model == "300" + _, expected_gpus = parsed_device if not isinstance(native, bool): errors.append(f"{label}: native_ci must be a boolean") - elif native != expected_native: - expected = "native" if expected_native else "legacy DinD" - errors.append(f"{label}: {device} must use {expected}") if native and (num_nodes or 1) > 1: errors.append(f"{label}: native AMD jobs cannot be multi-node") @@ -74,7 +69,7 @@ def _load(path: Path) -> dict[str, Any]: return data -def validate_legacy_config() -> tuple[list[str], int]: +def validate_test_amd_config() -> tuple[list[str], int]: errors: list[str] = [] validated = 0 for index, step in enumerate(_load(TEST_AMD).get("steps", []), start=1): @@ -117,7 +112,7 @@ def validate_test_areas() -> tuple[list[str], int]: ), num_nodes=step.get("num_nodes"), no_plugin=step.get("no_plugin", False), - require_explicit_gpu_count=False, + require_explicit_gpu_count=True, ) ) validated += 1 @@ -141,7 +136,7 @@ def validate_test_areas() -> tuple[list[str], int]: ), num_nodes=amd.get("num_nodes", step.get("num_nodes")), no_plugin=amd.get("no_plugin", step.get("no_plugin", False)), - require_explicit_gpu_count=False, + require_explicit_gpu_count=True, ) ) validated += 1 @@ -149,17 +144,17 @@ def validate_test_areas() -> tuple[list[str], int]: def main() -> int: - legacy_errors, legacy_count = validate_legacy_config() + test_amd_errors, test_amd_count = validate_test_amd_config() area_errors, area_count = validate_test_areas() - errors = legacy_errors + area_errors + errors = test_amd_errors + area_errors if errors: raise SystemExit( "Invalid AMD native CI configuration:\n- " + "\n- ".join(errors) ) print( - "AMD native CI policy validation passed " - f"({legacy_count} legacy steps, {area_count} test-area steps)" + "AMD native CI configuration validation passed " + f"({test_amd_count} test-amd steps, {area_count} test-area steps)" ) return 0 From a0b67814cf434d78c2ebb47397bac7b3eeadebe1 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sun, 12 Jul 2026 01:43:23 -0500 Subject: [PATCH 05/16] [CI][AMD] Remove auxiliary native CI validation Signed-off-by: Andreas Karatzas --- .pre-commit-config.yaml | 7 - tests/tools/test_amd_native_ci_scripts.py | 427 --------------------- tests/tools/test_validate_amd_native_ci.py | 226 ----------- tools/pre_commit/validate_amd_native_ci.py | 163 -------- 4 files changed, 823 deletions(-) delete mode 100644 tests/tools/test_amd_native_ci_scripts.py delete mode 100644 tests/tools/test_validate_amd_native_ci.py delete mode 100644 tools/pre_commit/validate_amd_native_ci.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f28bd577b8a5..ad767c823bef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -189,13 +189,6 @@ repos: entry: tools/pre_commit/shellcheck.sh language: script types: [shell] - - id: validate-amd-native-ci - name: Validate AMD native CI configuration - entry: python tools/pre_commit/validate_amd_native_ci.py - language: python - additional_dependencies: [pyyaml] - files: ^(\.buildkite/(hardware_tests/amd\.yaml|test-amd\.yaml|test_areas/.*\.yaml|scripts/(ci-bake-rocm\.sh|hardware_ci/run-amd-test\.sh|rocm/build-(ci-base|test-image)\.sh))|docker/(Dockerfile\.rocm(_base)?|ci-rocm\.hcl|docker-bake-rocm\.hcl)|requirements/(rocm|test/rocm)\.txt|tools/pre_commit/validate_amd_native_ci\.py)$ - pass_filenames: false - id: png-lint name: Lint PNG exports from excalidraw entry: tools/pre_commit/png-lint.sh diff --git a/tests/tools/test_amd_native_ci_scripts.py b/tests/tools/test_amd_native_ci_scripts.py deleted file mode 100644 index 5695706c16f8..000000000000 --- a/tests/tools/test_amd_native_ci_scripts.py +++ /dev/null @@ -1,427 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import hashlib -import os -import subprocess -import tarfile -from pathlib import Path - -import pytest - -REPO_ROOT = Path(__file__).resolve().parents[2] -RUNNER = REPO_ROOT / ".buildkite" / "scripts" / "hardware_ci" / "run-amd-test.sh" -CI_BAKE = REPO_ROOT / ".buildkite" / "scripts" / "ci-bake-rocm.sh" - -EXPECTED_COMMIT = "0123456789abcdef" -EXPECTED_BASE_IMAGE = "rocm/vllm-dev:ci_base-test" -WHEEL_NAME = "vllm-0.0.0-cp38-abi3-manylinux_2_28_x86_64.whl" - - -def _write_executable(path: Path, contents: str) -> None: - path.write_text(contents) - path.chmod(0o755) - - -def _create_artifact( - tmp_path: Path, - *, - recorded_commit: str = EXPECTED_COMMIT, - recorded_base: str = EXPECTED_BASE_IMAGE, - wheel_names: tuple[str, ...] = (WHEEL_NAME,), -) -> tuple[Path, Path]: - payload = tmp_path / "payload" - metadata = payload / ".vllm-ci-artifact" - metadata.mkdir(parents=True) - - (metadata / "commit.txt").write_text(f"{recorded_commit}\n") - (metadata / "native-base-image.txt").write_text(f"{recorded_base}\n") - (metadata / "wheel-filename.txt").write_text(f"{wheel_names[0]}\n") - (metadata / "ci-base-image.txt").write_text("ci-base\n") - (metadata / "fallback-image.txt").write_text("fallback\n") - - for required_dir in ("tests", ".buildkite", "requirements"): - directory = payload / required_dir - directory.mkdir() - (directory / "artifact-marker.txt").write_text(required_dir) - - for wheel_name in wheel_names: - (payload / wheel_name).write_bytes(b"not-a-real-wheel") - - source = tmp_path / "artifact-source" - source.mkdir() - archive = source / "vllm-rocm-install.tar.gz" - with tarfile.open(archive, "w:gz") as tar: - for child in payload.iterdir(): - tar.add(child, arcname=child.name) - - digest = hashlib.sha256(archive.read_bytes()).hexdigest() - checksum = source / "vllm-rocm-install.tar.gz.sha256" - checksum.write_text(f"{digest} {archive.name}\n") - return archive, checksum - - -def _create_fake_bin(tmp_path: Path) -> Path: - fake_bin = tmp_path / "fake-bin" - fake_bin.mkdir() - - _write_executable( - fake_bin / "buildkite-agent", - """#!/bin/bash -set -u -printf '%s\n' "$*" >> "${FAKE_BUILDKITE_LOG}" -if [[ "${1:-}" != "artifact" || "${2:-}" != "download" ]]; then - exit 2 -fi -query="${3}" -destination="${4}" -output_dir="${destination}/artifacts/vllm-rocm-install" -mkdir -p "${output_dir}" - -case "${query}" in - *.sha256) - attempts=0 - if [[ -f "${FAKE_CHECKSUM_STATE}" ]]; then - read -r attempts < "${FAKE_CHECKSUM_STATE}" - fi - attempts=$((attempts + 1)) - printf '%s\n' "${attempts}" > "${FAKE_CHECKSUM_STATE}" - if (( attempts <= FAKE_CHECKSUM_FAILURES )); then - exit 1 - fi - cp "${FAKE_CHECKSUM}" \ - "${output_dir}/vllm-rocm-install.tar.gz.sha256" - ;; - *) - cp "${FAKE_ARCHIVE}" "${output_dir}/vllm-rocm-install.tar.gz" - ;; -esac -""", - ) - _write_executable( - fake_bin / "python3", - """#!/bin/bash -printf '%s\n' "$*" >> "${FAKE_PYTHON_LOG}" -exit 0 -""", - ) - _write_executable( - fake_bin / "id", - """#!/bin/bash -if [[ "${1:-}" == "-u" ]]; then - printf '0\n' - exit 0 -fi -exec /usr/bin/id "$@" -""", - ) - _write_executable(fake_bin / "sleep", "#!/bin/bash\nexit 0\n") - return fake_bin - - -def _native_env( - tmp_path: Path, - archive: Path, - checksum: Path, - *, - checksum_failures: int = 0, - workspace: Path | None = None, - checkout: Path | None = None, -) -> dict[str, str]: - fake_bin = _create_fake_bin(tmp_path) - workspace = workspace or tmp_path / "workspace" - checkout = checkout or tmp_path / "checkout" - temp_dir = tmp_path / "tmp" - hf_home = tmp_path / "huggingface" - temp_dir.mkdir() - - env = os.environ.copy() - for name in ("AMD_CI_RUNTIME", "NATIVE_CI", "NUM_NODES", "VLLM_TEST_COMMANDS"): - env.pop(name, None) - env.update( - { - "PATH": f"{fake_bin}:{env['PATH']}", - "VLLM_CI_RUN_AMD_TEST_LIB_ONLY": "1", - "VLLM_CI_USE_ARTIFACTS": "1", - "VLLM_CI_ARTIFACT_GLOB": ( - "artifacts/vllm-rocm-install/vllm-rocm-install.tar.gz" - ), - "VLLM_CI_ARTIFACT_STEP": "image-build-amd", - "VLLM_CI_REQUIRE_WORKSPACE_MOUNT": "0", - "VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE": "0", - "VLLM_CI_WORKSPACE": str(workspace), - "VLLM_CI_BASE_IMAGE": EXPECTED_BASE_IMAGE, - "BUILDKITE_BUILD_CHECKOUT_PATH": str(checkout), - "BUILDKITE_COMMIT": EXPECTED_COMMIT, - "BUILDKITE_JOB_ID": "native-runner-test", - "TMPDIR": str(temp_dir), - "HF_HOME": str(hf_home), - "FAKE_ARCHIVE": str(archive), - "FAKE_CHECKSUM": str(checksum), - "FAKE_CHECKSUM_FAILURES": str(checksum_failures), - "FAKE_CHECKSUM_STATE": str(tmp_path / "checksum-attempts"), - "FAKE_BUILDKITE_LOG": str(tmp_path / "buildkite-agent.log"), - "FAKE_PYTHON_LOG": str(tmp_path / "python3.log"), - } - ) - return env - - -def _run_prepare(env: dict[str, str]) -> subprocess.CompletedProcess[str]: - return subprocess.run( - [ - "bash", - "-c", - 'source "$1"\nartifact_work_dir=""\nprepare_native_workspace', - "run-amd-test", - str(RUNNER), - ], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - timeout=30, - ) - - -def _producer_env(tmp_path: Path, *, upload_failure: bool = False) -> dict[str, str]: - fake_bin = tmp_path / "producer-bin" - fake_bin.mkdir() - _write_executable( - fake_bin / "buildkite-agent", - """#!/bin/bash -printf '%s\n' "$*" >> "${FAKE_BUILDKITE_LOG}" -if [[ "${FAKE_UPLOAD_FAILURE}" == "1" ]]; then - exit 1 -fi -exit 0 -""", - ) - env = os.environ.copy() - env.update( - { - "PATH": f"{fake_bin}:{env['PATH']}", - "VLLM_CI_BAKE_ROCM_LIB_ONLY": "1", - "UPLOAD_ROCM_WHEEL_ARTIFACTS": "1", - "BUILDKITE": "true", - "BUILDKITE_COMMIT": EXPECTED_COMMIT, - "CI_BASE_IMAGE_TAG_COMMIT_REF": EXPECTED_BASE_IMAGE, - "CI_BASE_IMAGE": "rocm/vllm-dev:ci_base-content", - "IMAGE_TAG": "rocm/vllm-ci:test", - "FAKE_UPLOAD_FAILURE": "1" if upload_failure else "0", - "FAKE_BUILDKITE_LOG": str(tmp_path / "producer-agent.log"), - } - ) - return env - - -def _run_producer( - tmp_path: Path, - *, - wheel_names: tuple[str, ...], - upload_failure: bool = False, -) -> subprocess.CompletedProcess[str]: - wheel_export = tmp_path / "wheel-export" - wheel_export.mkdir() - for directory in ("tests", ".buildkite", "requirements"): - (wheel_export / directory).mkdir() - for wheel_name in wheel_names: - (wheel_export / wheel_name).write_bytes(b"wheel") - return subprocess.run( - [ - "bash", - "-c", - 'source "$1"\nupload_wheel_artifacts_if_present', - "ci-bake-rocm", - str(CI_BAKE), - ], - cwd=tmp_path, - env=_producer_env(tmp_path, upload_failure=upload_failure), - capture_output=True, - text=True, - timeout=30, - ) - - -def test_rocm_artifact_producer_packages_manifest_and_checksum(tmp_path: Path) -> None: - result = _run_producer(tmp_path, wheel_names=(WHEEL_NAME,)) - - assert result.returncode == 0, result.stdout + result.stderr - artifact_dir = tmp_path / "artifacts" / "vllm-rocm-install" - archive = artifact_dir / "vllm-rocm-install.tar.gz" - checksum = artifact_dir / "vllm-rocm-install.tar.gz.sha256" - expected_digest = checksum.read_text().split()[0] - assert hashlib.sha256(archive.read_bytes()).hexdigest() == expected_digest - with tarfile.open(archive, "r:gz") as tar: - members = {name.removeprefix("./") for name in tar.getnames()} - assert WHEEL_NAME in members - assert ".vllm-ci-artifact/commit.txt" in members - assert ".vllm-ci-artifact/native-base-image.txt" in members - assert ".vllm-ci-artifact/wheel-filename.txt" in members - assert (artifact_dir / "commit.txt").read_text().strip() == EXPECTED_COMMIT - assert (artifact_dir / "native-base-image.txt").read_text().strip() == ( - EXPECTED_BASE_IMAGE - ) - assert (artifact_dir / "wheel-filename.txt").read_text().strip() == WHEEL_NAME - assert ( - "artifact upload artifacts/vllm-rocm-install/*" - in (tmp_path / "producer-agent.log").read_text() - ) - assert not (tmp_path / "wheel-export").exists() - - -@pytest.mark.parametrize("wheel_names", [(), (WHEEL_NAME, "second.whl")]) -def test_rocm_artifact_producer_rejects_wrong_wheel_count( - tmp_path: Path, - wheel_names: tuple[str, ...], -) -> None: - result = _run_producer(tmp_path, wheel_names=wheel_names) - - assert result.returncode != 0 - assert f"found {len(wheel_names)}" in result.stderr - assert not (tmp_path / "producer-agent.log").exists() - - -def test_rocm_artifact_producer_propagates_upload_failure(tmp_path: Path) -> None: - result = _run_producer( - tmp_path, - wheel_names=(WHEEL_NAME,), - upload_failure=True, - ) - - assert result.returncode != 0 - assert ( - "artifact upload artifacts/vllm-rocm-install/*" - in (tmp_path / "producer-agent.log").read_text() - ) - assert (tmp_path / "wheel-export").exists() - - -def test_native_artifact_retries_when_checksum_is_initially_missing( - tmp_path: Path, -) -> None: - archive, checksum = _create_artifact(tmp_path) - env = _native_env(tmp_path, archive, checksum, checksum_failures=1) - - result = _run_prepare(env) - - assert result.returncode == 0, result.stdout + result.stderr - assert (tmp_path / "checksum-attempts").read_text().strip() == "2" - assert (tmp_path / "workspace" / "tests" / "artifact-marker.txt").is_file() - agent_log = (tmp_path / "buildkite-agent.log").read_text() - assert agent_log.count("vllm-rocm-install.tar.gz.sha256") == 2 - assert "--step image-build-amd" in agent_log - - -@pytest.mark.parametrize( - ("recorded_commit", "recorded_base", "expected_message"), - [ - ( - "wrong-commit", - EXPECTED_BASE_IMAGE, - "ROCm artifact commit wrong-commit does not match " + EXPECTED_COMMIT, - ), - ( - EXPECTED_COMMIT, - "rocm/wrong-base:latest", - "ROCm artifact base rocm/wrong-base:latest does not match " - + EXPECTED_BASE_IMAGE, - ), - ], -) -def test_native_artifact_rejects_wrong_provenance( - tmp_path: Path, - recorded_commit: str, - recorded_base: str, - expected_message: str, -) -> None: - archive, checksum = _create_artifact( - tmp_path, - recorded_commit=recorded_commit, - recorded_base=recorded_base, - ) - env = _native_env(tmp_path, archive, checksum) - - result = _run_prepare(env) - - assert result.returncode != 0 - assert expected_message in result.stderr - - -def test_native_artifact_rejects_duplicate_top_level_wheels(tmp_path: Path) -> None: - archive, checksum = _create_artifact( - tmp_path, - wheel_names=(WHEEL_NAME, "second-wheel-0.0.0-py3-none-any.whl"), - ) - env = _native_env(tmp_path, archive, checksum) - - result = _run_prepare(env) - - assert result.returncode != 0 - assert "must contain exactly one top-level wheel; found 2" in result.stderr - - -@pytest.mark.parametrize( - "relationship", - ["same", "checkout-parent", "workspace-parent"], -) -def test_native_workspace_refuses_to_overlap_checkout( - tmp_path: Path, - relationship: str, -) -> None: - paths = tmp_path / "paths" - if relationship == "same": - workspace = checkout = paths / "workspace" - elif relationship == "checkout-parent": - checkout = paths / "checkout" - workspace = checkout / "workspace" - else: - workspace = paths / "workspace" - checkout = workspace / "checkout" - - workspace.mkdir(parents=True) - checkout.mkdir(parents=True, exist_ok=True) - sentinel = workspace / "keep-me" - sentinel.write_text("legitimate checkout content") - archive, checksum = _create_artifact(tmp_path) - env = _native_env( - tmp_path, - archive, - checksum, - workspace=workspace, - checkout=checkout, - ) - - result = _run_prepare(env) - - assert result.returncode != 0 - assert "Refusing to replace" in result.stderr - assert sentinel.read_text() == "legitimate checkout content" - assert not (tmp_path / "python3.log").exists() - assert not (tmp_path / "buildkite-agent.log").exists() - - -def test_native_child_shell_propagates_pipeline_failure(tmp_path: Path) -> None: - archive, checksum = _create_artifact(tmp_path) - env = _native_env(tmp_path, archive, checksum) - env.update( - { - "AMD_CI_RUNTIME": "native", - "VLLM_CI_RUN_AMD_TEST_LIB_ONLY": "0", - "VLLM_CI_EXPECTED_GPU_COUNT": "0", - "VLLM_TEST_COMMANDS": "false | true", - } - ) - - result = subprocess.run( - ["bash", str(RUNNER)], - cwd=REPO_ROOT, - env=env, - capture_output=True, - text=True, - timeout=30, - ) - - assert result.returncode == 1, result.stdout + result.stderr - assert "Native test commands: false | true" in result.stdout - assert "Native CPU-only AMD job" in result.stdout diff --git a/tests/tools/test_validate_amd_native_ci.py b/tests/tools/test_validate_amd_native_ci.py deleted file mode 100644 index 154fdac7d599..000000000000 --- a/tests/tools/test_validate_amd_native_ci.py +++ /dev/null @@ -1,226 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from pathlib import Path -from typing import Any - -import pytest -import yaml - -from tools.pre_commit import validate_amd_native_ci as validator - - -@pytest.fixture -def config_paths(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): - test_amd = tmp_path / "test-amd.yaml" - test_areas = tmp_path / "test_areas" - test_areas.mkdir() - monkeypatch.setattr(validator, "TEST_AMD", test_amd) - monkeypatch.setattr(validator, "TEST_AREAS", test_areas) - return test_amd, test_areas - - -def _write_yaml(path: Path, steps: list[dict[str, Any]]) -> None: - path.write_text(yaml.safe_dump({"steps": steps}, sort_keys=False)) - - -def _legacy_step( - device: str, - *, - native_ci: Any = None, - **overrides: Any, -) -> dict[str, Any]: - step: dict[str, Any] = { - "label": f"Legacy {device}", - "mirror_hardwares": ["amdproduction"], - "agent_pool": device, - } - if native_ci is not None: - step["native_ci"] = native_ci - step.update(overrides) - return step - - -def test_legacy_accepts_explicit_modes_for_current_and_future_devices( - config_paths, -) -> None: - test_amd, _ = config_paths - steps = [ - _legacy_step("mi300_1", native_ci=True), - _legacy_step("mi300_1", native_ci=False), - _legacy_step("mi325_1", native_ci=True), - _legacy_step("mi325_1"), - _legacy_step("mi400_16", native_ci=True, num_gpus=16), - ] - _write_yaml(test_amd, steps) - - errors, count = validator.validate_test_amd_config() - - assert errors == [] - assert count == 5 - - -@pytest.mark.parametrize( - "step", - [ - _legacy_step("mi300_1", native_ci="true"), - _legacy_step("mi400_16", native_ci=1, num_gpus=16), - ], -) -def test_legacy_rejects_non_boolean_native_flag( - config_paths, - step: dict[str, Any], -) -> None: - test_amd, _ = config_paths - _write_yaml(test_amd, [step]) - - errors, _ = validator.validate_test_amd_config() - - assert any("native_ci must be a boolean" in error for error in errors) - - -@pytest.mark.parametrize( - "step", - [ - _legacy_step("mi400_16", native_ci=True), - _legacy_step("mi400_16", native_ci=True, num_gpus=8), - ], -) -def test_legacy_rejects_gpu_count_that_does_not_match_pool_suffix( - config_paths, - step: dict[str, Any], -) -> None: - test_amd, _ = config_paths - _write_yaml(test_amd, [step]) - - errors, _ = validator.validate_test_amd_config() - - assert any("does not match mi400_16 (16)" in error for error in errors) - - -@pytest.mark.parametrize( - ("overrides", "message"), - [ - ({"num_nodes": 2}, "cannot be multi-node"), - ({"no_plugin": True}, "cannot use no_plugin"), - ], -) -def test_legacy_rejects_unsupported_native_modes( - config_paths, - overrides: dict[str, Any], - message: str, -) -> None: - test_amd, _ = config_paths - step = _legacy_step("mi400_16", native_ci=True, num_gpus=16, **overrides) - _write_yaml(test_amd, [step]) - - errors, _ = validator.validate_test_amd_config() - - assert any(message in error for error in errors) - - -def test_test_areas_accept_direct_and_nested_amd_devices(config_paths) -> None: - _, test_areas = config_paths - _write_yaml( - test_areas / "valid.yaml", - [ - { - "label": "Direct native future device", - "device": "mi400_2", - "native_ci": True, - "num_devices": 2, - }, - { - "label": "Direct legacy current device", - "device": "mi300_1", - }, - { - "label": "Label does not determine runtime mode", - "device": "h100", - "mirror": {"amd": {"device": "mi300_1"}}, - }, - { - "label": "Nested native mirror", - "device": "h100", - "num_devices": 16, - "mirror": { - "amd": { - "device": "mi400_16", - "native_ci": True, - } - }, - }, - ], - ) - - errors, count = validator.validate_test_areas() - - assert errors == [] - assert count == 4 - - -def test_test_areas_reject_native_flag_on_non_amd_direct_step(config_paths) -> None: - _, test_areas = config_paths - _write_yaml( - test_areas / "invalid.yaml", - [{"label": "Not actually AMD", "device": "h100", "native_ci": True}], - ) - - errors, count = validator.validate_test_areas() - - assert count == 0 - assert any("native_ci is set on a non-AMD direct step" in error for error in errors) - - -@pytest.mark.parametrize( - ("amd", "message"), - [ - ({"device": "mi400_1", "native_ci": "true"}, "must be a boolean"), - ( - {"device": "mi400_1", "native_ci": True, "num_nodes": 2}, - "cannot be multi-node", - ), - ( - {"device": "mi400_1", "native_ci": True, "no_plugin": True}, - "cannot use no_plugin", - ), - ], -) -def test_test_areas_reject_invalid_nested_amd_policy( - config_paths, - amd: dict[str, Any], - message: str, -) -> None: - _, test_areas = config_paths - _write_yaml( - test_areas / "invalid.yaml", - [{"label": "Nested mirror", "device": "h100", "mirror": {"amd": amd}}], - ) - - errors, _ = validator.validate_test_areas() - - assert any(message in error for error in errors) - - -def test_mirror_gpu_count_inherits_top_level_count_for_validation(config_paths) -> None: - _, test_areas = config_paths - _write_yaml( - test_areas / "invalid.yaml", - [ - { - "label": "Mismatched mirror GPU count", - "device": "h100", - "num_devices": 8, - "mirror": { - "amd": { - "device": "mi300_4", - "native_ci": True, - } - }, - } - ], - ) - - errors, _ = validator.validate_test_areas() - - assert any("GPU count 8 does not match mi300_4 (4)" in error for error in errors) diff --git a/tools/pre_commit/validate_amd_native_ci.py b/tools/pre_commit/validate_amd_native_ci.py deleted file mode 100644 index ca3bbdd79e19..000000000000 --- a/tools/pre_commit/validate_amd_native_ci.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from pathlib import Path -from typing import Any - -import yaml - -TEST_AMD = Path(".buildkite/test-amd.yaml") -TEST_AREAS = Path(".buildkite/test_areas") - - -def _parse_amd_device(device: Any) -> tuple[str, int] | None: - model, separator, gpu_count = str(device or "").rpartition("_") - if ( - separator - and model.startswith("mi") - and model.removeprefix("mi").isdigit() - and gpu_count.isdigit() - and int(gpu_count) > 0 - ): - return model, int(gpu_count) - return None - - -def _first_configured(*values: Any) -> Any: - return next((value for value in values if value is not None), None) - - -def _validate_runtime( - *, - label: str, - device: Any, - native: Any, - num_gpus: Any, - num_nodes: Any, - no_plugin: Any, - require_explicit_gpu_count: bool, -) -> list[str]: - errors: list[str] = [] - parsed_device = _parse_amd_device(device) - if parsed_device is None: - return [f"{label}: invalid AMD device or agent pool {device!r}"] - - _, expected_gpus = parsed_device - if not isinstance(native, bool): - errors.append(f"{label}: native_ci must be a boolean") - - if native and (num_nodes or 1) > 1: - errors.append(f"{label}: native AMD jobs cannot be multi-node") - if native and no_plugin: - errors.append(f"{label}: native AMD jobs cannot use no_plugin") - - configured_gpus = 1 if require_explicit_gpu_count and num_gpus is None else num_gpus - if configured_gpus is not None and configured_gpus != expected_gpus: - errors.append( - f"{label}: GPU count {configured_gpus} does not match " - f"{device} ({expected_gpus})" - ) - - return errors - - -def _load(path: Path) -> dict[str, Any]: - data = yaml.safe_load(path.read_text()) - if not isinstance(data, dict): - raise ValueError(f"{path} must contain a YAML mapping") - return data - - -def validate_test_amd_config() -> tuple[list[str], int]: - errors: list[str] = [] - validated = 0 - for index, step in enumerate(_load(TEST_AMD).get("steps", []), start=1): - if not isinstance(step, dict) or "mirror_hardwares" not in step: - continue - label = f"{TEST_AMD}:{step.get('label', f'step {index}')}" - errors.extend( - _validate_runtime( - label=label, - device=step.get("agent_pool"), - native=step.get("native_ci", False), - num_gpus=step.get("num_gpus"), - num_nodes=step.get("num_nodes"), - no_plugin=step.get("no_plugin", False), - require_explicit_gpu_count=True, - ) - ) - validated += 1 - return errors, validated - - -def validate_test_areas() -> tuple[list[str], int]: - errors: list[str] = [] - validated = 0 - for path in sorted(TEST_AREAS.glob("*.yaml")): - for index, step in enumerate(_load(path).get("steps", []), start=1): - if not isinstance(step, dict): - continue - label = f"{path}:{step.get('label', f'step {index}')}" - direct_device = step.get("device") - direct_is_amd = _parse_amd_device(direct_device) is not None - if direct_is_amd: - errors.extend( - _validate_runtime( - label=label, - device=direct_device, - native=step.get("native_ci", False), - num_gpus=_first_configured( - step.get("num_devices"), step.get("num_gpus") - ), - num_nodes=step.get("num_nodes"), - no_plugin=step.get("no_plugin", False), - require_explicit_gpu_count=True, - ) - ) - validated += 1 - elif "native_ci" in step: - errors.append(f"{label}: native_ci is set on a non-AMD direct step") - - mirror = step.get("mirror") - amd = mirror.get("amd") if isinstance(mirror, dict) else None - if not isinstance(amd, dict): - continue - errors.extend( - _validate_runtime( - label=f"{label} [AMD mirror]", - device=amd.get("device"), - native=amd.get("native_ci", False), - num_gpus=_first_configured( - amd.get("num_devices"), - amd.get("num_gpus"), - step.get("num_devices"), - step.get("num_gpus"), - ), - num_nodes=amd.get("num_nodes", step.get("num_nodes")), - no_plugin=amd.get("no_plugin", step.get("no_plugin", False)), - require_explicit_gpu_count=True, - ) - ) - validated += 1 - return errors, validated - - -def main() -> int: - test_amd_errors, test_amd_count = validate_test_amd_config() - area_errors, area_count = validate_test_areas() - errors = test_amd_errors + area_errors - if errors: - raise SystemExit( - "Invalid AMD native CI configuration:\n- " + "\n- ".join(errors) - ) - - print( - "AMD native CI configuration validation passed " - f"({test_amd_count} test-amd steps, {area_count} test-area steps)" - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From ee681c9ca73d0bf07a4522933ebfd9d383d952aa Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sun, 12 Jul 2026 02:51:28 -0500 Subject: [PATCH 06/16] [CI][AMD] Remove migration-only cleanup hooks Signed-off-by: Andreas Karatzas --- .buildkite/scripts/ci-bake-rocm.sh | 4 ---- .buildkite/scripts/hardware_ci/run-amd-test.sh | 4 ---- .buildkite/test-amd.yaml | 4 +--- .buildkite/test_areas/misc.yaml | 2 -- 4 files changed, 1 insertion(+), 13 deletions(-) diff --git a/.buildkite/scripts/ci-bake-rocm.sh b/.buildkite/scripts/ci-bake-rocm.sh index 48cc58d4f6e6..2891291105f7 100644 --- a/.buildkite/scripts/ci-bake-rocm.sh +++ b/.buildkite/scripts/ci-bake-rocm.sh @@ -2121,8 +2121,4 @@ main() { upload_wheel_artifacts_if_present } -if [[ "${VLLM_CI_BAKE_ROCM_LIB_ONLY:-0}" == "1" ]]; then - return 0 2>/dev/null || exit 0 -fi - main "$@" diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 0aa394d6609c..3ac2a6fc46a0 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -600,10 +600,6 @@ re_quote_pytest_markers() { # Main ############################################################################### -if [[ "${VLLM_CI_RUN_AMD_TEST_LIB_ONLY:-0}" == "1" ]]; then - return 0 2>/dev/null || exit 0 -fi - if is_native_runtime; then echo "--- Native in-pod ROCm CI (AMD_CI_RUNTIME=${AMD_CI_RUNTIME:-unset}, NATIVE_CI=${NATIVE_CI:-unset})" artifact_work_dir="" diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 43b6a13de2c6..805aa2a685d5 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -16,7 +16,7 @@ # mirror_hardwares(list): the list of hardware to run the test on as well. currently only supports [amdexperimental] # native_ci(bool): when true, run the job directly in the AMD Kubernetes pod. # When false or omitted, use the legacy Docker-in-Docker path. -# num_gpus(int): override the number of GPUs for the test. defaults to 1 GPU. currently supports 2,4,8. +# num_gpus(int): override the number of GPUs for the test. defaults to 1 GPU. currently supports 2,4. # num_nodes(int): whether to simulate multi-node setup by launching multiple containers on one host, # in this case, commands must be specified. the first command runs on the first host, the second # command runs on the second host. @@ -3021,8 +3021,6 @@ steps: #----------------------------------------------------------- mi325 · misc ------------------------------------------------------------# -# This test keeps the legacy Docker path because it requires the full source -# tree rather than the thin native artifact. - label: Python-only Installation # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 4fe82790a3ce..36d9b4a74770 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -282,8 +282,6 @@ steps: commands: - bash standalone_tests/python_only_compile.sh mirror: - # This mirror intentionally leaves native_ci disabled because the test - # requires the full source tree rather than the thin native artifact. amd: device: mi325_1 timeout_in_minutes: 45 From 249217b4e0406ec615aa3b357748e6ad13a33289 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sun, 12 Jul 2026 19:12:04 -0500 Subject: [PATCH 07/16] [CI][AMD] Fix native MI300 test failures Signed-off-by: Andreas Karatzas --- .buildkite/scripts/hardware_ci/run-amd-test.sh | 9 +++++++-- .buildkite/test-amd.yaml | 10 +++++----- .buildkite/test_areas/disaggregated.yaml | 2 +- tests/basic_correctness/test_mem.py | 2 +- .../spec_decode_acceptance_test.sh | 15 ++++++++++++--- .../unit/test_mooncake_store_hma_e2e.py | 4 +++- 6 files changed, 29 insertions(+), 13 deletions(-) diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 3ac2a6fc46a0..f15a31bcf258 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -346,6 +346,7 @@ prepare_native_workspace() { initialize_native_environment() { local job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-local}}" + local job_id_suffix="" local native_root="" local hf_mount="" @@ -355,8 +356,11 @@ initialize_native_environment() { fi job_id="${job_id//[^A-Za-z0-9_.-]/_}" + job_id_suffix="${job_id##*-}" + job_id_suffix="${job_id_suffix:0:12}" native_root="/tmp/vllm-native-${job_id}" - : "${TMPDIR:=${native_root}/tmp}" + TMPDIR="/tmp/vllm-${job_id_suffix}/tmp" + VLLM_RPC_BASE_PATH="/tmp" : "${TORCHINDUCTOR_CACHE_DIR:=${native_root}/cache/torchinductor}" : "${TRITON_CACHE_DIR:=${native_root}/cache/triton}" : "${VLLM_CACHE_ROOT:=${native_root}/cache/vllm}" @@ -364,7 +368,8 @@ initialize_native_environment() { : "${HF_HOME:=/home/buildkite-agent/huggingface}" : "${HF_HUB_DOWNLOAD_TIMEOUT:=300}" : "${HF_HUB_ETAG_TIMEOUT:=60}" - export TMPDIR TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT XDG_CACHE_HOME + export TMPDIR VLLM_RPC_BASE_PATH + export TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT XDG_CACHE_HOME export HF_HOME HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT export PYTORCH_ROCM_ARCH="" diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 805aa2a685d5..13c81c78017e 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -1333,6 +1333,7 @@ steps: - tests/evals/ commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - export PYTORCH_ROCM_ARCH=gfx942 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt - label: ROCm LM Eval Large Models (8 GPUs) # TBD @@ -1580,8 +1581,9 @@ steps: commands: - apt-get update && apt-get install -y curl libsodium23 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s model_executor -m '(not slow_test)' - - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py + - export PYTHONFAULTHANDLER=1 + - pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread + - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread #---------------------------------------------------- mi300 · model_runner_v2 -------------------------------------------------------# @@ -1917,7 +1919,6 @@ steps: native_ci: true agent_pool: mi300_1 parallelism: 4 - no_gpu: true optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2523,7 +2524,6 @@ steps: mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] native_ci: true agent_pool: mi300_1 - no_gpu: true optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2630,7 +2630,7 @@ steps: - vllm/platforms/rocm.py commands: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh + - KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh - label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs) # TBD timeout_in_minutes: 180 diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index fff7f502d164..84da92330792 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -187,7 +187,7 @@ steps: - vllm/platforms/rocm.py commands: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh + - KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh - label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs) key: multiconnector-nixl-offloading-pd-edge-cases-2-gpus diff --git a/tests/basic_correctness/test_mem.py b/tests/basic_correctness/test_mem.py index c0f8a5922239..e4b3fb65842f 100644 --- a/tests/basic_correctness/test_mem.py +++ b/tests/basic_correctness/test_mem.py @@ -31,7 +31,7 @@ def test_python_error(): x = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE) tensors.append(x) # release the memory - allocator.sleep() + allocator.sleep(offload_tags=()) # allocate more memory than the total memory y = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE) diff --git a/tests/v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh b/tests/v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh index 10e119d48a97..501ec332d69e 100755 --- a/tests/v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh +++ b/tests/v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh @@ -20,7 +20,8 @@ # SD_MODEL - drafter model path # MODEL_NAME - target model (default: meta-llama/Llama-3.1-8B-Instruct) # NUM_SPEC_TOKENS - number of speculative tokens (default: 3) -# GPU_MEMORY_UTILIZATION - (default: 0.7) +# GPU_MEMORY_UTILIZATION - used when KV_CACHE_MEMORY_BYTES is unset (default: 0.7) +# KV_CACHE_MEMORY_BYTES - optional KV cache size per server # ATTENTION_BACKEND - attention backend to use # Default: TRITON_ATTN on ROCm, FLASH_ATTN on NVIDIA # ROCm options: TRITON_ATTN, ROCM_ATTN, ROCM_AITER_FA, @@ -52,10 +53,17 @@ NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1} PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1} DECODER_TP_SIZE=${DECODER_TP_SIZE:-1} GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.7} +KV_CACHE_MEMORY_BYTES=${KV_CACHE_MEMORY_BYTES:-} BLOCK_SIZE=${BLOCK_SIZE:-16} SERVER_HOST="${SERVER_HOST:-127.0.0.1}" NIXL_SIDE_CHANNEL_HOST="${NIXL_SIDE_CHANNEL_HOST:-$SERVER_HOST}" +if [[ -n "$KV_CACHE_MEMORY_BYTES" ]]; then + KV_CACHE_ARGS=(--kv-cache-memory-bytes "$KV_CACHE_MEMORY_BYTES") +else + KV_CACHE_ARGS=(--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION") +fi + # Resolve the repository root from the script location instead of `.git`. SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}" @@ -209,6 +217,7 @@ run_test_for_device() { echo "SD model: ${SD_MODEL}" echo "Spec tokens: ${NUM_SPEC_TOKENS}" echo "KV buffer device: ${kv_device}" + echo "KV cache memory: ${KV_CACHE_MEMORY_BYTES:-auto-sized}" echo "Attention backend: ${ATTENTION_BACKEND}" echo "GPU platform: ${GPU_PLATFORM}" echo "Server host: ${SERVER_HOST}" @@ -248,7 +257,7 @@ run_test_for_device() { --enforce-eager \ --max-model-len $MAX_MODEL_LEN \ --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \ + "${KV_CACHE_ARGS[@]}" \ --tensor-parallel-size $PREFILLER_TP_SIZE \ --kv-transfer-config "$kv_config_p" \ --speculative-config "$PREFILL_SPEC_CONFIG" \ @@ -287,7 +296,7 @@ run_test_for_device() { --enforce-eager \ --max-model-len $MAX_MODEL_LEN \ --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \ + "${KV_CACHE_ARGS[@]}" \ --tensor-parallel-size $DECODER_TP_SIZE \ --kv-transfer-config "$kv_config_d" \ --speculative-config "$DECODE_SPEC_CONFIG" \ diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py index d0a50f465796..f09e0a24729b 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py @@ -97,14 +97,16 @@ def _build_worker_with_dict_store(vllm_config, kv_cache_config, store): with ( patch.dict(sys.modules, {"mooncake.store": fake_mooncake_store}), patch.object(mooncake_store_worker, "MooncakeStoreConfig") as MCfg, + patch.object(mooncake_store_worker, "LookupKeyServer"), ): - sc = MCfg.load_from_env.return_value + sc = MCfg.load_from_config.return_value sc.metadata_server = "" sc.global_segment_size = 1 << 20 sc.local_buffer_size = 1 << 20 sc.protocol = "tcp" sc.device_name = "" sc.master_server_address = "" + sc.mode = "embedded" sc.enable_offload = False with ( patch( From 3fe5dae06c6d97c23c959fbc02ce9d7061f61ec4 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sun, 12 Jul 2026 23:16:59 -0500 Subject: [PATCH 08/16] [CI][ROCm] Respect pod memory for model loading Signed-off-by: Andreas Karatzas --- ...lama-4-Maverick-17B-128E-Instruct-FP8.yaml | 1 + .../test_lm_eval_correctness.py | 5 +++ tests/model_executor/test_weight_utils.py | 34 +++++++++++++++++++ .../model_loader/weight_utils.py | 16 +++++++-- 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml index 9a5af8540118..a9c9857fc30f 100644 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml +++ b/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml @@ -1,6 +1,7 @@ # For hf script, without -t option (tensor parallel size). # bash .buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh -m meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 -l 250 -t 8 -f 5 model_name: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8" +rocm_safetensors_load_strategy: lazy required_gpu_arch: - gfx942 - gfx950 diff --git a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py b/.buildkite/lm-eval-harness/test_lm_eval_correctness.py index dd2fd5f05b42..9335ce02900c 100644 --- a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py +++ b/.buildkite/lm-eval-harness/test_lm_eval_correctness.py @@ -72,6 +72,11 @@ def launch_lm_eval(eval_config, tp_size): if moe_backend is not None: model_args += f"moe_backend={moe_backend}," + if current_platform.is_rocm(): + rocm_load_strategy = eval_config.get("rocm_safetensors_load_strategy") + if rocm_load_strategy is not None: + model_args += f"safetensors_load_strategy={rocm_load_strategy}," + env_vars = eval_config.get("env_vars", None) with scoped_env_vars(env_vars): results = lm_eval.simple_evaluate( diff --git a/tests/model_executor/test_weight_utils.py b/tests/model_executor/test_weight_utils.py index 9e67609b78e4..16cb3878e263 100644 --- a/tests/model_executor/test_weight_utils.py +++ b/tests/model_executor/test_weight_utils.py @@ -2,15 +2,49 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import tempfile +from types import SimpleNamespace import huggingface_hub.constants +import psutil import pytest from huggingface_hub.utils import LocalEntryNotFoundError +import vllm.model_executor.model_loader.weight_utils as weight_utils from vllm.model_executor.model_loader.weight_utils import ( download_weights_from_hf, maybe_remap_kv_scale_name, ) +from vllm.utils import cpu_resource_utils + + +@pytest.mark.parametrize( + ("is_rocm", "cgroup_memory", "expected"), + [ + (False, (500, 200), 800), + (True, (None, None), 800), + (True, (500, None), 500), + (True, (500, 200), 300), + (True, (500, 600), 0), + ], +) +def test_get_available_ram_bytes(monkeypatch, is_rocm, cgroup_memory, expected): + monkeypatch.setattr( + psutil, + "virtual_memory", + lambda: SimpleNamespace(available=800), + ) + monkeypatch.setattr( + weight_utils, + "current_platform", + SimpleNamespace(is_rocm=lambda: is_rocm), + ) + monkeypatch.setattr( + cpu_resource_utils, + "get_cgroup_memory_limit", + lambda: cgroup_memory, + ) + + assert weight_utils._get_available_ram_bytes() == expected def test_download_weights_from_hf(): diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index ef25a156d9ea..8e673ae7fcb5 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -694,10 +694,22 @@ def _get_checkpoints_size_bytes(files: list[str]) -> int: def _get_available_ram_bytes() -> int: - """Return the available RAM in bytes.""" + """Return available RAM, honoring cgroup limits on ROCm.""" import psutil - return psutil.virtual_memory().available + host_available = psutil.virtual_memory().available + if not current_platform.is_rocm(): + return host_available + + from vllm.utils.cpu_resource_utils import get_cgroup_memory_limit + + cgroup_limit, cgroup_usage = get_cgroup_memory_limit() + if cgroup_limit is None: + return host_available + cgroup_available = ( + cgroup_limit if cgroup_usage is None else max(0, cgroup_limit - cgroup_usage) + ) + return min(host_available, cgroup_available) def _get_fs_type(files: list[str]) -> str: From e9ff4e9f17c7f0280cc4c73ff703c8c79e639125 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sun, 12 Jul 2026 23:40:49 -0500 Subject: [PATCH 09/16] [Bugfix] Fix multimodal CUDA graph capture inputs Signed-off-by: Andreas Karatzas --- tests/v1/cudagraph/test_cudagraph_dispatch.py | 34 +++++++++++++++++++ vllm/v1/worker/gpu/model_states/default.py | 2 ++ 2 files changed, 36 insertions(+) diff --git a/tests/v1/cudagraph/test_cudagraph_dispatch.py b/tests/v1/cudagraph/test_cudagraph_dispatch.py index c10835821f58..88412f4f5381 100644 --- a/tests/v1/cudagraph/test_cudagraph_dispatch.py +++ b/tests/v1/cudagraph/test_cudagraph_dispatch.py @@ -22,6 +22,7 @@ from vllm.forward_context import BatchDescriptor, set_forward_context from vllm.platforms import current_platform from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher +from vllm.v1.worker.gpu.model_states.default import DefaultModelState DEVICE_TYPE = current_platform.device_type @@ -268,6 +269,39 @@ def test_get_capture_descs_empty_when_not_initialized(self): assert dispatcher.get_capture_descs() == [] +@pytest.mark.parametrize( + ("supports_mm_inputs", "requires_raw_input_tokens", "expect_input_ids"), + [(True, False, False), (True, True, True), (False, False, True)], +) +def test_prepare_model_inputs_for_capture( + supports_mm_inputs, requires_raw_input_tokens, expect_input_ids +): + model_state = object.__new__(DefaultModelState) + model_state.supports_mm_inputs = supports_mm_inputs + model_state.model = MagicMock(requires_raw_input_tokens=requires_raw_input_tokens) + model_state.rope_state = None + inputs_embeds = torch.randn(4, 8) + model_state.encoder_runner = MagicMock(inputs_embeds=inputs_embeds) + input_ids = torch.arange(8) + positions = torch.arange(8) + + model_inputs = { + "input_ids": input_ids[:4], + "positions": positions[:4], + **model_state.prepare_dummy_inputs(num_reqs=2, num_tokens=4), + } + + if expect_input_ids: + torch.testing.assert_close(model_inputs["input_ids"], input_ids[:4]) + else: + assert model_inputs["input_ids"] is None + if supports_mm_inputs: + assert model_inputs["inputs_embeds"].data_ptr() == inputs_embeds.data_ptr() + else: + assert "inputs_embeds" not in model_inputs + torch.testing.assert_close(model_inputs["positions"], positions[:4]) + + @pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda") class TestCUDAGraphWrapper: def setup_method(self): diff --git a/vllm/v1/worker/gpu/model_states/default.py b/vllm/v1/worker/gpu/model_states/default.py index 854b71b69fcb..14175cc577fe 100644 --- a/vllm/v1/worker/gpu/model_states/default.py +++ b/vllm/v1/worker/gpu/model_states/default.py @@ -125,6 +125,8 @@ def prepare_dummy_inputs(self, num_reqs: int, num_tokens: int) -> dict[str, Any] if self.supports_mm_inputs: inputs_embeds = self.encoder_runner.inputs_embeds[:num_tokens] model_inputs["inputs_embeds"] = inputs_embeds + if not self.model.requires_raw_input_tokens: + model_inputs["input_ids"] = None if self.rope_state is not None: model_inputs["positions"] = self.rope_state.get_positions(num_tokens) return model_inputs From 3de26dad1e977ad666652b9568d20e8b73afcede Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Mon, 13 Jul 2026 02:23:32 -0500 Subject: [PATCH 10/16] [CI][ROCm] Run V1 spec decode on MI300 Signed-off-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 13c81c78017e..1086fce15b61 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -2442,6 +2442,18 @@ steps: - export VLLM_ALLOW_INSECURE_SERIALIZATION=1 - pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test +- label: V1 Spec Decode # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + native_ci: true + agent_pool: mi300_1 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/v1/spec_decode + commands: + - pytest -v -s -m 'not slow_test' v1/spec_decode + - label: Extract Hidden States Integration # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -3006,19 +3018,6 @@ steps: - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma" - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model -#------------------------------------------------------------ mi325 · v1 -------------------------------------------------------------# - -- label: V1 Spec Decode # TBD - timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - agent_pool: mi325_1 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/v1/spec_decode - commands: - - pytest -v -s -m 'not slow_test' v1/spec_decode - #----------------------------------------------------------- mi325 · misc ------------------------------------------------------------# - label: Python-only Installation # TBD From ff784d450d8d46e34435159854607dfe2e13d2fd Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Mon, 13 Jul 2026 02:23:37 -0500 Subject: [PATCH 11/16] [CI][ROCm] Add pytest hang diagnostics Signed-off-by: Andreas Karatzas --- .../scripts/hardware_ci/run-amd-test.sh | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index f15a31bcf258..4415e2750a26 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -34,10 +34,27 @@ set -o pipefail : "${CLICOLOR_FORCE:=1}" : "${PY_COLORS:=1}" : "${ROCM_DOCKER_TTY:=1}" +: "${PYTHONFAULTHANDLER:=1}" +: "${PYTEST_TIMEOUT:=2100}" if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes" fi -export BUILDKIT_PROGRESS TERM FORCE_COLOR CLICOLOR_FORCE PY_COLORS PYTEST_ADDOPTS ROCM_DOCKER_TTY +if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations="* ]]; then + PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations=25" +fi +if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations-min="* ]]; then + PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations-min=1.0" +fi +# Dump stacks after 15 minutes, then stop an individual test after 35 minutes. +if [[ " ${PYTEST_ADDOPTS:-} " != *" faulthandler_timeout="* ]]; then + PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }-o faulthandler_timeout=900" +fi +if [[ " ${PYTEST_ADDOPTS:-} " != *" --timeout-method="* && + " ${PYTEST_ADDOPTS:-} " != *" --timeout-method "* ]]; then + PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--timeout-method=thread" +fi +export BUILDKIT_PROGRESS TERM FORCE_COLOR CLICOLOR_FORCE PY_COLORS PYTEST_ADDOPTS PYTEST_TIMEOUT ROCM_DOCKER_TTY +export PYTHONFAULTHANDLER # Export Python path for commands that run directly on the host. Containerized # tests set this to /vllm-workspace below so spawned Python processes do not @@ -893,7 +910,9 @@ else -e FORCE_COLOR \ -e CLICOLOR_FORCE \ -e PY_COLORS \ + -e PYTHONFAULTHANDLER \ -e PYTEST_ADDOPTS \ + -e PYTEST_TIMEOUT \ -v "${HF_CACHE}:${HF_MOUNT}" \ -e "HF_HOME=${HF_MOUNT}" \ -e "PYTHONPATH=${MYPYTHONPATH}" \ From 85997268011423038689b8c95e0606a3bdf1ed21 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Mon, 13 Jul 2026 14:44:07 -0500 Subject: [PATCH 12/16] [CI] Remove out-of-scope regression tests Signed-off-by: Andreas Karatzas --- tests/model_executor/test_weight_utils.py | 34 ------------------- tests/v1/cudagraph/test_cudagraph_dispatch.py | 34 ------------------- 2 files changed, 68 deletions(-) diff --git a/tests/model_executor/test_weight_utils.py b/tests/model_executor/test_weight_utils.py index 16cb3878e263..9e67609b78e4 100644 --- a/tests/model_executor/test_weight_utils.py +++ b/tests/model_executor/test_weight_utils.py @@ -2,49 +2,15 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import tempfile -from types import SimpleNamespace import huggingface_hub.constants -import psutil import pytest from huggingface_hub.utils import LocalEntryNotFoundError -import vllm.model_executor.model_loader.weight_utils as weight_utils from vllm.model_executor.model_loader.weight_utils import ( download_weights_from_hf, maybe_remap_kv_scale_name, ) -from vllm.utils import cpu_resource_utils - - -@pytest.mark.parametrize( - ("is_rocm", "cgroup_memory", "expected"), - [ - (False, (500, 200), 800), - (True, (None, None), 800), - (True, (500, None), 500), - (True, (500, 200), 300), - (True, (500, 600), 0), - ], -) -def test_get_available_ram_bytes(monkeypatch, is_rocm, cgroup_memory, expected): - monkeypatch.setattr( - psutil, - "virtual_memory", - lambda: SimpleNamespace(available=800), - ) - monkeypatch.setattr( - weight_utils, - "current_platform", - SimpleNamespace(is_rocm=lambda: is_rocm), - ) - monkeypatch.setattr( - cpu_resource_utils, - "get_cgroup_memory_limit", - lambda: cgroup_memory, - ) - - assert weight_utils._get_available_ram_bytes() == expected def test_download_weights_from_hf(): diff --git a/tests/v1/cudagraph/test_cudagraph_dispatch.py b/tests/v1/cudagraph/test_cudagraph_dispatch.py index 88412f4f5381..c10835821f58 100644 --- a/tests/v1/cudagraph/test_cudagraph_dispatch.py +++ b/tests/v1/cudagraph/test_cudagraph_dispatch.py @@ -22,7 +22,6 @@ from vllm.forward_context import BatchDescriptor, set_forward_context from vllm.platforms import current_platform from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher -from vllm.v1.worker.gpu.model_states.default import DefaultModelState DEVICE_TYPE = current_platform.device_type @@ -269,39 +268,6 @@ def test_get_capture_descs_empty_when_not_initialized(self): assert dispatcher.get_capture_descs() == [] -@pytest.mark.parametrize( - ("supports_mm_inputs", "requires_raw_input_tokens", "expect_input_ids"), - [(True, False, False), (True, True, True), (False, False, True)], -) -def test_prepare_model_inputs_for_capture( - supports_mm_inputs, requires_raw_input_tokens, expect_input_ids -): - model_state = object.__new__(DefaultModelState) - model_state.supports_mm_inputs = supports_mm_inputs - model_state.model = MagicMock(requires_raw_input_tokens=requires_raw_input_tokens) - model_state.rope_state = None - inputs_embeds = torch.randn(4, 8) - model_state.encoder_runner = MagicMock(inputs_embeds=inputs_embeds) - input_ids = torch.arange(8) - positions = torch.arange(8) - - model_inputs = { - "input_ids": input_ids[:4], - "positions": positions[:4], - **model_state.prepare_dummy_inputs(num_reqs=2, num_tokens=4), - } - - if expect_input_ids: - torch.testing.assert_close(model_inputs["input_ids"], input_ids[:4]) - else: - assert model_inputs["input_ids"] is None - if supports_mm_inputs: - assert model_inputs["inputs_embeds"].data_ptr() == inputs_embeds.data_ptr() - else: - assert "inputs_embeds" not in model_inputs - torch.testing.assert_close(model_inputs["positions"], positions[:4]) - - @pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda") class TestCUDAGraphWrapper: def setup_method(self): From d6016e1b59405208a3d3c9fb11126b2514504a50 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Mon, 13 Jul 2026 14:44:13 -0500 Subject: [PATCH 13/16] [CI][ROCm] Isolate runner variables from tests Signed-off-by: Andreas Karatzas --- .../scripts/hardware_ci/run-amd-test.sh | 65 +++++++++++++------ tests/standalone_tests/python_only_compile.sh | 8 +-- 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 4415e2750a26..4e8735c841a7 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -70,6 +70,28 @@ report_docker_usage() { docker system df || true } +clear_ci_orchestration_env() { + unset -v \ + VLLM_TEST_GROUP_NAME \ + VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE \ + VLLM_CI_ARTIFACT_STEP \ + VLLM_TEST_CACHE \ + VLLM_CI_EXECUTION_MODE \ + VLLM_CI_WORKSPACE \ + VLLM_CI_REQUIRE_WORKSPACE_MOUNT \ + VLLM_TEST_COMMANDS \ + VLLM_CI_BRANCH \ + VLLM_CI_BASE_IMAGE \ + VLLM_CI_FALLBACK_IMAGE \ + VLLM_CI_DOCKER_DISABLED \ + VLLM_CI_ARTIFACT_GLOB \ + VLLM_CI_ARTIFACT_CHECKSUM_GLOB \ + VLLM_CI_EXPECTED_GPU_COUNT \ + VLLM_CI_USE_ARTIFACTS \ + VLLM_CI_RESULTS_ROOT \ + VLLM_ALLOW_DEPRECATED_BEAM_SEARCH +} + cleanup_network() { local max_nodes=${NUM_NODES:-2} for node in $(seq 0 $((max_nodes - 1))); do @@ -667,6 +689,8 @@ if is_native_runtime; then echo "Native test commands: $commands" run_native_preflight || exit 1 + # Keep AMD CI orchestration variables out of vLLM's runtime environment. + clear_ci_orchestration_env /bin/bash -o pipefail -c "${commands}" handle_pytest_exit "$?" fi @@ -773,25 +797,27 @@ fi echo "Final commands: $commands" -# The ROCm test image often ships /vllm-workspace without .git (artifact tarball unpack). -# tests/standalone_tests/python_only_compile.sh uses merge-base(HEAD, origin/main) for -# wheels.vllm.ai; compute on the agent (full git checkout) and pass into the container. -vllm_standalone_merge_base="" -checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}" -if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then - checkout="." -fi -# Pass safe.directory per-command (-c) because buildkite runs will always fail -# the next check on git 2.35.2+ due to mixed uses of root and buildkite-agent/uids. -if git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then - vllm_standalone_merge_base="$( - git -c "safe.directory=${checkout}" -C "${checkout}" merge-base HEAD origin/main 2>/dev/null || true - )" -fi -if [[ -z "${vllm_standalone_merge_base}" ]]; then - vllm_standalone_merge_base="${BUILDKITE_COMMIT:-}" +standalone_merge_base_env=() +if [[ "$commands" == *python_only_compile.sh* ]]; then + # The ROCm test image often ships /vllm-workspace without .git. Resolve the + # wheels.vllm.ai commit from the agent checkout for this test only. + vllm_standalone_merge_base="" + checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}" + if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then + checkout="." + fi + # Pass safe.directory per-command because Buildkite uses mixed user IDs. + if git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + vllm_standalone_merge_base="$( + git -c "safe.directory=${checkout}" -C "${checkout}" merge-base HEAD origin/main 2>/dev/null || true + )" + fi + if [[ -z "${vllm_standalone_merge_base}" ]]; then + vllm_standalone_merge_base="${BUILDKITE_COMMIT:-}" + fi + echo "INFO: passing CI_STANDALONE_MERGE_BASE into container: ${vllm_standalone_merge_base}" + standalone_merge_base_env=(-e "CI_STANDALONE_MERGE_BASE=${vllm_standalone_merge_base}") fi -echo "INFO: passing VLLM_STANDALONE_MERGE_BASE into container: ${vllm_standalone_merge_base}" MYPYTHONPATH="/vllm-workspace" @@ -822,6 +848,7 @@ else fi # --- Route: multi-node vs single-node --- +clear_ci_orchestration_env if is_multi_node "$commands"; then echo "--- Multi-node job detected" export DCKR_VER=$(docker --version | sed 's/Docker version \(.*\), build .*/\1/') @@ -922,7 +949,7 @@ else -e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \ -e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \ -e "PYTORCH_ROCM_ARCH=" \ - -e "VLLM_STANDALONE_MERGE_BASE=${vllm_standalone_merge_base}" \ + "${standalone_merge_base_env[@]}" \ --name "${container_name}" \ "${image_name}" \ /bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}" diff --git a/tests/standalone_tests/python_only_compile.sh b/tests/standalone_tests/python_only_compile.sh index ea9d2441ca01..14f563ae6fbb 100644 --- a/tests/standalone_tests/python_only_compile.sh +++ b/tests/standalone_tests/python_only_compile.sh @@ -5,16 +5,16 @@ set -e # ROCm CI runs this script inside `run-amd-test.sh` where /vllm-workspace often has no .git -# (wheel artifact layout). The wrapper passes VLLM_STANDALONE_MERGE_BASE from the agent checkout. +# (wheel artifact layout). The wrapper passes CI_STANDALONE_MERGE_BASE from the agent checkout. merge_base_commit="" -if [[ -n "${VLLM_STANDALONE_MERGE_BASE:-}" ]]; then - merge_base_commit="${VLLM_STANDALONE_MERGE_BASE}" +if [[ -n "${CI_STANDALONE_MERGE_BASE:-}" ]]; then + merge_base_commit="${CI_STANDALONE_MERGE_BASE}" elif merge_base_commit="$(git -C /vllm-workspace merge-base HEAD origin/main 2>/dev/null)"; then : elif merge_base_commit="$(git merge-base HEAD origin/main 2>/dev/null)"; then : else - echo "ERROR: need a git checkout or VLLM_STANDALONE_MERGE_BASE to resolve wheels.vllm.ai commit." >&2 + echo "ERROR: need a git checkout or CI_STANDALONE_MERGE_BASE to resolve wheels.vllm.ai commit." >&2 exit 1 fi From d4036558268341f68b008c99dba65fe5f19b1604 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 14 Jul 2026 00:50:48 -0500 Subject: [PATCH 14/16] [CI][ROCm] Replace native_ci with dind flag Signed-off-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 246 +++++++++--------- .buildkite/test_areas/benchmarks.yaml | 2 +- .buildkite/test_areas/disaggregated.yaml | 10 +- .buildkite/test_areas/distributed.yaml | 2 +- .buildkite/test_areas/engine.yaml | 2 +- .buildkite/test_areas/expert_parallelism.yaml | 2 +- .buildkite/test_areas/lm_eval.yaml | 2 +- .buildkite/test_areas/misc.yaml | 2 +- .buildkite/test_areas/model_executor.yaml | 2 +- .buildkite/test_areas/models_language.yaml | 4 +- .buildkite/test_areas/models_multimodal.yaml | 2 +- .buildkite/test_areas/pytorch.yaml | 4 +- .buildkite/test_areas/weight_loading.yaml | 2 +- 13 files changed, 141 insertions(+), 141 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 1086fce15b61..b879760aa293 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -14,8 +14,8 @@ # command(str): the single command to run for tests. incompatible with commands. # commands(list): the list of commands to run for the test. incompatible with command. # mirror_hardwares(list): the list of hardware to run the test on as well. currently only supports [amdexperimental] -# native_ci(bool): when true, run the job directly in the AMD Kubernetes pod. -# When false or omitted, use the legacy Docker-in-Docker path. +# dind(bool): when false, run the job directly in the AMD Kubernetes pod. +# When true or omitted, use the legacy Docker-in-Docker path. # num_gpus(int): override the number of GPUs for the test. defaults to 1 GPU. currently supports 2,4. # num_nodes(int): whether to simulate multi-node setup by launching multiple containers on one host, # in this case, commands must be specified. the first command runs on the first host, the second @@ -471,7 +471,7 @@ steps: - label: Basic Correctness # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -501,7 +501,7 @@ steps: - label: Distributed Model Tests (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -531,7 +531,7 @@ steps: - label: Benchmarks CLI Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -546,7 +546,7 @@ steps: - label: PyTorch Compilation Unit Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -566,7 +566,7 @@ steps: - label: Fusion E2E Config Sweep (H100-MI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 num_gpus: 1 working_dir: "/vllm-workspace/" @@ -587,7 +587,7 @@ steps: - label: Fusion E2E Quick (H100-MI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 num_gpus: 1 working_dir: "/vllm-workspace/" @@ -609,7 +609,7 @@ steps: - label: PyTorch Compilation Passes Unit Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -621,7 +621,7 @@ steps: - label: PyTorch Fullgraph # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -639,7 +639,7 @@ steps: - label: Pytorch Nightly Dependency Override Check # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true soft_fail: true @@ -653,7 +653,7 @@ steps: - label: Distributed Compile Unit Tests (2xH100-2xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 working_dir: "/vllm-workspace/" @@ -674,7 +674,7 @@ steps: - label: Platform Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -689,7 +689,7 @@ steps: - label: Async Engine, Inputs, Utils, Worker # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -707,7 +707,7 @@ steps: - label: Distributed Comm Ops # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -724,7 +724,7 @@ steps: - label: EPLB Algorithm # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -740,7 +740,7 @@ steps: - label: EPLB Execution # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -756,7 +756,7 @@ steps: - label: Distributed Tests (2xH100-2xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 working_dir: "/vllm-workspace/" @@ -783,7 +783,7 @@ steps: - label: Distributed Tests (4xA100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -799,7 +799,7 @@ steps: - label: Distributed Torchrun + Examples (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -825,7 +825,7 @@ steps: - label: Elastic EP Scaling Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -843,7 +843,7 @@ steps: - label: RayExecutorV2 (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -866,7 +866,7 @@ steps: - label: Distributed Tests (8xH100-8xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_8 num_gpus: 8 optional: true @@ -887,7 +887,7 @@ steps: - label: Entrypoints Unit Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -903,7 +903,7 @@ steps: - label: Entrypoints Integration (LLM) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true fast_check: true @@ -920,7 +920,7 @@ steps: - label: Entrypoints Integration (API Server) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true fast_check: true @@ -938,7 +938,7 @@ steps: - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 fast_check: true optional: true @@ -954,7 +954,7 @@ steps: - label: Entrypoints Integration (API Server OpenAI - Part 2) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 fast_check: true optional: true @@ -971,7 +971,7 @@ steps: - label: Entrypoints Integration (API Server Generate) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true fast_check: true @@ -992,7 +992,7 @@ steps: - label: Entrypoints Integration (Responses API) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -1006,7 +1006,7 @@ steps: - label: Entrypoints Integration (Speech to Text) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -1020,7 +1020,7 @@ steps: - label: Entrypoints Integration (Multimodal) timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -1034,7 +1034,7 @@ steps: - label: Entrypoints Integration (Pooling) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -1048,7 +1048,7 @@ steps: - label: OpenAI API correctness # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1070,7 +1070,7 @@ steps: - label: DeepSeek V2-Lite Prefetch Offload Accuracy (H100-MI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 num_gpus: 1 working_dir: "/vllm-workspace" @@ -1091,7 +1091,7 @@ steps: - label: LM Eval Small Models # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1109,7 +1109,7 @@ steps: - label: MRCR Eval Small Models # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1127,7 +1127,7 @@ steps: - label: LM Eval Small Models (MI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/.buildkite/lm-eval-harness" @@ -1146,7 +1146,7 @@ steps: - label: Multi-Modal Accuracy Eval (Small Models) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/.buildkite/lm-eval-harness" @@ -1162,7 +1162,7 @@ steps: - label: GPQA Eval (GPT-OSS) (2xH100-2xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -1185,7 +1185,7 @@ steps: - label: LM Eval Small Models (2xB200-2xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -1204,7 +1204,7 @@ steps: - label: DeepSeek V2-Lite Sync EPLB Accuracy (4xH100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -1227,7 +1227,7 @@ steps: - label: LM Eval Large Models (4xA100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -1248,7 +1248,7 @@ steps: - label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (4xH100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -1270,7 +1270,7 @@ steps: - label: Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy (4xH100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -1292,7 +1292,7 @@ steps: - label: Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -1315,7 +1315,7 @@ steps: - label: LM Eval Large Models (8xH200-8xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_8 optional: true num_gpus: 8 @@ -1339,7 +1339,7 @@ steps: - label: ROCm LM Eval Large Models (8 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_8 optional: true num_gpus: 8 @@ -1363,7 +1363,7 @@ steps: - label: Examples # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/examples" @@ -1400,7 +1400,7 @@ steps: - label: vLLM IR Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/" @@ -1416,7 +1416,7 @@ steps: - label: Kernels Attention Test %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true parallelism: 2 @@ -1435,7 +1435,7 @@ steps: - label: Kernels Core Operation Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1452,7 +1452,7 @@ steps: - label: Kernels KDA Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1468,7 +1468,7 @@ steps: - label: Kernels MoE Test %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true parallelism: 5 @@ -1490,7 +1490,7 @@ steps: - label: Kernels Quantization Test %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true parallelism: 2 @@ -1509,7 +1509,7 @@ steps: - label: Kernels FP8 MoE Test (2xH100-2xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -1530,7 +1530,7 @@ steps: - label: LoRA %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 parallelism: 4 optional: true @@ -1545,7 +1545,7 @@ steps: - label: LoRA TP (Distributed) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -1566,7 +1566,7 @@ steps: - label: Model Executor # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1590,7 +1590,7 @@ steps: - label: Model Runner V2 Core Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1615,7 +1615,7 @@ steps: - label: Model Runner V2 Examples # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/examples" @@ -1650,7 +1650,7 @@ steps: - label: Model Runner V2 Distributed (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -1672,7 +1672,7 @@ steps: - label: Model Runner V2 Pipeline Parallelism (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -1692,7 +1692,7 @@ steps: - label: Model Runner V2 Spec Decode # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1717,7 +1717,7 @@ steps: - label: Basic Models Tests (Extra Initialization) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 parallelism: 6 working_dir: "/vllm-workspace/tests" @@ -1734,7 +1734,7 @@ steps: - label: Basic Models Tests (Initialization) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1747,7 +1747,7 @@ steps: - label: Basic Models Tests (Other) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1764,7 +1764,7 @@ steps: - label: Language Models Test (Extended Pooling) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1777,7 +1777,7 @@ steps: - label: Language Models Tests (Standard) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1791,7 +1791,7 @@ steps: - label: Language Models Tests (Extra Standard) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true parallelism: 2 @@ -1816,7 +1816,7 @@ steps: - label: Multi-Modal Models (Extended Generation 1) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1831,7 +1831,7 @@ steps: - label: Multi-Modal Models (Extended Generation 2) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1845,7 +1845,7 @@ steps: - label: Multi-Modal Models (Extended Generation 3) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1858,7 +1858,7 @@ steps: - label: "Multi-Modal Models (Standard) 1: qwen2" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1872,7 +1872,7 @@ steps: - label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1887,7 +1887,7 @@ steps: - label: "Multi-Modal Models (Standard) 4: other + whisper" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1902,7 +1902,7 @@ steps: - label: Multi-Modal Processor # 1h 42m timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1916,7 +1916,7 @@ steps: - label: Multi-Modal Processor (CPU) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 parallelism: 4 optional: true @@ -1933,7 +1933,7 @@ steps: - label: Quantized Models Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1950,7 +1950,7 @@ steps: - label: Transformers Nightly Models (Shardable) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 parallelism: 4 optional: true @@ -1973,7 +1973,7 @@ steps: - label: Transformers Nightly Models (Single) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/" @@ -2001,7 +2001,7 @@ steps: - label: Plugin Tests (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -2053,7 +2053,7 @@ steps: - label: GGUF Plugin # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true soft_fail: true @@ -2072,7 +2072,7 @@ steps: - label: Rust Frontend OpenAI Coverage # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2105,7 +2105,7 @@ steps: - label: Rust Frontend Serve Admin Coverage # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2133,7 +2133,7 @@ steps: - label: Rust Frontend Core Correctness # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2151,7 +2151,7 @@ steps: - label: Rust Frontend Tool Use # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2170,7 +2170,7 @@ steps: - label: Rust Frontend Distributed # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -2199,7 +2199,7 @@ steps: - label: Quantization # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2226,7 +2226,7 @@ steps: - label: ROCm AITER Ops Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2244,7 +2244,7 @@ steps: - label: Samplers Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2265,7 +2265,7 @@ steps: - label: Regression # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2280,7 +2280,7 @@ steps: - label: Ray Dependency Compatibility Check # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 working_dir: "/" source_file_dependencies: @@ -2295,7 +2295,7 @@ steps: - label: Acceptance Length Test (Large Models) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2311,7 +2311,7 @@ steps: - label: e2e Scheduling (1 GPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2325,7 +2325,7 @@ steps: - label: Engine (1 GPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2340,7 +2340,7 @@ steps: - label: Spec Decode Draft Model # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2358,7 +2358,7 @@ steps: - label: Spec Decode Eagle # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2376,7 +2376,7 @@ steps: - label: Spec Decode Ngram + Suffix # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2394,7 +2394,7 @@ steps: - label: Spec Decode Speculators + MTP # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2413,7 +2413,7 @@ steps: - label: Speculators Correctness # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2445,7 +2445,7 @@ steps: - label: V1 Spec Decode # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2457,7 +2457,7 @@ steps: - label: Extract Hidden States Integration # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -2489,7 +2489,7 @@ steps: - label: V1 attention (H100-MI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2507,7 +2507,7 @@ steps: - label: V1 Core + KV + Metrics # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2534,7 +2534,7 @@ steps: - label: V1 others (CPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2551,7 +2551,7 @@ steps: - label: V1 Sample + Logits # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2572,7 +2572,7 @@ steps: - label: Distributed DP Tests (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 optional: true num_gpus: 2 @@ -2597,7 +2597,7 @@ steps: - label: Metrics, Tracing (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 optional: true num_gpus: 2 @@ -2616,7 +2616,7 @@ steps: - label: V1 e2e (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -2630,7 +2630,7 @@ steps: - label: NixlConnector PD + Spec Decode acceptance (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -2647,7 +2647,7 @@ steps: - label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -2663,7 +2663,7 @@ steps: - label: Distributed DP Tests (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -2686,7 +2686,7 @@ steps: - label: Distributed NixlConnector PD accuracy (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -2702,7 +2702,7 @@ steps: - label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -2718,7 +2718,7 @@ steps: - label: Hybrid SSM NixlConnector PD accuracy tests (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -2734,7 +2734,7 @@ steps: - label: Hybrid SSM NixlConnector PD prefix cache test (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -2752,7 +2752,7 @@ steps: - label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -2771,7 +2771,7 @@ steps: - label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -2790,7 +2790,7 @@ steps: - label: V1 e2e (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -2804,7 +2804,7 @@ steps: - label: V1 e2e (4xH100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -2820,7 +2820,7 @@ steps: - label: Weight Loading Multiple GPU # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -2833,7 +2833,7 @@ steps: - label: Weight Loading Multiple GPU - Large Models # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - native_ci: true + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true diff --git a/.buildkite/test_areas/benchmarks.yaml b/.buildkite/test_areas/benchmarks.yaml index e8ef928569e7..4b709f3962fd 100644 --- a/.buildkite/test_areas/benchmarks.yaml +++ b/.buildkite/test_areas/benchmarks.yaml @@ -13,7 +13,7 @@ steps: - pytest -v -s benchmarks/ mirror: amd: - native_ci: true + dind: false device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index 84da92330792..b25884a96f54 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -15,7 +15,7 @@ steps: - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: - native_ci: true + dind: false device: mi300_4 timeout_in_minutes: 85 depends_on: @@ -66,7 +66,7 @@ steps: - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: - native_ci: true + dind: false device: mi300_4 timeout_in_minutes: 60 depends_on: @@ -92,7 +92,7 @@ steps: - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: - native_ci: true + dind: false device: mi300_4 timeout_in_minutes: 85 depends_on: @@ -118,7 +118,7 @@ steps: - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: - native_ci: true + dind: false device: mi300_4 timeout_in_minutes: 80 depends_on: @@ -175,7 +175,7 @@ steps: - bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh mirror: amd: - native_ci: true + dind: false device: mi300_2 timeout_in_minutes: 70 depends_on: diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index 72f8ab402819..d9fcff19f056 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -39,7 +39,7 @@ steps: - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py mirror: amd: - native_ci: true + dind: false device: mi300_2 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index b32f3ae9c4cb..4557aa0fcbec 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -114,7 +114,7 @@ steps: - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism" mirror: amd: - native_ci: true + dind: false device: mi300_2 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/expert_parallelism.yaml b/.buildkite/test_areas/expert_parallelism.yaml index 910c24dcdacc..113ccdc3d1a3 100644 --- a/.buildkite/test_areas/expert_parallelism.yaml +++ b/.buildkite/test_areas/expert_parallelism.yaml @@ -16,7 +16,7 @@ steps: - pytest -v -s distributed/test_eplb_utils.py mirror: amd: - native_ci: true + dind: false device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index a701f967337d..d2cee7b6365a 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -117,7 +117,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200.txt mirror: amd: - native_ci: true + dind: false device: mi300_8 timeout_in_minutes: 60 depends_on: diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 36d9b4a74770..fb08373c9a64 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -23,7 +23,7 @@ steps: - pytest -v -s -m 'not slow_test' v1/spec_decode mirror: amd: - native_ci: true + dind: false device: mi300_1 timeout_in_minutes: 75 depends_on: diff --git a/.buildkite/test_areas/model_executor.yaml b/.buildkite/test_areas/model_executor.yaml index 30ca2f038ec3..1f511590ffb3 100644 --- a/.buildkite/test_areas/model_executor.yaml +++ b/.buildkite/test_areas/model_executor.yaml @@ -25,7 +25,7 @@ steps: - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread mirror: amd: - native_ci: true + dind: false device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index 654648d43502..83d2fa26c22c 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -15,7 +15,7 @@ steps: - pytest -v -s models/language -m 'core_model and (not slow_test)' mirror: amd: - native_ci: true + dind: false device: mi300_1 depends_on: - image-build-amd @@ -36,7 +36,7 @@ steps: parallelism: 2 mirror: amd: - native_ci: true + dind: false device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index 18c57f08966d..32a27a56fde0 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -107,7 +107,7 @@ steps: - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1 mirror: amd: - native_ci: true + dind: false device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/pytorch.yaml b/.buildkite/test_areas/pytorch.yaml index 6915eb962a40..cdc5a1fd0442 100644 --- a/.buildkite/test_areas/pytorch.yaml +++ b/.buildkite/test_areas/pytorch.yaml @@ -109,7 +109,7 @@ steps: - pytest -s -v compile/passes --ignore compile/passes/distributed mirror: amd: - native_ci: true + dind: false device: mi300_1 timeout_in_minutes: 65 depends_on: @@ -198,7 +198,7 @@ steps: - bash standalone_tests/pytorch_nightly_dependency.sh mirror: amd: - native_ci: true + dind: false device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/weight_loading.yaml b/.buildkite/test_areas/weight_loading.yaml index 0538902d50d0..46a874178b5a 100644 --- a/.buildkite/test_areas/weight_loading.yaml +++ b/.buildkite/test_areas/weight_loading.yaml @@ -15,7 +15,7 @@ steps: - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt mirror: amd: - native_ci: true + dind: false device: mi300_2 depends_on: - image-build-amd From 67a1bd0fc0ccfdbb62cda12273802244b33326a0 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 14 Jul 2026 12:00:46 -0500 Subject: [PATCH 15/16] [CI][ROCm] Clean up native test configuration Signed-off-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 18 ++---------------- .buildkite/test_areas/expert_parallelism.yaml | 11 ----------- 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index b879760aa293..ce6ee3857bda 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -476,18 +476,6 @@ steps: fast_check: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - - .buildkite/test-amd.yaml - - .buildkite/hardware_tests/amd.yaml - - .buildkite/scripts/ci-bake-rocm.sh - - .buildkite/scripts/hardware_ci/run-amd-test.sh - - .buildkite/scripts/rocm/build-ci-base.sh - - .buildkite/scripts/rocm/build-test-image.sh - - docker/Dockerfile.rocm - - docker/Dockerfile.rocm_base - - docker/ci-rocm.hcl - - docker/docker-bake-rocm.hcl - - requirements/rocm.txt - - requirements/test/rocm.txt - vllm/ - tests/basic_correctness/test_basic_correctness - tests/basic_correctness/test_cpu_offload @@ -1333,7 +1321,6 @@ steps: - tests/evals/ commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - export PYTORCH_ROCM_ARCH=gfx942 - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt - label: ROCm LM Eval Large Models (8 GPUs) # TBD @@ -1581,9 +1568,8 @@ steps: commands: - apt-get update && apt-get install -y curl libsodium23 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - export PYTHONFAULTHANDLER=1 - - pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread - - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread + - pytest -v -s model_executor -m '(not slow_test)' + - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py #---------------------------------------------------- mi300 · model_runner_v2 -------------------------------------------------------# diff --git a/.buildkite/test_areas/expert_parallelism.yaml b/.buildkite/test_areas/expert_parallelism.yaml index 113ccdc3d1a3..d2278408facc 100644 --- a/.buildkite/test_areas/expert_parallelism.yaml +++ b/.buildkite/test_areas/expert_parallelism.yaml @@ -21,17 +21,6 @@ steps: depends_on: - image-build-amd source_file_dependencies: - - .buildkite/hardware_tests/amd.yaml - - .buildkite/scripts/ci-bake-rocm.sh - - .buildkite/scripts/hardware_ci/run-amd-test.sh - - .buildkite/scripts/rocm/build-ci-base.sh - - .buildkite/scripts/rocm/build-test-image.sh - - docker/Dockerfile.rocm - - docker/Dockerfile.rocm_base - - docker/ci-rocm.hcl - - docker/docker-bake-rocm.hcl - - requirements/rocm.txt - - requirements/test/rocm.txt - vllm/distributed/eplb - tests/distributed/test_eplb_algo.py - tests/distributed/test_eplb_utils.py From aa7c1342fbbfbf7888ec3845a744fd81e42fb304 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 14 Jul 2026 14:47:51 -0500 Subject: [PATCH 16/16] [CI][ROCm] Split multimodal CUDA graph fix Signed-off-by: Andreas Karatzas --- vllm/v1/worker/gpu/model_states/default.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/vllm/v1/worker/gpu/model_states/default.py b/vllm/v1/worker/gpu/model_states/default.py index 14175cc577fe..854b71b69fcb 100644 --- a/vllm/v1/worker/gpu/model_states/default.py +++ b/vllm/v1/worker/gpu/model_states/default.py @@ -125,8 +125,6 @@ def prepare_dummy_inputs(self, num_reqs: int, num_tokens: int) -> dict[str, Any] if self.supports_mm_inputs: inputs_embeds = self.encoder_runner.inputs_embeds[:num_tokens] model_inputs["inputs_embeds"] = inputs_embeds - if not self.model.requires_raw_input_tokens: - model_inputs["input_ids"] = None if self.rope_state is not None: model_inputs["positions"] = self.rope_state.get_positions(num_tokens) return model_inputs