diff --git a/.buildkite/ci_config.yaml b/.buildkite/ci_config.yaml index a60a4194e9b3..21ffa1b9b8d7 100644 --- a/.buildkite/ci_config.yaml +++ b/.buildkite/ci_config.yaml @@ -8,6 +8,7 @@ run_all_patterns: - "CMakeLists.txt" - "requirements/common.txt" - "requirements/cuda.txt" + - "requirements/kv_connectors.txt" - "requirements/build/cuda.txt" - "requirements/test/cuda.txt" - "setup.py" diff --git a/.buildkite/ci_config_rocm.yaml b/.buildkite/ci_config_rocm.yaml new file mode 100644 index 000000000000..23f323400719 --- /dev/null +++ b/.buildkite/ci_config_rocm.yaml @@ -0,0 +1,23 @@ +name: vllm_rocm_ci +job_dirs: + - ".buildkite/hardware_tests" +run_all_patterns: + - "docker/Dockerfile.rocm" + - "docker/Dockerfile.rocm_base" + - "docker/ci-rocm.hcl" + - "docker/docker-bake-rocm.hcl" + - ".buildkite/hardware_tests/amd.yaml" + - ".buildkite/scripts/ci-bake-rocm.sh" + - ".buildkite/scripts/hardware_ci/run-amd-test.py" + - ".buildkite/scripts/hardware_ci/run-amd-test.sh" + - "CMakeLists.txt" + - "requirements/common.txt" + - "requirements/rocm.txt" + - "requirements/build/rocm.txt" + - "requirements/test/rocm.txt" + - "setup.py" + - "csrc/" + - "cmake/" +run_all_exclude_patterns: + - "csrc/cpu/" + - "cmake/cpu_extension.cmake" diff --git a/.buildkite/hardware_tests/amd.yaml b/.buildkite/hardware_tests/amd.yaml index 0c514647dc2b..c2510f38aabb 100644 --- a/.buildkite/hardware_tests/amd.yaml +++ b/.buildkite/hardware_tests/amd.yaml @@ -1,22 +1,73 @@ -group: Hardware - AMD Build +group: Hardware - AMD Build steps: - - label: "AMD: :docker: build image" - key: image-build-amd + # Ensure ci_base is up-to-date before building the test image. + # Compares a content hash of ci_base-affecting files against the remote + # image label. If hashes match the build is skipped (< 30 s); if they + # differ ci_base is rebuilt and pushed automatically. + - label: "AMD: :docker: ensure ci_base" + key: ensure-ci-base-amd depends_on: [] device: amd_cpu no_plugin: true commands: - - > - docker build - --build-arg max_jobs=16 - --build-arg REMOTE_VLLM=1 - --build-arg ARG_PYTORCH_ROCM_ARCH='gfx90a;gfx942;gfx950' - --build-arg VLLM_BRANCH=$BUILDKITE_COMMIT - --tag "rocm/vllm-ci:${BUILDKITE_COMMIT}" - -f docker/Dockerfile.rocm - --target test - --no-cache - --progress plain . - - docker push "rocm/vllm-ci:${BUILDKITE_COMMIT}" + - bash .buildkite/scripts/ci-bake-rocm.sh ci-base-rocm-ci-with-deps + env: + DOCKER_BUILDKIT: "1" + VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl" + PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950" + REMOTE_VLLM: "1" + VLLM_BRANCH: "$BUILDKITE_COMMIT" + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 1 + - exit_status: -10 # Agent was lost + limit: 1 + + - label: "AMD: :docker: build test image and artifacts" + key: image-build-amd + depends_on: + - ensure-ci-base-amd + device: amd_cpu + no_plugin: true + commands: + - | + if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then + echo "ROCM_CI_ARTIFACT_ONLY=1; building ROCm wheel artifact only" + IMAGE_TAG="" bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-artifacts + else + bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-wheel + fi + - | + docker run --rm --network=none --entrypoint /bin/bash "rocm/vllm-ci:${BUILDKITE_COMMIT}" -ec ' + if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi + if [ ! -d /vllm-workspace/tests ]; then echo Missing directory: /vllm-workspace/tests >&2; exit 1; fi + if [ ! -d /vllm-workspace/src/vllm ]; then echo Missing directory: /vllm-workspace/src/vllm >&2; exit 1; fi + if [ ! -x /vllm-workspace/src/vllm/vllm-rs ]; then echo Missing executable: /vllm-workspace/src/vllm/vllm-rs >&2; exit 1; fi + command -v python3 + command -v uv + command -v pytest + if ! command -v amd-smi >/dev/null 2>&1 && ! command -v rocminfo >/dev/null 2>&1; then + echo No ROCm CLI found in image >&2 + exit 1 + fi + python3 - </dev/null | grep -qx root; then + echo FAIL: GID 0 not root group + exit 1 + fi + touch /home/vllm/.smoke && rm /home/vllm/.smoke + touch /opt/uv/cache/.smoke && rm /opt/uv/cache/.smoke + if ! test -x /usr/local/bin/vllm-nonroot-entrypoint.sh; then + echo FAIL: wrapper missing + exit 1 + fi + if ! test -w /etc/passwd; then + echo FAIL: /etc/passwd not group-writable + exit 1 + fi + if ! test -w /etc/group; then + echo FAIL: /etc/group not group-writable + exit 1 + fi + echo non-root invariants OK + ' + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - exit_status: -10 # Agent was lost + limit: 2 + - label: ":docker: Build CPU image" key: image-build-cpu depends_on: [] @@ -56,3 +110,21 @@ steps: limit: 2 - exit_status: -10 # Agent was lost limit: 2 + + - label: ":docker: Build arm64 image" + key: arm64-image-build + depends_on: [] + source_file_dependencies: + - ".buildkite/image_build/image_build.yaml" + - ".buildkite/image_build/image_build_arm64.sh" + - "docker/Dockerfile" + commands: + - .buildkite/image_build/image_build_arm64.sh $REGISTRY $REPO $BUILDKITE_COMMIT + env: + DOCKER_BUILDKIT: "1" + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - exit_status: -10 # Agent was lost + limit: 2 diff --git a/.buildkite/image_build/image_build_arm64.sh b/.buildkite/image_build/image_build_arm64.sh new file mode 100755 index 000000000000..5baa55a19659 --- /dev/null +++ b/.buildkite/image_build/image_build_arm64.sh @@ -0,0 +1,37 @@ +#!/bin/bash +set -e + +if [[ $# -lt 3 ]]; then + echo "Usage: $0 " + exit 1 +fi + +REGISTRY=$1 +REPO=$2 +BUILDKITE_COMMIT=$3 + +# authenticate with AWS ECR +aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true + +# skip build if image already exists +if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64) ]]; then + echo "Image not found, proceeding with build..." +else + echo "Image found" + exit 0 +fi + +# build (Grace/GH200 is the arm64 GPU target; sm_90) +docker build --file docker/Dockerfile \ + --platform linux/arm64 \ + --build-arg max_jobs=16 \ + --build-arg nvcc_threads=4 \ + --build-arg torch_cuda_arch_list="9.0" \ + --build-arg USE_SCCACHE=1 \ + --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ + --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64 \ + --target test \ + --progress plain . + +# push +docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64 diff --git a/.buildkite/image_build/image_build_hpu.sh b/.buildkite/image_build/image_build_hpu.sh index 60fa1789fa06..df900dc60342 100755 --- a/.buildkite/image_build/image_build_hpu.sh +++ b/.buildkite/image_build/image_build_hpu.sh @@ -11,7 +11,7 @@ REPO=$2 BUILDKITE_COMMIT=$3 # authenticate with AWS ECR -aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" +aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true # skip build if image already exists if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu) ]]; then diff --git a/.buildkite/image_build/image_build_xpu.sh b/.buildkite/image_build/image_build_xpu.sh index c3734dce13ca..45417b7339be 100755 --- a/.buildkite/image_build/image_build_xpu.sh +++ b/.buildkite/image_build/image_build_xpu.sh @@ -11,8 +11,8 @@ REPO=$2 BUILDKITE_COMMIT=$3 # authenticate with AWS ECR -aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" -aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com +aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true +aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true # skip build if image already exists if ! docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-xpu &> /dev/null; then diff --git a/.buildkite/intel_jobs/basic_correctness.yaml b/.buildkite/intel_jobs/basic_correctness.yaml new file mode 100644 index 000000000000..1b67454d2afe --- /dev/null +++ b/.buildkite/intel_jobs/basic_correctness.yaml @@ -0,0 +1,22 @@ +group: Basic Correctness +depends_on: + - image-build-xpu +steps: +- label: XPU Sleep Mode + timeout_in_minutes: 30 + device: intel_gpu + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/basic_correctness/test_cumem.py + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && + pytest -v -s basic_correctness/test_mem.py::test_end_to_end' diff --git a/.buildkite/intel_jobs/expert_parallelism_intel.yaml b/.buildkite/intel_jobs/expert_parallelism_intel.yaml new file mode 100644 index 000000000000..953e9ddcc55e --- /dev/null +++ b/.buildkite/intel_jobs/expert_parallelism_intel.yaml @@ -0,0 +1,23 @@ +group: Expert Parallelism +depends_on: + - image-build-xpu +steps: +- label: EPLB Algorithm + key: eplb-algorithm + timeout_in_minutes: 45 + device: intel_gpu + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/distributed/eplb + - tests/distributed/test_eplb_algo.py + - tests/distributed/test_eplb_utils.py + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + pytest -v -s distributed/test_eplb_algo.py' diff --git a/.buildkite/intel_jobs/lora_intel.yaml b/.buildkite/intel_jobs/lora_intel.yaml index 366d9daa24dd..32a56ef59b3f 100644 --- a/.buildkite/intel_jobs/lora_intel.yaml +++ b/.buildkite/intel_jobs/lora_intel.yaml @@ -18,17 +18,18 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && pytest -v -s lora/test_layers.py && pytest -v -s lora/test_lora_checkpoints.py && - (pytest -v -s lora/test_lora_functions.py --deselect="tests/lora/test_lora_functions.py::test_lora_functions_sync" --deselect="tests/lora/test_lora_functions.py::test_lora_functions_async" || true) && + pytest -v -s lora/test_lora_functions.py && pytest -v -s lora/test_lora_huggingface.py && pytest -v -s lora/test_lora_manager.py && pytest -v -s lora/test_lora_utils.py && pytest -v -s lora/test_peft_helper.py && pytest -v -s lora/test_resolver.py && pytest -v -s lora/test_utils.py && - (pytest -v -s lora/test_add_lora.py --deselect="tests/lora/test_add_lora.py::test_add_lora" || true) && - (pytest -v -s lora/test_worker.py --deselect="tests/lora/test_worker.py::test_worker_apply_lora" || true)' + pytest -v -s lora/test_add_lora.py && + pytest -v -s lora/test_worker.py' - label: LoRA Fused/MoE Kernels timeout_in_minutes: 45 @@ -46,8 +47,9 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && pytest -v -s lora/test_fused_moe_lora_kernel.py && - pytest -v -s lora/test_moe_lora_align_sum.py' + pytest -v -s lora/test_moe_lora_align_sum.py --deselect="tests/lora/test_moe_lora_align_sum.py::test_moe_lora_align_block_size_mixed_base_and_lora[1]"' - label: LoRA Punica Kernels timeout_in_minutes: 45 @@ -65,8 +67,9 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && set -o pipefail && - pytest -v -s lora/test_punica_ops.py --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-2-2049-64-32-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype1-2-64000-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-1-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-1-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-8-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype0-3-2049-128-8-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-8-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype1-1-2049-256-128-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-64256-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-2-29696-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-3-49408-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-2-16384-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-2-51328-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype1-1-102656-32-4-4]"' + pytest -v -s lora/test_punica_ops.py --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-3-43264-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype1-1-2049-64-128-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-1-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-1-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-8-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype0-3-2049-128-8-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-8-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype1-1-2049-256-128-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-64256-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-2-29696-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-3-49408-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-2-16384-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-2-51328-32-4-4]"' - label: LoRA Punica FP8/XPU Ops timeout_in_minutes: 45 @@ -84,6 +87,7 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && pytest -v -s lora/test_punica_ops_fp8.py && pytest -v -s lora/test_punica_xpu_ops.py' @@ -103,10 +107,12 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && (pytest -v -s lora/test_mixtral.py --deselect="tests/lora/test_mixtral.py::test_mixtral_lora[4]" || true) && pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" && - pytest -v -s lora/test_qwen35_densemodel_lora.py && - pytest -v -s lora/test_transformers_model.py' + pytest -v -s lora/test_transformers_model.py && + pytest -v -s lora/test_chatglm3_tp.py && + pytest -s -v lora/test_minicpmv_tp.py' - label: LoRA Multimodal timeout_in_minutes: 45 @@ -124,6 +130,6 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && pytest -v -s lora/test_default_mm_loras.py && - (pytest -v -s lora/test_qwen3_unembed.py || true) && pytest -v -s lora/test_whisper.py' diff --git a/.buildkite/intel_jobs/misc_intel.yaml b/.buildkite/intel_jobs/misc_intel.yaml index 39634ccccacf..20a9f5bd1734 100644 --- a/.buildkite/intel_jobs/misc_intel.yaml +++ b/.buildkite/intel_jobs/misc_intel.yaml @@ -38,7 +38,17 @@ steps: REPO: "vllm-ci-test-repo" VLLM_TEST_DEVICE: "xpu" source_file_dependencies: - - vllm/ + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/logger.py + - vllm/model_executor/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ - tests/v1/sample - tests/v1/logits_processors - tests/v1/test_oracle.py @@ -49,7 +59,129 @@ steps: bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'export VLLM_WORKER_MULTIPROC_METHOD=spawn && cd tests && - pytest -v -s v1/logits_processors && + pytest -v -s v1/logits_processors --ignore=v1/logits_processors/test_custom_online.py --ignore=v1/logits_processors/test_custom_offline.py && pytest -v -s v1/test_oracle.py && pytest -v -s v1/test_request.py && - pytest -v -s v1/test_outputs.py' + pytest -v -s v1/test_outputs.py && + pytest -v -s v1/sample/test_topk_topp_sampler.py' + +- label: XPU CPU Offload + timeout_in_minutes: 60 + device: intel_gpu + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - vllm/v1/kv_offload/ + - vllm/v1/kv_connector/ + - tests/v1/kv_offload/ + - tests/v1/kv_connector/unit/test_offloading_connector.py + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'export VLLM_WORKER_MULTIPROC_METHOD=spawn && + cd tests && + pytest -v -s v1/kv_offload && + pytest -v -s v1/kv_connector/unit/test_offloading_connector.py' + +- label: Regression + key: regression + timeout_in_minutes: 30 + device: intel_gpu + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/test_regression + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'pip install modelscope && + cd tests && + pytest -v -s test_regression.py' + +- label: Metrics, Tracing (2 GPUs) + key: metrics-tracing-2-gpus + timeout_in_minutes: 30 + num_devices: 2 + device: intel_gpu + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/tracing/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/v1/tracing + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'pip install opentelemetry-sdk\>=1.26.0 opentelemetry-api\>=1.26.0 opentelemetry-exporter-otlp\>=1.26.0 opentelemetry-semantic-conventions-ai\>=0.4.1 && + cd tests && + pytest -v -s v1/tracing' + +- label: Async Engine, Inputs, Utils, Worker + key: async-engine-inputs-utils-worker + timeout_in_minutes: 30 + device: intel_gpu + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/assets/ + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/detokenizer + - tests/multimodal + - tests/utils_ + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + pip install av && + pytest -v -s detokenizer && + pytest -v -s -m "not cpu_test" ./multimodal && + pytest -v -s utils_ --ignore=utils_/test_mem_utils.py' diff --git a/.buildkite/intel_jobs/models_multimodal_intel.yaml b/.buildkite/intel_jobs/models_multimodal_intel.yaml new file mode 100644 index 000000000000..cf5b51c4b89f --- /dev/null +++ b/.buildkite/intel_jobs/models_multimodal_intel.yaml @@ -0,0 +1,111 @@ +group: Models - Multimodal +depends_on: + - image-build-xpu +steps: +- label: "Multi-Modal Models (Standard) 1: qwen2" + key: multi-modal-models-standard-1-qwen2 + timeout_in_minutes: 45 + device: intel_gpu + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/models/multimodal + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git && + cd tests && + pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" && + pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model' + +- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma" + key: multi-modal-models-standard-2-qwen3-gemma + timeout_in_minutes: 45 + device: intel_gpu + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/models/multimodal + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git && + cd tests && + pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model' + +- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl" + key: multi-modal-models-standard-3-llava-qwen2-vl + timeout_in_minutes: 45 + device: intel_gpu + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/models/multimodal + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git && + cd tests && + pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" && + pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model' + +- label: "Multi-Modal Models (Standard) 4: other + whisper" + key: multi-modal-models-standard-4-other-whisper + timeout_in_minutes: 45 + device: intel_gpu + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/models/multimodal + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git && + cd tests && + pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing' + +- label: Multi-Modal Processor # 44min + key: multi-modal-processor + timeout_in_minutes: 45 + device: intel_gpu + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/models/multimodal + - tests/models/registry.py + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'pip install av matplotlib ftfy git+https://github.com/TIGER-AI-Lab/Mantis.git && + pip install open-clip-torch --no-deps && + cd tests && + pytest -v -s models/multimodal/processing/test_tensor_schema.py + --deselect "tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4]" + --deselect "tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[Qwen/Qwen2.5-Omni-7B-AWQ]" + --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB' + parallelism: 4 diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index c59be699502f..4bf14f7064be 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -36,9 +36,14 @@ steps: python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN && python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 && python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 && + python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 && python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 && python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 && - python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel' + python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel && + python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192 && + VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192 && + python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC --enforce-eager -tp 2 --max-model-len 8192 + ' - label: "XPU V1 test" depends_on: - image-build-xpu @@ -61,5 +66,24 @@ steps: pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py && pytest -v -s v1/structured_output && pytest -v -s v1/test_serial_utils.py && - pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_tree_attention.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py && - pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py' + pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py && + pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py' + - label: "XPU server test" + depends_on: + - image-build-xpu + timeout_in_minutes: 30 + device: intel_gpu + no_plugin: true + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + source_file_dependencies: + - vllm/ + - .buildkite/intel_jobs/test-intel.yaml + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'pip install av && + cd tests && + pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py && + pytest -v -s benchmarks/test_serve_cli.py' diff --git a/.buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh b/.buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh index 518af9a66018..b495c0d123a6 100755 --- a/.buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh +++ b/.buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh @@ -2,7 +2,7 @@ # We can use this script to compute baseline accuracy on chartqa for vllm. # # Make sure you have lm-eval-harness installed: -# pip install "lm-eval[api]>=0.4.11" +# pip install "lm-eval[api]>=0.4.12" usage() { echo`` diff --git a/.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh b/.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh index f010ffe6752d..e430e6183b2d 100755 --- a/.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh +++ b/.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh @@ -2,7 +2,7 @@ # We can use this script to compute baseline accuracy on GSM for transformers. # # Make sure you have lm-eval-harness installed: -# pip install "lm-eval[api]>=0.4.11" +# pip install "lm-eval[api]>=0.4.12" usage() { echo`` diff --git a/.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh b/.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh index fec4a94e63e4..f1a541ddbefc 100644 --- a/.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh +++ b/.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh @@ -3,7 +3,7 @@ # We use this for fp8, which HF does not support. # # Make sure you have lm-eval-harness installed: -# pip install "lm-eval[api]>=0.4.11" +# pip install "lm-eval[api]>=0.4.12" usage() { echo`` diff --git a/.buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh b/.buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh index e3c6e16bd6b3..ba8da9fc3f55 100644 --- a/.buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh +++ b/.buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh @@ -3,7 +3,7 @@ # We use this for fp8, which HF does not support. # # Make sure you have lm-eval-harness installed: -# pip install "lm-eval[api]>=0.4.11" +# pip install "lm-eval[api]>=0.4.12" usage() { echo`` diff --git a/.buildkite/performance-benchmarks/tests/latency-tests-arm64-cpu.json b/.buildkite/performance-benchmarks/tests/latency-tests-arm64-cpu.json index fba695041e3e..98811049d395 100644 --- a/.buildkite/performance-benchmarks/tests/latency-tests-arm64-cpu.json +++ b/.buildkite/performance-benchmarks/tests/latency-tests-arm64-cpu.json @@ -2,7 +2,6 @@ { "test_name": "latency_llama8B_tp1", "environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120, "VLLM_CPU_KVCACHE_SPACE": 40 diff --git a/.buildkite/performance-benchmarks/tests/latency-tests-cpu.json b/.buildkite/performance-benchmarks/tests/latency-tests-cpu.json index 77d1694ec864..5f048df5f6ab 100644 --- a/.buildkite/performance-benchmarks/tests/latency-tests-cpu.json +++ b/.buildkite/performance-benchmarks/tests/latency-tests-cpu.json @@ -2,7 +2,6 @@ { "test_name": "latency_llama8B_tp2", "environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120, "VLLM_CPU_SGL_KERNEL": 1, diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-arm64-cpu.json b/.buildkite/performance-benchmarks/tests/serving-tests-arm64-cpu.json index 9f226ef2f819..75b80b2c2125 100644 --- a/.buildkite/performance-benchmarks/tests/serving-tests-arm64-cpu.json +++ b/.buildkite/performance-benchmarks/tests/serving-tests-arm64-cpu.json @@ -13,7 +13,6 @@ 200 ], "server_environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120, "VLLM_CPU_SGL_KERNEL": 1, diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-asr.json b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-asr.json index 30879b5e9dc5..16e8b0600ac8 100644 --- a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-asr.json +++ b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-asr.json @@ -5,7 +5,6 @@ ], "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200], "server_environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120 }, "server_parameters": { diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-embed.json b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-embed.json index 6d3455c478ca..c62f244fc763 100644 --- a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-embed.json +++ b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-embed.json @@ -9,7 +9,6 @@ 128 ], "server_environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120, "VLLM_CPU_SGL_KERNEL": 1, diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json index 6c4591f05b3b..9aa76c110898 100644 --- a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json +++ b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json @@ -5,7 +5,6 @@ ], "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200], "server_environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120, "VLLM_CPU_SGL_KERNEL": 1, @@ -31,30 +30,9 @@ } }, "tests": [ - { - "test_name": "serving_llama8B_tp1_sharegpt", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json" - } - }, - { - "test_name": "serving_llama8B_tp2_sharegpt", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json" - } - }, { "test_name": "serving_llama8B_tp1_random_128_128", "server_parameters": { - "tensor_parallel_size": 1 }, "client_parameters": { "dataset_name": "random", @@ -63,290 +41,244 @@ } }, { - "test_name": "serving_llama8B_tp2_random_128_128", + "test_name": "serving_llama8B_int4_tp1_random_128_128", "server_parameters": { - "tensor_parallel_size": 2 + "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4" }, "client_parameters": { + "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_tp4_random_128_128", + "test_name": "serving_llama8B_int8_tp1_random_128_128", "server_parameters": { - "tensor_parallel_size": 4 + "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8" }, "client_parameters": { + "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_tp1_random_128_2048", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp2_random_128_2048", + "test_name": "serving_llama1B_tp1_random_128_128", "server_parameters": { - "tensor_parallel_size": 2 + "model": "meta-llama/Llama-3.2-1B" }, "client_parameters": { + "model": "meta-llama/Llama-3.2-1B", "dataset_name": "random", "random-input-len": 128, - "random-output-len": 2048 + "random-output-len": 128 } }, { - "test_name": "serving_llama8B_tp4_random_128_2048", + "test_name": "serving_llama3B_tp1_random_128_128", "server_parameters": { - "tensor_parallel_size": 4 + "model": "meta-llama/Llama-3.2-3B-Instruct" }, "client_parameters": { + "model": "meta-llama/Llama-3.2-3B-Instruct", "dataset_name": "random", "random-input-len": 128, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp1_random_2048_128", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_tp2_random_2048_128", + "test_name": "serving_llama70B_tp1_random_128_128", "server_parameters": { - "tensor_parallel_size": 2 + "model": "meta-llama/Llama-3.3-70B-Instruct" }, "client_parameters": { + "model": "meta-llama/Llama-3.3-70B-Instruct", "dataset_name": "random", - "random-input-len": 2048, + "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_tp4_random_2048_128", + "test_name": "serving_granite2B_tp1_random_128_128", "server_parameters": { - "tensor_parallel_size": 4 + "model": "ibm-granite/granite-3.2-2b-instruct" }, "client_parameters": { + "model": "ibm-granite/granite-3.2-2b-instruct", "dataset_name": "random", - "random-input-len": 2048, + "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_tp1_random_2048_2048", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp2_random_2048_2048", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp4_random_2048_2048", - "server_parameters": { - "tensor_parallel_size": 4 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_int4_tp1_random_128_128", + "test_name": "serving_qwen1.7B_tp1_random_128_128", "server_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", - "tensor_parallel_size": 1 + "model": "Qwen/Qwen3-1.7B" }, "client_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", + "model": "Qwen/Qwen3-1.7B", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_int4_tp2_random_128_128", + "test_name": "serving_qwen4B_tp1_random_128_128", "server_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", - "tensor_parallel_size": 2 + "model": "Qwen/Qwen3-4B" }, "client_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", + "model": "Qwen/Qwen3-4B", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_int4_tp4_random_128_128", + "test_name": "serving_qwen8B_tp1_random_128_128", "server_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", - "tensor_parallel_size": 4 + "model": "Qwen/Qwen3-8B" }, "client_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", + "model": "Qwen/Qwen3-8B", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_int8_tp1_random_128_128", + "test_name": "serving_qwen14B_tp1_random_128_128", "server_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", - "tensor_parallel_size": 1 + "model": "Qwen/Qwen3-14B" }, "client_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", + "model": "Qwen/Qwen3-14B", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_int8_tp2_random_128_128", + "test_name": "serving_qwen30B_tp1_random_128_128", "server_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", - "tensor_parallel_size": 2 + "model": "Qwen/Qwen3-30B-A3B" }, "client_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", + "model": "Qwen/Qwen3-30B-A3B", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_int8_tp4_random_128_128", + "test_name": "serving_glm9B_tp1_random_128_128", "server_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", - "tensor_parallel_size": 4 + "model": "zai-org/glm-4-9b-hf" }, "client_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", + "model": "zai-org/glm-4-9b-hf", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama3B_tp1_random_128_128", + "test_name": "serving_gemma7B_tp1_random_128_128", "server_parameters": { - "model": "meta-llama/Llama-3.2-3B-Instruct", - "tensor_parallel_size": 1 + "model": "google/gemma-7b" }, "client_parameters": { - "model": "meta-llama/Llama-3.2-3B-Instruct", + "model": "google/gemma-7b", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_granite2B_tp1_random_128_128", + "test_name": "serving_gemma3-4b_tp1_random_128_128", + "server_environment_variables": { + "VLLM_CPU_SGL_KERNEL": 0 + }, "server_parameters": { - "model": "ibm-granite/granite-3.2-2b-instruct", - "tensor_parallel_size": 1 + "model": "google/gemma-3-4b-it" }, "client_parameters": { - "model": "ibm-granite/granite-3.2-2b-instruct", + "model": "google/gemma-3-4b-it", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_qwen1.7B_tp1_random_128_128", + "test_name": "serving_gemma3-12b_tp1_random_128_128", + "server_environment_variables": { + "VLLM_CPU_SGL_KERNEL": 0 + }, "server_parameters": { - "model": "Qwen/Qwen3-1.7B", - "tensor_parallel_size": 1 + "model": "google/gemma-3-12b-it" }, "client_parameters": { - "model": "Qwen/Qwen3-1.7B", + "model": "google/gemma-3-12b-it", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_qwen4B_tp1_random_128_128", + "test_name": "serving_gemma4-4b_tp1_random_128_128", + "server_environment_variables": { + "VLLM_CPU_SGL_KERNEL": 0 + }, "server_parameters": { - "model": "Qwen/Qwen3-4B", - "tensor_parallel_size": 1 + "model": "google/gemma-4-E4B-it" }, "client_parameters": { - "model": "Qwen/Qwen3-4B", + "model": "google/gemma-4-E4B-it", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_qwen8B_tp1_random_128_128", + "test_name": "serving_gemma4-2b_tp1_random_128_128", + "server_environment_variables": { + "VLLM_CPU_SGL_KERNEL": 0 + }, "server_parameters": { - "model": "Qwen/Qwen3-8B", - "tensor_parallel_size": 1 + "model": "google/gemma-4-E2B-it" }, "client_parameters": { - "model": "Qwen/Qwen3-8B", + "model": "google/gemma-4-E2B-it", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_glm9B_tp1_random_128_128", + "test_name": "serving_gemma4-26b_tp1_random_128_128", + "server_environment_variables": { + "VLLM_CPU_SGL_KERNEL": 0, + "VLLM_CPU_ATTN_SPLIT_KV": 0 + }, "server_parameters": { - "model": "zai-org/glm-4-9b-hf", - "tensor_parallel_size": 1 + "model": "google/gemma-4-26B-A4B-it" }, "client_parameters": { - "model": "zai-org/glm-4-9b-hf", + "model": "google/gemma-4-26B-A4B-it", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_gemma7B_tp1_random_128_128", + "test_name": "serving_phi4_tp1_random_128_128", "server_parameters": { - "model": "google/gemma-7b", - "tensor_parallel_size": 1 + "model": "microsoft/Phi-4-reasoning" }, "client_parameters": { - "model": "google/gemma-7b", + "model": "microsoft/Phi-4-reasoning", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-cpu.json b/.buildkite/performance-benchmarks/tests/serving-tests-cpu.json index c2d7768e2026..0b7e7499965a 100644 --- a/.buildkite/performance-benchmarks/tests/serving-tests-cpu.json +++ b/.buildkite/performance-benchmarks/tests/serving-tests-cpu.json @@ -5,7 +5,6 @@ ], "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200], "server_environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120, "VLLM_CPU_SGL_KERNEL": 1, diff --git a/.buildkite/performance-benchmarks/tests/serving-tests.json b/.buildkite/performance-benchmarks/tests/serving-tests.json index 36e3d4170a1b..2cbd472295e7 100644 --- a/.buildkite/performance-benchmarks/tests/serving-tests.json +++ b/.buildkite/performance-benchmarks/tests/serving-tests.json @@ -1,77 +1,112 @@ -[ +{ + "defaults": { + "qps_list": [ + "inf" + ], + "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200], + "server_parameters": { + "model": "meta-llama/Llama-3.1-8B-Instruct", + "tensor_parallel_size": 1, + "dtype": "bfloat16" + }, + "client_parameters": { + "model": "meta-llama/Llama-3.1-8B-Instruct", + "backend": "vllm", + "ignore-eos": "", + "temperature": 0, + "num_prompts": 200 + } + }, + "tests": [ + { + "test_name": "serving_llama8B_tp1_sharegpt", + "server_parameters": { + "tensor_parallel_size": 1 + }, + "client_parameters": { + "dataset_name": "sharegpt", + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json" + } + }, + { + "dataset_name": "sharegpt", + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json" + } + }, + { + "test_name": "serving_llama8B_tp1_random_128_128", + "server_parameters": { + "tensor_parallel_size": 1 + }, + "client_parameters": { + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 128 + } + }, + { + "test_name": "serving_llama8B_tp1_random_128_2048", + "server_parameters": { + "tensor_parallel_size": 1 + }, + "client_parameters": { + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 2048 + } + }, { - "test_name": "serving_llama8B_tp1_sharegpt", - "qps_list": [1, 4, 16, "inf"], - "server_parameters": { - "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", - "tensor_parallel_size": 1, - "disable_log_stats": "", - "load_format": "dummy" - }, - "client_parameters": { - "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", - "backend": "vllm", - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "temperature": 0, - "num_prompts": 200 - } + "test_name": "serving_llama8B_tp1_random_2048_128", + "server_parameters": { + "tensor_parallel_size": 1 + }, + "client_parameters": { + "dataset_name": "random", + "random-input-len": 2048, + "random-output-len": 128 + } }, { - "test_name": "serving_llama70B_tp4_sharegpt", - "qps_list": [1, 4, 16, "inf"], - "server_parameters": { - "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "tensor_parallel_size": 4, - "disable_log_stats": "", - "load_format": "dummy" - }, - "client_parameters": { - "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "backend": "vllm", - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "temperature": 0, - "num_prompts": 200 - } + "test_name": "serving_llama8B_tp1_random_2048_2048", + "server_parameters": { + "tensor_parallel_size": 1 + }, + "client_parameters": { + "dataset_name": "random", + "random-input-len": 2048, + "random-output-len": 2048 + } }, { - "test_name": "serving_mixtral8x7B_tp2_sharegpt", - "qps_list": [1, 4, 16, "inf"], - "server_parameters": { - "model": "mistralai/Mixtral-8x7B-Instruct-v0.1", - "tensor_parallel_size": 2, - "disable_log_stats": "", - "load_format": "dummy" - }, - "client_parameters": { - "model": "mistralai/Mixtral-8x7B-Instruct-v0.1", - "backend": "vllm", - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "temperature": 0, - "num_prompts": 200 - } + "test_name": "serving_llama70B_tp4_random_128_128", + "server_parameters": { + "model": "meta-llama/Llama-3.3-70B-Instruct", + "async_scheduling": "", + "no_enable_prefix_caching": "", + "max_num_batched_tokens": 8192 + }, + "client_parameters": { + "model": "meta-llama/Llama-3.3-70B-Instruct", + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 128 + } }, { - "test_name": "serving_llama70B_tp4_sharegpt_specdecode", - "qps_list": [2], - "server_parameters": { - "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "tensor_parallel_size": 4, - "speculative_config": { - "model": "turboderp/Qwama-0.5B-Instruct", - "num_speculative_tokens": 4, - "draft_tensor_parallel_size": 1 - } - }, - "client_parameters": { - "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "backend": "vllm", - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "temperature": 0, - "num_prompts": 200 - } + "test_name": "serving_gemma4-e4b_tp1_random_128_128", + "server_parameters": { + "model": "google/gemma-4-E4B-it", + "enable_auto_tool_choice": "", + "tool_call_parser": "gemma4", + "chat_template": "examples/tool_chat_template_gemma4.jinja", + "reasoning_parser": "gemma4" + }, + "client_parameters": { + "model": "google/gemma-4-E4B-it", + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 128 + } } -] + ] +} diff --git a/.buildkite/performance-benchmarks/tests/throughput-tests-arm64-cpu.json b/.buildkite/performance-benchmarks/tests/throughput-tests-arm64-cpu.json index da84dd4d0c67..3863cccf43a3 100644 --- a/.buildkite/performance-benchmarks/tests/throughput-tests-arm64-cpu.json +++ b/.buildkite/performance-benchmarks/tests/throughput-tests-arm64-cpu.json @@ -2,7 +2,6 @@ { "test_name": "throughput_llama8B_tp1", "environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120, "VLLM_CPU_KVCACHE_SPACE": 40 diff --git a/.buildkite/performance-benchmarks/tests/throughput-tests-cpu.json b/.buildkite/performance-benchmarks/tests/throughput-tests-cpu.json index dc214ddfb27e..d3f16eff116b 100644 --- a/.buildkite/performance-benchmarks/tests/throughput-tests-cpu.json +++ b/.buildkite/performance-benchmarks/tests/throughput-tests-cpu.json @@ -2,7 +2,6 @@ { "test_name": "throughput_llama8B_tp2", "environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120, "VLLM_CPU_SGL_KERNEL": 1, diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index 8fce15680173..a34f534e54da 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -7,6 +7,9 @@ env: CUDA_ARCH_AARCH64: "8.0 8.7 8.9 9.0 10.0 11.0 12.0+PTX" CUDA_ARCH_X86_CU129: "7.5 8.0 8.6 8.9 9.0 10.0 12.0" CUDA_ARCH_AARCH64_CU129: "8.0 8.7 8.9 9.0 10.0 12.0" + MOONCAKE_WHEEL_AARCH64_2_35: "https://vllm-wheels.s3.amazonaws.com/mooncake/mooncake_transfer_engine-0.3.10.post2-0da9dfea3-cp312-cp312-manylinux_2_35_aarch64.whl" + MOONCAKE_WHEEL_AARCH64_2_39: "https://vllm-wheels.s3.amazonaws.com/mooncake/mooncake_transfer_engine-0.3.10.post2-0da9dfea3-cp312-cp312-manylinux_2_39_aarch64.whl" + MOONCAKE_WHEEL_X86_64: "https://vllm-wheels.s3.amazonaws.com/mooncake/mooncake_transfer_engine-0.3.10.post2-0da9dfea3-cp312-cp312-manylinux_2_35_x86_64.whl" steps: - input: "Provide Release version here" @@ -24,10 +27,11 @@ steps: agents: queue: arm64_cpu_queue_release commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda12.9 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_31" + - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)"' env: DOCKER_BUILDKIT: "1" @@ -37,10 +41,11 @@ steps: agents: queue: arm64_cpu_queue_release commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35" + - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)"' env: DOCKER_BUILDKIT: "1" @@ -53,7 +58,8 @@ steps: - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_BUILD_ACL=ON --tag vllm-ci:build-image --target vllm-build --progress plain -f docker/Dockerfile.cpu ." - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35" + - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)"' env: DOCKER_BUILDKIT: "1" @@ -63,10 +69,11 @@ steps: agents: queue: cpu_queue_release commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda12.9 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_31" + - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)"' env: DOCKER_BUILDKIT: "1" @@ -76,10 +83,11 @@ steps: agents: queue: cpu_queue_release commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35" + - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)"' env: DOCKER_BUILDKIT: "1" @@ -92,7 +100,8 @@ steps: - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_X86=true --tag vllm-ci:build-image --target vllm-build --progress plain -f docker/Dockerfile.cpu ." - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35" + - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)"' env: DOCKER_BUILDKIT: "1" @@ -121,11 +130,26 @@ steps: queue: cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ." + - | + DOCKER_BUILDKIT=1 docker build \ + $(bash .buildkite/scripts/docker-build-metadata-args.sh) \ + --build-arg max_jobs=16 \ + --build-arg USE_SCCACHE=1 \ + --build-arg GIT_REPO_CHECK=1 \ + --build-arg CUDA_VERSION=13.0.2 \ + --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86}" \ + --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_35}" \ + --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \ + --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 \ + --target vllm-openai \ + --progress plain \ + -f docker/Dockerfile . - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)" # re-tag to default image tag and push, just in case arm64 build fails - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT" - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)"' - label: "Build release image - aarch64 - CUDA 13.0" depends_on: ~ @@ -134,8 +158,23 @@ steps: queue: arm64_cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ." + - | + DOCKER_BUILDKIT=1 docker build \ + $(bash .buildkite/scripts/docker-build-metadata-args.sh) \ + --build-arg max_jobs=16 \ + --build-arg USE_SCCACHE=1 \ + --build-arg GIT_REPO_CHECK=1 \ + --build-arg CUDA_VERSION=13.0.2 \ + --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64}" \ + --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_35}" \ + --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \ + --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 \ + --target vllm-openai \ + --progress plain \ + -f docker/Dockerfile . - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)"' - label: "Build release image - x86_64 - CUDA 12.9" depends_on: ~ @@ -144,11 +183,25 @@ steps: queue: cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129 --target vllm-openai --progress plain -f docker/Dockerfile ." + - | + DOCKER_BUILDKIT=1 docker build \ + $(bash .buildkite/scripts/docker-build-metadata-args.sh cu129) \ + --build-arg max_jobs=16 \ + --build-arg USE_SCCACHE=1 \ + --build-arg GIT_REPO_CHECK=1 \ + --build-arg CUDA_VERSION=12.9.1 \ + --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86_CU129}" \ + --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_35}" \ + --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \ + --target vllm-openai \ + --progress plain \ + -f docker/Dockerfile . - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129" # re-tag to default image tag and push, just in case arm64 build fails - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129"' - label: "Build release image - aarch64 - CUDA 12.9" depends_on: ~ @@ -157,8 +210,22 @@ steps: queue: arm64_cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129 --target vllm-openai --progress plain -f docker/Dockerfile ." + - | + DOCKER_BUILDKIT=1 docker build \ + $(bash .buildkite/scripts/docker-build-metadata-args.sh cu129) \ + --build-arg max_jobs=16 \ + --build-arg USE_SCCACHE=1 \ + --build-arg GIT_REPO_CHECK=1 \ + --build-arg CUDA_VERSION=12.9.1 \ + --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64_CU129}" \ + --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_35}" \ + --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \ + --target vllm-openai \ + --progress plain \ + -f docker/Dockerfile . - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129"' - label: "Build release image - x86_64 - CUDA 13.0 - Ubuntu 24.04" depends_on: ~ @@ -167,10 +234,27 @@ steps: queue: cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ." + - | + DOCKER_BUILDKIT=1 docker build \ + $(bash .buildkite/scripts/docker-build-metadata-args.sh ubuntu2404) \ + --build-arg max_jobs=16 \ + --build-arg USE_SCCACHE=1 \ + --build-arg GIT_REPO_CHECK=1 \ + --build-arg CUDA_VERSION=13.0.2 \ + --build-arg UBUNTU_VERSION=24.04 \ + --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \ + --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86}" \ + --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_39}" \ + --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \ + --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu24.04 \ + --target vllm-openai \ + --progress plain \ + -f docker/Dockerfile . - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404" - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404" - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404"' - label: "Build release image - aarch64 - CUDA 13.0 - Ubuntu 24.04" depends_on: ~ @@ -179,8 +263,25 @@ steps: queue: arm64_cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ." + - | + DOCKER_BUILDKIT=1 docker build \ + $(bash .buildkite/scripts/docker-build-metadata-args.sh ubuntu2404) \ + --build-arg max_jobs=16 \ + --build-arg USE_SCCACHE=1 \ + --build-arg GIT_REPO_CHECK=1 \ + --build-arg CUDA_VERSION=13.0.2 \ + --build-arg UBUNTU_VERSION=24.04 \ + --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \ + --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64}" \ + --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_39}" \ + --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \ + --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu24.04 \ + --target vllm-openai \ + --progress plain \ + -f docker/Dockerfile . - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404"' - label: "Build release image - x86_64 - CUDA 12.9 - Ubuntu 24.04" depends_on: ~ @@ -189,10 +290,26 @@ steps: queue: cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ." + - | + DOCKER_BUILDKIT=1 docker build \ + $(bash .buildkite/scripts/docker-build-metadata-args.sh cu129-ubuntu2404) \ + --build-arg max_jobs=16 \ + --build-arg USE_SCCACHE=1 \ + --build-arg GIT_REPO_CHECK=1 \ + --build-arg CUDA_VERSION=12.9.1 \ + --build-arg UBUNTU_VERSION=24.04 \ + --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \ + --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86_CU129}" \ + --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_39}" \ + --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \ + --target vllm-openai \ + --progress plain \ + -f docker/Dockerfile . - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404" - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404" - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404"' - label: "Build release image - aarch64 - CUDA 12.9 - Ubuntu 24.04" depends_on: ~ @@ -201,14 +318,31 @@ steps: queue: arm64_cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ." + - | + DOCKER_BUILDKIT=1 docker build \ + $(bash .buildkite/scripts/docker-build-metadata-args.sh cu129-ubuntu2404) \ + --build-arg max_jobs=16 \ + --build-arg USE_SCCACHE=1 \ + --build-arg GIT_REPO_CHECK=1 \ + --build-arg CUDA_VERSION=12.9.1 \ + --build-arg UBUNTU_VERSION=24.04 \ + --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \ + --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64_CU129}" \ + --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_39}" \ + --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \ + --target vllm-openai \ + --progress plain \ + -f docker/Dockerfile . - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404"' - block: "Build release image for x86_64 CPU" key: block-cpu-release-image-build depends_on: ~ - label: "Build release image - x86_64 - CPU" + key: build-cpu-release-image-x86 depends_on: - block-cpu-release-image-build - input-release-version @@ -219,6 +353,7 @@ steps: - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_X86=true --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest --progress plain --target vllm-openai -f docker/Dockerfile.cpu ." - "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest" - "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version)" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version)"' env: DOCKER_BUILDKIT: "1" @@ -227,7 +362,8 @@ steps: depends_on: ~ - label: "Build release image - arm64 - CPU" - depends_on: + key: build-cpu-release-image-arm64 + depends_on: - block-arm64-cpu-release-image-build - input-release-version agents: @@ -237,6 +373,7 @@ steps: - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --tag public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:latest --progress plain --target vllm-openai -f docker/Dockerfile.cpu ." - "docker push public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:latest" - "docker push public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$(buildkite-agent meta-data get release-version)" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$(buildkite-agent meta-data get release-version)"' env: DOCKER_BUILDKIT: "1" @@ -254,15 +391,7 @@ steps: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64 --amend" - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT" - - - label: "Annotate release workflow - CUDA 13.0" - depends_on: - - create-multi-arch-manifest - id: annotate-release-workflow - agents: - queue: small_cpu_queue_release - commands: - - "bash .buildkite/scripts/annotate-release.sh" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 13.0" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"' - label: "Create multi-arch manifest - CUDA 12.9" depends_on: @@ -275,6 +404,7 @@ steps: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu129 --amend" - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 12.9" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129"' - label: "Create multi-arch manifest - CUDA 13.0 - Ubuntu 24.04" depends_on: @@ -287,6 +417,7 @@ steps: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-ubuntu2404 --amend" - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 13.0 Ubuntu 24.04" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404"' - label: "Create multi-arch manifest - CUDA 12.9 - Ubuntu 24.04" depends_on: @@ -299,6 +430,7 @@ steps: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu129-ubuntu2404 --amend" - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 12.9 Ubuntu 24.04" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404"' - label: "Publish nightly multi-arch image to DockerHub" depends_on: @@ -336,24 +468,6 @@ steps: DOCKER_BUILDKIT: "1" DOCKERHUB_USERNAME: "vllmbot" - - group: "Publish wheels" - key: "publish-wheels" - steps: - - block: "Confirm update release wheels to PyPI (experimental, use with caution)?" - key: block-upload-release-wheels - depends_on: - - input-release-version - - build-wheels - - - label: "Upload release wheels to PyPI" - depends_on: - - block-upload-release-wheels - id: upload-release-wheels - agents: - queue: small_cpu_queue_release - commands: - - "bash .buildkite/scripts/upload-release-wheels-pypi.sh" - # ============================================================================= # ROCm Release Pipeline (x86_64 only) # ============================================================================= @@ -467,7 +581,7 @@ steps: echo "" echo " Build complete - Image and wheels cached" fi - + artifact_paths: - "artifacts/rocm-base-wheels/*.whl" env: @@ -623,7 +737,7 @@ steps: - "bash tools/vllm-rocm/generate-rocm-wheels-root-index.sh" env: S3_BUCKET: "vllm-wheels" - VARIANT: "rocm721" + VARIANT: "rocm723" # ROCm Job 6: Build ROCm Release Docker Image - label: ":docker: Build release image - x86_64 - ROCm" @@ -683,7 +797,7 @@ steps: # Push to ECR docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm - + echo "" echo " Successfully built and pushed ROCm release image" echo " Image: public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm" @@ -710,3 +824,60 @@ steps: env: DOCKER_BUILDKIT: "1" DOCKERHUB_USERNAME: "vllmbot" + + # ============================================================================= + # Publish to DockerHub and PyPI (at the end so all builds complete first) + # ============================================================================= + + - block: "Publish release images to DockerHub" + key: block-publish-release-images + depends_on: + - create-multi-arch-manifest + - create-multi-arch-manifest-cuda-12-9 + - create-multi-arch-manifest-ubuntu2404 + - create-multi-arch-manifest-cuda-12-9-ubuntu2404 + - build-rocm-release-image + - input-release-version + # Wait for CPU builds if their block steps were unblocked, so publish + # doesn't race the in-progress CPU build. allow_failure lets publish + # proceed when the operator legitimately leaves the CPU block steps + # unblocked or the CPU build fails. + - step: build-cpu-release-image-x86 + allow_failure: true + - step: build-cpu-release-image-arm64 + allow_failure: true + if: build.env("NIGHTLY") != "1" + + - label: "Publish release images to DockerHub" + depends_on: + - block-publish-release-images + key: publish-release-images-dockerhub + agents: + queue: small_cpu_queue_release + commands: + - "bash .buildkite/scripts/publish-release-images.sh" + plugins: + - docker-login#v3.0.0: + username: vllmbot + password-env: DOCKERHUB_TOKEN + env: + DOCKER_BUILDKIT: "1" + DOCKERHUB_USERNAME: "vllmbot" + + - group: "Publish wheels" + key: "publish-wheels" + steps: + - block: "Confirm update release wheels to PyPI (experimental, use with caution)?" + key: block-upload-release-wheels + depends_on: + - input-release-version + - build-wheels + + - label: "Upload release wheels to PyPI" + depends_on: + - block-upload-release-wheels + id: upload-release-wheels + agents: + queue: small_cpu_queue_release + commands: + - "bash .buildkite/scripts/upload-release-wheels-pypi.sh" diff --git a/.buildkite/scripts/annotate-build-artifact.sh b/.buildkite/scripts/annotate-build-artifact.sh new file mode 100755 index 000000000000..67cdf7923658 --- /dev/null +++ b/.buildkite/scripts/annotate-build-artifact.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# Append a build artifact line to the Buildkite annotation. +# Usage: annotate-build-artifact.sh