[CI][ROCm] Make image dependencies cache-safe

.buildkite/ci_config_rocm.yaml

@@ -17,7 +17,9 @@ run_all_patterns:
   - "requirements/common.txt"
   - "requirements/rocm.txt"
   - "requirements/build/rust.txt"
+  - "requirements/build/rocm.in"
   - "requirements/build/rocm.txt"
+  - "requirements/test/rocm.in"
   - "requirements/test/rocm.txt"
   - "pyproject.toml"
   - "setup.py"

.buildkite/release-pipeline.yaml

@@ -603,106 +603,28 @@ steps:
         depends_on: ~
         agents:
           queue: cpu_queue_release
+        timeout_in_minutes: 360
+        retry:
+          automatic:
+            - exit_status: 1  # Transient registry/object-store/build failure
+              limit: 1
+            - exit_status: -1  # Agent was lost
+              limit: 1
+            - exit_status: -10  # Agent was lost
+              limit: 1
         commands:
-          - |
-            set -euo pipefail
-
-            # Generate cache key
-            CACHE_KEY=$$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
-            ECR_CACHE_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:$${CACHE_KEY}-rocm-base"
-
-            echo "========================================"
-            echo "ROCm Base Build Configuration"
-            echo "========================================"
-            echo "  CACHE_KEY: $${CACHE_KEY}"
-            echo "  ECR_CACHE_TAG: $${ECR_CACHE_TAG}"
-            echo "========================================"
-
-            # Login to ECR
-            aws ecr-public get-login-password --region us-east-1 | \
-              docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
-
-            IMAGE_EXISTS=false
-            WHEELS_EXIST=false
-
-            # Check ECR for Docker image
-
-            if docker manifest inspect "$${ECR_CACHE_TAG}" > /dev/null 2>&1; then
-              IMAGE_EXISTS=true
-              echo "ECR image cache HIT"
-            fi
-
-            # Check S3 for wheels
-            WHEEL_CACHE_STATUS=$(.buildkite/scripts/cache-rocm-base-wheels.sh check)
-            if [ "$${WHEEL_CACHE_STATUS}" = "hit" ]; then
-              WHEELS_EXIST=true
-              echo "S3 wheels cache HIT"
-            fi
-
-
-            # Scenario 1: Both cached (best case)
-            if [ "$${IMAGE_EXISTS}" = "true" ] && [ "$${WHEELS_EXIST}" = "true" ]; then
-              echo ""
-              echo "FULL CACHE HIT - Reusing both image and wheels"
-              echo ""
-
-              # Download wheels
-              .buildkite/scripts/cache-rocm-base-wheels.sh download
-
-              # Save ECR tag for downstream jobs
-              buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
-
-            # Scenario 2: Full rebuild needed
-            else
-              echo ""
-              echo " CACHE MISS - Building from scratch..."
-              echo ""
-
-              # Build full base image and push to ECR
-              DOCKER_BUILDKIT=1 docker buildx build \
-                --file docker/Dockerfile.rocm_base \
-                --tag "$${ECR_CACHE_TAG}" \
-                --build-arg USE_SCCACHE=1 \
-                --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
-                --build-arg SCCACHE_REGION_NAME=us-west-2 \
-                --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
-                --push \
-                .
-
-              # Build wheel extraction stage
-              DOCKER_BUILDKIT=1 docker buildx build \
-                --file docker/Dockerfile.rocm_base \
-                --tag rocm-base-debs:$${BUILDKITE_BUILD_NUMBER} \
-                --target debs_wheel_release \
-                --build-arg USE_SCCACHE=1 \
-                --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
-                --build-arg SCCACHE_REGION_NAME=us-west-2 \
-                --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
-                --load \
-                .
-
-              # Extract and upload wheels
-              mkdir -p artifacts/rocm-base-wheels
-              cid=$(docker create rocm-base-debs:$${BUILDKITE_BUILD_NUMBER})
-              docker cp $${cid}:/app/debs/. artifacts/rocm-base-wheels/
-              docker rm $${cid}
-
-              .buildkite/scripts/cache-rocm-base-wheels.sh upload
-
-              # Cache base docker image to ECR
-              docker push "$${ECR_CACHE_TAG}"
-
-              buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
-
-              echo ""
-              echo " Build complete - Image and wheels cached"
-            fi
+          - bash .buildkite/scripts/build-rocm-base-wheels.sh
 
         artifact_paths:
           - "artifacts/rocm-base-wheels/*.whl"
         env:
           DOCKER_BUILDKIT: "1"
+          DOCKERHUB_USERNAME: "vllmbot"
           S3_BUCKET: "vllm-wheels"
+        plugins:
+          - docker-login#v3.0.0:
+              username: vllmbot
+              password-env: DOCKERHUB_TOKEN
 
       # ROCm Job 2: Build vLLM ROCm Wheel
       - label: ":python: Build vLLM ROCm Wheel - x86_64"
@@ -748,6 +670,10 @@ steps:
               echo "This should have been set by the build-rocm-base-wheels job"
               exit 1
             fi
+            if [[ ! "$${ECR_IMAGE_TAG}" =~ ^public\.ecr\.aws/q9t5s3a7/vllm-release-repo@sha256:[0-9a-f]{64}$$ ]]; then
+              echo "ERROR: refusing unexpected ROCm base image ref: $${ECR_IMAGE_TAG}" >&2
+              exit 1
+            fi
 
             echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}"
 
@@ -881,6 +807,10 @@ steps:
               echo "This should have been set by the build-rocm-base-wheels job"
               exit 1
             fi
+            if [[ ! "$${ECR_IMAGE_TAG}" =~ ^public\.ecr\.aws/q9t5s3a7/vllm-release-repo@sha256:[0-9a-f]{64}$$ ]]; then
+              echo "ERROR: refusing unexpected ROCm base image ref: $${ECR_IMAGE_TAG}" >&2
+              exit 1
+            fi
 
             echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}"
 

.buildkite/scripts/annotate-rocm-release.sh

@@ -18,7 +18,11 @@ if [ -z "${RELEASE_VERSION}" ]; then
   RELEASE_VERSION="1.0.0.dev"
 fi
 
-ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
+ROCM_BASE_IMAGE_TAG=$(buildkite-agent meta-data get rocm-base-image-tag 2>/dev/null || true)
+if [ -z "${ROCM_BASE_IMAGE_TAG}" ]; then
+  echo "ERROR: rocm-base-image-tag metadata not found" >&2
+  exit 1
+fi
 
 # S3 URLs
 S3_BUCKET="${S3_BUCKET:-vllm-wheels}"
@@ -94,10 +98,10 @@ aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/flash-
 To download and upload the image:
 
 \`\`\`
-docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base
+docker pull ${ROCM_BASE_IMAGE_TAG}
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm
 
-docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base
+docker tag ${ROCM_BASE_IMAGE_TAG} vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base
 docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:latest-base
 docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
 docker push vllm/vllm-openai-rocm:latest-base

.buildkite/scripts/build-rocm-base-wheels.sh

@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# Run from the repository root to build or reuse paired ROCm base image/wheels.
+set -euo pipefail
+
+if [[ "${SCCACHE_ENDPOINT:-}" == *[@?#]* ||
+      "${SCCACHE_ENDPOINT:-}" =~ [[:space:][:cntrl:]] ]]; then
+    echo "ERROR: SCCACHE_ENDPOINT must not contain credentials, query parameters, fragments, or whitespace" >&2
+    exit 1
+fi
+
+TEMP_FILES=()
+WHEEL_CONTAINER=""
+cleanup() {
+    if [[ -n "${WHEEL_CONTAINER}" ]]; then
+        docker rm -f "${WHEEL_CONTAINER}" >/dev/null || true
+    fi
+    if ((${#TEMP_FILES[@]})); then
+        rm -f -- "${TEMP_FILES[@]}"
+    fi
+}
+trap cleanup EXIT
+
+# Keep the cache key, image, and exported wheels on the same release inputs.
+export ROCM_BASE_DOCKERFILE=docker/Dockerfile.rocm_base
+export ROCM_BASE_CONTENT_FILES="docker/Dockerfile.rocm_base .dockerignore requirements/build/rocm.txt"
+export ROCM_BASE_IMAGE_TARGET=final
+export ROCM_BASE_WHEEL_TARGET=debs_wheel_release
+export ROCM_BASE_PLATFORM=linux/amd64
+export ROCM_BASE_USE_SCCACHE=1
+export SCCACHE_BUCKET_NAME=vllm-build-sccache
+export SCCACHE_REGION_NAME=us-west-2
+export SCCACHE_S3_NO_CREDENTIALS=0
+# Build ARGs are public: release builds use the checksum-pinned download default.
+unset ROCM_BASE_PARENT_DIGEST SCCACHE_DOWNLOAD_URL
+
+dockerfile_arg_default() {
+    sed -n -E "s/^ARG ${1}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
+        "${ROCM_BASE_DOCKERFILE}" | head -1
+}
+
+PYTORCH_ROCM_ARCH="${ROCM_BASE_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH:-$(dockerfile_arg_default PYTORCH_ROCM_ARCH)}}"
+SCCACHE_VERSION="${SCCACHE_VERSION:-$(dockerfile_arg_default SCCACHE_VERSION)}"
+SCCACHE_DOWNLOAD_SHA256="${SCCACHE_DOWNLOAD_SHA256:-$(dockerfile_arg_default SCCACHE_DOWNLOAD_SHA256)}"
+export PYTORCH_ROCM_ARCH SCCACHE_VERSION SCCACHE_DOWNLOAD_SHA256
+
+ROCM_BASE_CACHE_POLICY_ARGS=()
+if [[ "${ROCM_BASE_WHEEL_CACHE_FORCE:-0}" == "1" ]]; then
+    ROCM_BASE_CACHE_POLICY_ARGS+=(--no-cache)
+elif [[ "${ROCM_BASE_WHEEL_CACHE_FORCE:-0}" != "0" ]]; then
+    echo "ERROR: ROCM_BASE_WHEEL_CACHE_FORCE must be 0 or 1" >&2
+    exit 1
+fi
+
+PINNED_BASE_IMAGE=$(.buildkite/scripts/cache-rocm-base-wheels.sh parent)
+export ROCM_BASE_PARENT_IMAGE="${PINNED_BASE_IMAGE%@*}"
+export ROCM_BASE_PARENT_DIGEST="${PINNED_BASE_IMAGE##*@}"
+CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
+ECR_REPOSITORY="public.ecr.aws/q9t5s3a7/vllm-release-repo"
+ECR_CACHE_TAG="${ECR_REPOSITORY}:${CACHE_KEY}-rocm-base"
+ECR_IMAGE_REF=""
+
+echo "ROCm Base Build Configuration"
+echo "  CACHE_KEY: ${CACHE_KEY}"
+echo "  BASE_IMAGE: ${PINNED_BASE_IMAGE}"
+echo "  ECR_CACHE_TAG: ${ECR_CACHE_TAG}"
+
+aws ecr-public get-login-password --region us-east-1 | \
+    docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
+
+IMAGE_EXISTS=false
+WHEELS_EXIST=false
+WHEEL_CACHE_STATUS=$(.buildkite/scripts/cache-rocm-base-wheels.sh check)
+if [[ "${WHEEL_CACHE_STATUS}" == "hit" ]]; then
+    WHEELS_EXIST=true
+    echo "S3 wheels cache HIT"
+
+    # Validate the immutable wheel set before trusting its paired image digest.
+    CACHED_IMAGE_DIGEST_FILE=$(mktemp)
+    TEMP_FILES+=("${CACHED_IMAGE_DIGEST_FILE}")
+    export ROCM_BASE_IMAGE_DIGEST_FILE="${CACHED_IMAGE_DIGEST_FILE}"
+    if ! .buildkite/scripts/cache-rocm-base-wheels.sh download; then
+        WHEELS_EXIST=false
+        echo "S3 wheel cache validation failed; rebuilding"
+    else
+        CACHED_IMAGE_DIGEST=$(cat "${CACHED_IMAGE_DIGEST_FILE}")
+        ECR_IMAGE_REF="${ECR_REPOSITORY}@${CACHED_IMAGE_DIGEST}"
+        if docker manifest inspect "${ECR_IMAGE_REF}" >/dev/null 2>&1; then
+            IMAGE_EXISTS=true
+            echo "Paired ECR image cache HIT"
+        else
+            echo "Paired ECR image digest is unavailable; rebuilding"
+        fi
+    fi
+    unset ROCM_BASE_IMAGE_DIGEST_FILE
+fi
+
+if [[ "${IMAGE_EXISTS}" == "true" && "${WHEELS_EXIST}" == "true" ]]; then
+    echo "FULL CACHE HIT - Reusing both image and wheels"
+else
+    echo "CACHE MISS - Building from scratch..."
+    ROCM_BASE_BUILD_ARGS=(
+        --file "${ROCM_BASE_DOCKERFILE}"
+        --platform "${ROCM_BASE_PLATFORM}"
+        --build-arg "BASE_IMAGE=${PINNED_BASE_IMAGE}"
+        --build-arg "USE_SCCACHE=${ROCM_BASE_USE_SCCACHE}"
+        --build-arg "PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH}"
+        --build-arg "SCCACHE_VERSION=${SCCACHE_VERSION}"
+        --build-arg "SCCACHE_DOWNLOAD_SHA256=${SCCACHE_DOWNLOAD_SHA256}"
+        --build-arg "SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT:-}"
+        --build-arg "SCCACHE_BUCKET_NAME=${SCCACHE_BUCKET_NAME}"
+        --build-arg "SCCACHE_REGION_NAME=${SCCACHE_REGION_NAME}"
+        --build-arg "SCCACHE_S3_NO_CREDENTIALS=${SCCACHE_S3_NO_CREDENTIALS}"
+    )
+
+    ROCM_BASE_BUILD_METADATA=$(mktemp)
+    TEMP_FILES+=("${ROCM_BASE_BUILD_METADATA}")
+    DOCKER_BUILDKIT=1 docker buildx build \
+        "${ROCM_BASE_BUILD_ARGS[@]}" \
+        "${ROCM_BASE_CACHE_POLICY_ARGS[@]}" \
+        --tag "${ECR_CACHE_TAG}" \
+        --target "${ROCM_BASE_IMAGE_TARGET}" \
+        --label "vllm.rocm_base.cache_key=${CACHE_KEY}" \
+        --label "vllm.rocm_base.pytorch_rocm_arch=${PYTORCH_ROCM_ARCH}" \
+        --label "vllm.rocm_base.sccache_version=${SCCACHE_VERSION}" \
+        --label "vllm.rocm_base.sccache_sha256=${SCCACHE_DOWNLOAD_SHA256}" \
+        --metadata-file "${ROCM_BASE_BUILD_METADATA}" \
+        --push \
+        .
+    ECR_IMAGE_DIGEST=$(awk -F'"' \
+        '$2 == "containerimage.digest" { print $4; exit }' \
+        "${ROCM_BASE_BUILD_METADATA}")
+    if [[ ! "${ECR_IMAGE_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
+        echo "ERROR: build returned an invalid ROCm base image digest" >&2
+        exit 1
+    fi
+    ECR_IMAGE_REF="${ECR_REPOSITORY}@${ECR_IMAGE_DIGEST}"
+    export ROCM_BASE_IMAGE_DIGEST="${ECR_IMAGE_DIGEST}"
+
+    # Reuse the image build's layers even when that build was forced.
+    DOCKER_BUILDKIT=1 docker buildx build \
+        "${ROCM_BASE_BUILD_ARGS[@]}" \
+        --tag "rocm-base-debs:${BUILDKITE_BUILD_NUMBER}" \
+        --target "${ROCM_BASE_WHEEL_TARGET}" \
+        --load \
+        .
+
+    mkdir -p artifacts/rocm-base-wheels
+    find artifacts/rocm-base-wheels -mindepth 1 -depth -delete
+    WHEEL_CONTAINER=$(docker create "rocm-base-debs:${BUILDKITE_BUILD_NUMBER}")
+    docker cp "${WHEEL_CONTAINER}:/app/debs/." artifacts/rocm-base-wheels/
+    docker rm "${WHEEL_CONTAINER}"
+    WHEEL_CONTAINER=""
+    .buildkite/scripts/cache-rocm-base-wheels.sh upload
+    echo "Build complete - Image and wheels cached"
+fi
+
+if [[ ! "${ECR_IMAGE_REF}" =~ ^public\.ecr\.aws/q9t5s3a7/vllm-release-repo@sha256:[0-9a-f]{64}$ ]]; then
+    echo "ERROR: refusing unexpected ROCm base image ref: ${ECR_IMAGE_REF}" >&2
+    exit 1
+fi
+buildkite-agent meta-data set "rocm-base-image-tag" "${ECR_IMAGE_REF}"