[CI][ROCm] Avoid redundant image pulls during smoke validation

.buildkite/scripts/ci-bake-rocm.sh

@@ -842,6 +842,29 @@ should_upload_wheel_artifacts() {
         || "${TARGET}" == *"artifact"* ]]
 }
 
+should_export_rocm_smoke() {
+    [[ "${TARGET}" == "test-rocm-ci-with-wheel" \
+        || "${TARGET}" == "smoke-test-rocm-ci" ]]
+}
+
+verify_rocm_smoke_export() {
+    local marker="./build/rocm-smoke-export/vllm-smoke-ok"
+    local expected_smoke_id="${BUILDKITE_BUILD_ID:-local}"
+    local actual_smoke_id=""
+
+    should_export_rocm_smoke || return 0
+    if [[ ! -f "${marker}" ]]; then
+        echo "ROCm BuildKit smoke marker is missing: ${marker}" >&2
+        return 1
+    fi
+    actual_smoke_id="$(< "${marker}")"
+    if [[ "${actual_smoke_id}" != "${expected_smoke_id}" ]]; then
+        echo "ROCm BuildKit smoke marker belongs to ${actual_smoke_id}, not ${expected_smoke_id}" \
+            >&2
+        return 1
+    fi
+}
+
 get_remote_image_label() {
     local image_ref="$1"
     local label_key="$2"
@@ -1386,8 +1409,9 @@ maybe_skip_existing_image() {
         echo "FORCE_BUILD=1 set; skipping existing-image check"
         return 0
     fi
-    if ! is_ci_base_target && should_upload_wheel_artifacts; then
-        echo "Artifact-producing targets always run for the current build"
+    if ! is_ci_base_target \
+        && { should_upload_wheel_artifacts || should_export_rocm_smoke; }; then
+        echo "Local-output targets always run for the current build"
         return 0
     fi
 
@@ -1741,7 +1765,12 @@ EOF
 
 uses_rocm_csrc_cache() {
     case "${TARGET}" in
-        csrc-rocm-ci|test-rocm-ci|test-rocm-ci-with-wheel|test-rocm-ci-with-artifacts|export-wheel-rocm)
+        csrc-rocm-ci \
+            | test-rocm-ci \
+            | test-rocm-ci-with-wheel \
+            | test-rocm-ci-with-artifacts \
+            | export-wheel-rocm \
+            | smoke-test-rocm-ci)
             return 0
             ;;
         *)
@@ -1752,7 +1781,12 @@ uses_rocm_csrc_cache() {
 
 uses_rocm_rust_cache() {
     case "${TARGET}" in
-        rust-rocm-ci|test-rocm-ci|test-rocm-ci-with-wheel|test-rocm-ci-with-artifacts|export-wheel-rocm)
+        rust-rocm-ci \
+            | test-rocm-ci \
+            | test-rocm-ci-with-wheel \
+            | test-rocm-ci-with-artifacts \
+            | export-wheel-rocm \
+            | smoke-test-rocm-ci)
             return 0
             ;;
         *)
@@ -2140,7 +2174,7 @@ write_rocm_cache_override() {
     # exporter unless it is requested explicitly.
     if ((${#rust_cache_to[@]} > 0)); then
         case "${TARGET}" in
-            test-rocm-ci|export-wheel-rocm)
+            test-rocm-ci|export-wheel-rocm|smoke-test-rocm-ci)
                 BAKE_TARGETS=("rust-rocm-ci" "${BAKE_TARGETS[@]}")
                 ;;
         esac
@@ -2190,6 +2224,15 @@ EOF
         cat <<EOF
 }
 
+target "smoke-test-rocm-ci" {
+  cache-from = concat(
+    get_cache_from_rocm(),
+EOF
+        write_hcl_string_list "    " "${combined_content_cache_from[@]}"
+        cat <<EOF
+  )
+}
+
 target "export-wheel-rocm" {
   cache-from = concat(
     get_cache_from_rocm(),
@@ -2675,8 +2718,14 @@ main() {
         # clean prevents a failed/retried export from packaging a stale wheel.
         rm -rf ./wheel-export
     fi
+    if should_export_rocm_smoke; then
+        # The marker is a build output, not a cache. Never accept stale output
+        # from an earlier build or retry.
+        rm -rf ./build/rocm-smoke-export
+    fi
     seed_dependency_caches_if_needed
     run_bake
+    verify_rocm_smoke_export
     promote_stable_ci_base_tag
     publish_ci_base_handoff_ref
     upload_wheel_artifacts_if_present

.buildkite/scripts/rocm/smoke-test-image.sh

@@ -3,6 +3,53 @@
 
 set -euo pipefail
 
+run_smoke_checks() {
+    local required_dir=""
+
+    for required_dir in \
+        /vllm-workspace \
+        /vllm-workspace/tests \
+        /vllm-workspace/src/vllm; do
+        if [[ ! -d "${required_dir}" ]]; then
+            echo "Missing directory: ${required_dir}" >&2
+            return 1
+        fi
+    done
+    if [[ ! -x /vllm-workspace/src/vllm/vllm-rs ]]; then
+        echo "Missing executable: /vllm-workspace/src/vllm/vllm-rs" >&2
+        return 1
+    fi
+
+    command -v python3
+    command -v uv
+    command -v pytest
+
+    if ! command -v amd-smi >/dev/null 2>&1 \
+        && ! command -v rocminfo >/dev/null 2>&1; then
+        echo "No ROCm CLI found in image" >&2
+        return 1
+    fi
+
+    PYTHONDONTWRITEBYTECODE=1 python3 - <<'PY'
+import torch
+import vllm
+
+print(torch.__version__)
+print(vllm.__version__)
+PY
+
+    echo "AMD image smoke OK"
+}
+
+if [[ "${1:-}" == "--inside" ]]; then
+    run_smoke_checks
+    exit
+fi
+if (($#)); then
+    echo "Usage: $0 [--inside]" >&2
+    exit 2
+fi
+
 if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then
     base_refreshed=""
     if command -v buildkite-agent >/dev/null 2>&1; then
@@ -14,30 +61,24 @@ if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then
     fi
 fi
 
-image_ref="${VLLM_CI_SMOKE_IMAGE:-${IMAGE_TAG:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}}"
-
-docker run --rm --network=none --entrypoint /bin/bash "${image_ref}" -ec '
-  if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi
-  if [ ! -d /vllm-workspace/tests ]; then echo Missing directory: /vllm-workspace/tests >&2; exit 1; fi
-  if [ ! -d /vllm-workspace/src/vllm ]; then echo Missing directory: /vllm-workspace/src/vllm >&2; exit 1; fi
-  if [ ! -x /vllm-workspace/src/vllm/vllm-rs ]; then echo Missing executable: /vllm-workspace/src/vllm/vllm-rs >&2; exit 1; fi
-
-  command -v python3
-  command -v uv
-  command -v pytest
-
-  if ! command -v amd-smi >/dev/null 2>&1 && ! command -v rocminfo >/dev/null 2>&1; then
-    echo No ROCm CLI found in image >&2
-    exit 1
-  fi
-
-  python3 - <<PY
-import torch
-import vllm
+smoke_marker="./build/rocm-smoke-export/vllm-smoke-ok"
+expected_smoke_id="${BUILDKITE_BUILD_ID:-local}"
+if [[ -f "${smoke_marker}" \
+    && ( -z "${VLLM_CI_SMOKE_IMAGE:-}" \
+        || "${VLLM_CI_SMOKE_IMAGE}" == "${IMAGE_TAG:-}" ) ]]; then
+    actual_smoke_id="$(< "${smoke_marker}")"
+    if [[ "${actual_smoke_id}" != "${expected_smoke_id}" ]]; then
+        echo "ROCm smoke marker belongs to ${actual_smoke_id}, not ${expected_smoke_id}" \
+            >&2
+        exit 1
+    fi
+    echo "AMD image smoke OK (verified inside BuildKit)"
+    rm -f -- "${smoke_marker}"
+    rmdir -- "$(dirname "${smoke_marker}")" 2>/dev/null || true
+    exit
+fi
 
-print(torch.__version__)
-print(vllm.__version__)
-PY
+image_ref="${VLLM_CI_SMOKE_IMAGE:-${IMAGE_TAG:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}}"
 
-  echo AMD image smoke OK
-'
+docker run --rm -i --network=none --entrypoint /bin/bash "${image_ref}" \
+    -s -- --inside < "${BASH_SOURCE[0]}"

docker/Dockerfile.rocm

@@ -899,6 +899,17 @@ RUN mkdir src && mv vllm src/vllm
 # Catch GPU<->CPU syncs in execute_model/sample_tokens during tests.
 ENV VLLM_GPU_SYNC_CHECK=error
 
+# Run the structural smoke test inside BuildKit, where the test image layers
+# are already available. Export only a build-scoped success marker.
+FROM test AS test_smoke
+ARG ROCM_SMOKE_ID
+RUN --network=none \
+    bash /vllm-workspace/.buildkite/scripts/rocm/smoke-test-image.sh --inside \
+    && printf '%s\n' "${ROCM_SMOKE_ID:-local}" > /vllm-smoke-ok
+
+FROM scratch AS export_test_smoke
+COPY --from=test_smoke /vllm-smoke-ok /
+
 # -----------------------
 # Final vLLM image
 FROM mori_base AS final_common

docker/ci-rocm.hcl

@@ -281,6 +281,7 @@ target "_ci-rocm" {
   args = {
     ARG_PYTORCH_ROCM_ARCH = PYTORCH_ROCM_ARCH
     CI_BASE_IMAGE         = CI_BASE_IMAGE
+    ROCM_SMOKE_ID         = BUILDKITE_BUILD_ID
     max_jobs              = CI_MAX_JOBS
   }
 }
@@ -297,6 +298,15 @@ target "test-rocm-ci" {
   output = ["type=registry"]
 }
 
+# Validate the test image in the shared BuildKit graph and export only the
+# success marker. This avoids pulling the multi-GB image into the host daemon.
+target "smoke-test-rocm-ci" {
+  inherits   = ["_common-rocm", "_ci-rocm"]
+  target     = "export_test_smoke"
+  cache-from = get_cache_from_rocm()
+  output     = ["type=local,dest=./build/rocm-smoke-export"]
+}
+
 # Cache-only target for the source-scoped ROCm native build stage.
 # This persists the csrc-build stage in the registry cache even though the
 # final test image only consumes it indirectly while packaging the wheel.
@@ -341,7 +351,13 @@ group "test-rocm-ci-with-artifacts" {
 # Full test image + wheel export. Kept for fallback/debugging when a pushed,
 # build-scoped image is useful.
 group "test-rocm-ci-with-wheel" {
-  targets = ["rust-rocm-ci", "csrc-rocm-ci", "test-rocm-ci", "export-wheel-rocm"]
+  targets = [
+    "rust-rocm-ci",
+    "csrc-rocm-ci",
+    "test-rocm-ci",
+    "smoke-test-rocm-ci",
+    "export-wheel-rocm",
+  ]
 }
 
 # Primary output tag for the ci_base build. In Buildkite this is a unique,