From e1506ae32d56ca5956e1335919141b26baf5745f Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sun, 2 Aug 2026 22:27:50 -0500 Subject: [PATCH 1/5] [CI][AMD] Support build-scoped ROCm images Signed-off-by: Andreas Karatzas --- buildkite/pipeline_generator/amd.py | 30 ++++----- .../pipeline_generator/buildkite_step.py | 38 ++++++++++- .../pipeline_generator/pipeline_generator.py | 2 +- buildkite/pipeline_generator/step.py | 21 +++++- .../tests/pipeline_generator/test_amd.py | 67 +++++++++++++------ .../tests/pipeline_generator/test_step.py | 65 +++++++++++++++++- 6 files changed, 178 insertions(+), 45 deletions(-) diff --git a/buildkite/pipeline_generator/amd.py b/buildkite/pipeline_generator/amd.py index 4f5da49c5..d57f55111 100644 --- a/buildkite/pipeline_generator/amd.py +++ b/buildkite/pipeline_generator/amd.py @@ -6,8 +6,7 @@ from constants import AgentQueue, DeviceType AMD_TEST_COMMAND = "bash .buildkite/scripts/hardware_ci/run-amd-test.sh" -AMD_STABLE_CI_BASE_IMAGE = "rocm/vllm-dev:ci_base" -AMD_NATIVE_BASE_IMAGE = "rocm/vllm-dev:ci_base-$BUILDKITE_COMMIT" +AMD_BUILD_CI_BASE_IMAGE = "rocm/vllm-dev:ci_base-build-$BUILDKITE_BUILD_ID" AMD_FALLBACK_CI_IMAGE = "rocm/vllm-ci:$BUILDKITE_COMMIT" AMD_ARTIFACT_GLOB = "artifacts/vllm-rocm-install/vllm-rocm-install.tar.gz" AMD_ARTIFACT_CHECKSUM_GLOB = f"{AMD_ARTIFACT_GLOB}.sha256" @@ -22,13 +21,14 @@ ) AMD_STANDARD_TIMEOUT_MINUTES = 3 * 60 AMD_ROCM_BASE_REFRESH_STEP_KEY = "refresh-rocm-base-amd" -AMD_ROCM_BASE_DOCKERFILE = "docker/Dockerfile.rocm_base" +AMD_STABLE_IMAGE_PROMOTION_STEP_KEY = "promote-stable-rocm-images-amd" AMD_ROCM_BASE_REFRESH_TIMEOUT_MINUTES = 9 * 60 AMD_ROCM_BASE_REFRESH_NOOP_TIMEOUT_MINUTES = 15 AMD_ALWAYS_RUN_STEP_KEYS = frozenset( { "ensure-ci-base-amd", AMD_ROCM_BASE_REFRESH_STEP_KEY, + AMD_STABLE_IMAGE_PROMOTION_STEP_KEY, } ) AMD_STACK_ERROR_RETRY = {"signal_reason": "stack_error", "limit": 1} @@ -85,9 +85,7 @@ def ensure_amd_stack_error_retry( ): return retry_policy else: - raise ValueError( - "AMD retry.automatic must be a boolean, mapping, or list." - ) + raise ValueError("AMD retry.automatic must be a boolean, mapping, or list.") retry_policy["automatic"] = [ dict(AMD_STACK_ERROR_RETRY), @@ -96,16 +94,12 @@ def ensure_amd_stack_error_retry( return retry_policy -def get_rocm_base_refresh_timeout(list_file_diff: List[str]) -> int: +def get_rocm_base_refresh_timeout(_list_file_diff: List[str]) -> int: if os.getenv("ROCM_BASE_REFRESH_SKIP", "0") == "1": return AMD_ROCM_BASE_REFRESH_NOOP_TIMEOUT_MINUTES - if ( - os.getenv("ROCM_BASE_REFRESH_FORCE", "0") == "1" - or os.getenv("ROCM_BASE_REFRESH_DIFF_UNAVAILABLE", "0") == "1" - or AMD_ROCM_BASE_DOCKERFILE in list_file_diff - ): - return AMD_ROCM_BASE_REFRESH_TIMEOUT_MINUTES - return AMD_ROCM_BASE_REFRESH_NOOP_TIMEOUT_MINUTES + # Every selection performs an exact registry lookup. A missing tag or a + # changed parent digest can turn any lookup into a self-healing build. + return AMD_ROCM_BASE_REFRESH_TIMEOUT_MINUTES def get_rocm_base_refresh_env() -> Dict[str, str]: @@ -295,7 +289,7 @@ def _get_amd_env( env.setdefault("HF_HUB_ETAG_TIMEOUT", "60") env.update( { - "VLLM_CI_BASE_IMAGE": AMD_NATIVE_BASE_IMAGE, + "VLLM_CI_BASE_IMAGE": AMD_BUILD_CI_BASE_IMAGE, "VLLM_CI_USE_ARTIFACTS": "1", "VLLM_CI_ARTIFACT_GLOB": AMD_ARTIFACT_GLOB, "VLLM_CI_RESULTS_ROOT": AMD_RESULTS_ROOT, @@ -313,8 +307,8 @@ def _get_amd_env( env.update( { "DOCKER_BUILDKIT": "1", - "DOCKER_IMAGE_NAME": AMD_STABLE_CI_BASE_IMAGE, - "VLLM_CI_BASE_IMAGE": AMD_STABLE_CI_BASE_IMAGE, + "DOCKER_IMAGE_NAME": AMD_BUILD_CI_BASE_IMAGE, + "VLLM_CI_BASE_IMAGE": AMD_BUILD_CI_BASE_IMAGE, "VLLM_CI_FALLBACK_IMAGE": AMD_FALLBACK_CI_IMAGE, "VLLM_CI_USE_ARTIFACTS": "1", "VLLM_CI_ARTIFACT_GLOB": AMD_ARTIFACT_GLOB, @@ -429,7 +423,7 @@ def build_amd_step_options( } plugins = [ get_amd_k8s_plugin( - image=AMD_NATIVE_BASE_IMAGE, + image=AMD_BUILD_CI_BASE_IMAGE, gpu_count=gpu_count, workspace=AMD_NATIVE_WORKSPACE, workspace_volume_name=AMD_NATIVE_WORKSPACE_VOLUME, diff --git a/buildkite/pipeline_generator/buildkite_step.py b/buildkite/pipeline_generator/buildkite_step.py index 217f7ae06..d439af949 100644 --- a/buildkite/pipeline_generator/buildkite_step.py +++ b/buildkite/pipeline_generator/buildkite_step.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel +from pydantic import BaseModel, Field, field_validator, model_validator from typing import Dict, List, Optional, Any, Union, Literal from copy import deepcopy import os @@ -162,9 +162,27 @@ class BuildkiteCommandStep(BaseModel): plugins: Optional[List[Dict[str, Any]]] = None env: Optional[Dict[str, str]] = None parallelism: Optional[int] = None + concurrency: Optional[int] = Field(default=None, gt=0, strict=True) + concurrency_group: Optional[str] = Field(default=None, min_length=1) + if_condition: Optional[str] = Field(default=None, serialization_alias="if") timeout_in_minutes: Optional[int] = None priority: Optional[int] = None + @field_validator("concurrency_group") + @classmethod + def validate_concurrency_group(cls, value: Optional[str]) -> Optional[str]: + if value is not None and not value.strip(): + raise ValueError("'concurrency_group' must be a nonempty string.") + return value + + @model_validator(mode="after") + def validate_concurrency_pair(self): + if (self.concurrency is None) != (self.concurrency_group is None): + raise ValueError( + "'concurrency' and 'concurrency_group' must be defined together." + ) + return self + def to_yaml(self): return { "label": self.label, @@ -178,6 +196,9 @@ def to_yaml(self): "plugins": self.plugins, "env": self.env, "parallelism": self.parallelism, + "concurrency": self.concurrency, + "concurrency_group": self.concurrency_group, + "if": self.if_condition, "timeout_in_minutes": self.timeout_in_minutes, "priority": self.priority, } @@ -530,6 +551,9 @@ def convert_group_step_to_buildkite_step( num_nodes=step.num_nodes, soft_fail=step.soft_fail, parallelism=step.parallelism, + concurrency=step.concurrency, + concurrency_group=step.concurrency_group, + if_condition=step.if_condition, timeout_in_minutes=step.timeout_in_minutes, agent_tags=step.agent_tags, ) @@ -554,6 +578,9 @@ def convert_group_step_to_buildkite_step( soft_fail=step.soft_fail, agents=_get_step_agents(step), priority=1000 if os.getenv("PRIORITY", "") == "HIGH" else 0, + concurrency=step.concurrency, + concurrency_group=step.concurrency_group, + if_condition=step.if_condition, ) if step.env: @@ -669,6 +696,9 @@ def convert_group_step_to_buildkite_step( num_nodes=amd.get("num_nodes", step.num_nodes), soft_fail=amd.get("soft_fail", step.soft_fail or False), parallelism=step.parallelism, + concurrency=step.concurrency, + concurrency_group=step.concurrency_group, + if_condition=step.if_condition, timeout_in_minutes=amd.get("timeout_in_minutes"), agent_tags=amd.get("agent_tags"), ) @@ -792,6 +822,9 @@ def _create_amd_step( num_nodes: Optional[int], soft_fail: Optional[bool], parallelism: Optional[int], + concurrency: Optional[int], + concurrency_group: Optional[str], + if_condition: Optional[str], key: Optional[str] = None, timeout_in_minutes: Optional[int] = None, agent_tags: Optional[Dict[str, str]] = None, @@ -815,6 +848,9 @@ def _create_amd_step( key=key, soft_fail=soft_fail or False, parallelism=parallelism, + concurrency=concurrency, + concurrency_group=concurrency_group, + if_condition=if_condition, timeout_in_minutes=_get_timeout_in_minutes( get_amd_timeout_in_minutes(timeout_in_minutes) ), diff --git a/buildkite/pipeline_generator/pipeline_generator.py b/buildkite/pipeline_generator/pipeline_generator.py index ae0ab0ea4..e7f028229 100644 --- a/buildkite/pipeline_generator/pipeline_generator.py +++ b/buildkite/pipeline_generator/pipeline_generator.py @@ -75,7 +75,7 @@ def generate(self): buildkite_steps_dict = {"steps": []} for buildkite_group_step in buildkite_group_steps: buildkite_steps_dict["steps"].append( - buildkite_group_step.dict(exclude_none=True) + buildkite_group_step.model_dump(exclude_none=True, by_alias=True) ) with open(self.output_file_path, "w") as f: yaml.dump( diff --git a/buildkite/pipeline_generator/step.py b/buildkite/pipeline_generator/step.py index 315d5ffd0..7393f161b 100644 --- a/buildkite/pipeline_generator/step.py +++ b/buildkite/pipeline_generator/step.py @@ -1,6 +1,5 @@ -from pydantic import BaseModel +from pydantic import BaseModel, Field, field_validator, model_validator from typing import Optional, List, Dict, Any -from pydantic import model_validator from typing_extensions import Self from collections import defaultdict from global_config import get_global_config @@ -22,6 +21,9 @@ class Step(BaseModel): source_file_dependencies: Optional[List[str]] = None soft_fail: Optional[bool] = False parallelism: Optional[int] = None + concurrency: Optional[int] = Field(default=None, gt=0, strict=True) + concurrency_group: Optional[str] = Field(default=None, min_length=1) + if_condition: Optional[str] = None timeout_in_minutes: Optional[int] = None mount_buildkite_agent: Optional[bool] = False env: Optional[Dict[str, str]] = None @@ -36,8 +38,19 @@ class Step(BaseModel): def validate_multi_node(self) -> Self: if self.num_nodes and not self.num_devices: raise ValueError("'num_devices' must be defined if 'num_nodes' is defined.") + if (self.concurrency is None) != (self.concurrency_group is None): + raise ValueError( + "'concurrency' and 'concurrency_group' must be defined together." + ) return self + @field_validator("concurrency_group") + @classmethod + def validate_concurrency_group(cls, value: Optional[str]) -> Optional[str]: + if value is not None and not value.strip(): + raise ValueError("'concurrency_group' must be a nonempty string.") + return value + @classmethod def from_yaml(cls, yaml_data: dict): return cls(**yaml_data) @@ -74,7 +87,9 @@ def read_steps_from_job_dir(job_dir: str): and global_config["github_repo_name"] == "vllm-project/vllm" ): step.working_dir = "/vllm-workspace/tests" - step.source_file_dependencies = getattr(step, "source_file_dependencies", []) + step.source_file_dependencies = getattr( + step, "source_file_dependencies", [] + ) if not step.source_file_dependencies: step.source_file_dependencies = [] step.source_file_dependencies.append(os.path.relpath(yaml_path)) diff --git a/buildkite/tests/pipeline_generator/test_amd.py b/buildkite/tests/pipeline_generator/test_amd.py index 91bcb44cb..3b9e7ab90 100644 --- a/buildkite/tests/pipeline_generator/test_amd.py +++ b/buildkite/tests/pipeline_generator/test_amd.py @@ -11,9 +11,7 @@ def test_legacy_amd_template_retries_gpu_hang_abort(): - template = ( - Path(__file__).parents[2] / "test-template-amd.j2" - ).read_text() + template = (Path(__file__).parents[2] / "test-template-amd.j2").read_text() assert ( "{{ indent }} - exit_status: 134 # ROCm/KFD GPU hang (SIGABRT)\n" @@ -40,19 +38,25 @@ def _rocm_base_refresh_step(): ) +def test_amd_ci_base_consumer_image_is_build_scoped(): + assert ( + amd.AMD_BUILD_CI_BASE_IMAGE == "rocm/vllm-dev:ci_base-build-$BUILDKITE_BUILD_ID" + ) + + @pytest.mark.parametrize( ("list_file_diff", "expected_timeout"), [ - ([], 15), - (["vllm/config.py"], 15), - (["run_all"], 15), - (["nightly"], 15), - ([amd.AMD_ROCM_BASE_DOCKERFILE], 540), - (["run_all", amd.AMD_ROCM_BASE_DOCKERFILE], 540), - (["nightly", amd.AMD_ROCM_BASE_DOCKERFILE], 540), + ([], 540), + (["vllm/config.py"], 540), + (["run_all"], 540), + (["nightly"], 540), + (["docker/Dockerfile.rocm_base"], 540), + (["run_all", "docker/Dockerfile.rocm_base"], 540), + (["nightly", "docker/Dockerfile.rocm_base"], 540), ], ) -def test_rocm_base_refresh_timeout_tracks_dockerfile_change( +def test_rocm_base_refresh_has_self_healing_build_timeout( fake_global_config, list_file_diff, expected_timeout ): fake_global_config["list_file_diff"] = list_file_diff @@ -60,21 +64,35 @@ def test_rocm_base_refresh_timeout_tracks_dockerfile_change( command_step = _render_single_step(_rocm_base_refresh_step()).steps[0] assert command_step.timeout_in_minutes == expected_timeout + assert command_step.env["ROCM_BASE_REFRESH_FORCE"] == "0" -def test_rocm_base_refresh_force_uses_build_timeout(monkeypatch): +def test_rocm_stable_promotion_always_runs(fake_global_config): + fake_global_config["list_file_diff"] = ["vllm/config.py"] + step = Step( + label="AMD: promote stable ROCm images", + group="Hardware - AMD Build", + key=amd.AMD_STABLE_IMAGE_PROMOTION_STEP_KEY, + device="amd_cpu", + no_plugin=True, + commands=["promote"], + ) + + assert buildkite_step._step_should_run(step, fake_global_config["list_file_diff"]) + + +def test_rocm_base_refresh_force_continues_to_pass_through(monkeypatch): monkeypatch.setenv("ROCM_BASE_REFRESH_FORCE", "1") command_step = _render_single_step(_rocm_base_refresh_step()).steps[0] + assert command_step.env["ROCM_BASE_REFRESH_FORCE"] == "1" assert command_step.timeout_in_minutes == 540 -def test_skip_timeout_omits_rocm_base_refresh_timeout( - fake_global_config, monkeypatch -): +def test_skip_timeout_omits_rocm_base_refresh_timeout(fake_global_config, monkeypatch): monkeypatch.setenv(buildkite_step.SKIP_TIMEOUT_ENV_VAR, "1") - fake_global_config["list_file_diff"] = [amd.AMD_ROCM_BASE_DOCKERFILE] + fake_global_config["list_file_diff"] = ["docker/Dockerfile.rocm_base"] command_step = _render_single_step(_rocm_base_refresh_step()).steps[0] @@ -91,9 +109,7 @@ def test_skip_timeout_omits_rocm_base_refresh_timeout( ("mi325_1", AgentQueue.AMD_MI325_1, True, "1"), ], ) -def test_direct_amd_gpu_steps_use_dind_flag( - device, queue, dind, expected_gpu_count -): +def test_direct_amd_gpu_steps_use_dind_flag(device, queue, dind, expected_gpu_count): step = Step( label="AMD direct test", group="Direct AMD", @@ -103,6 +119,9 @@ def test_direct_amd_gpu_steps_use_dind_flag( dind=dind, optional=True, soft_fail=True, + concurrency=2, + concurrency_group="vllm/test/direct-amd", + if_condition="build.branch == pipeline.default_branch", working_dir="/vllm-workspace/tests", commands=["pytest tests/foo.py"], ) @@ -116,6 +135,9 @@ def test_direct_amd_gpu_steps_use_dind_flag( assert command_step.label == f"AMD: AMD direct test ({device})" assert command_step.depends_on == ["image-build-amd", block_step.key] assert command_step.agents == {"queue": queue} + assert command_step.concurrency == 2 + assert command_step.concurrency_group == "vllm/test/direct-amd" + assert command_step.if_condition == "build.branch == pipeline.default_branch" assert command_step.commands == [ "bash .buildkite/scripts/hardware_ci/run-amd-test.sh", ] @@ -123,6 +145,7 @@ def test_direct_amd_gpu_steps_use_dind_flag( assert command_step.plugins is not None pod_patch = command_step.plugins[0]["kubernetes"]["podSpecPatch"] container = pod_patch["containers"][0] + assert container["image"] == amd.AMD_BUILD_CI_BASE_IMAGE assert container["resources"]["limits"]["amd.com/gpu"] == expected_gpu_count assert container["resources"]["requests"]["amd.com/gpu"] == expected_gpu_count assert command_step.env["AMD_CI_RUNTIME"] == "native" @@ -131,7 +154,9 @@ def test_direct_amd_gpu_steps_use_dind_flag( else: assert command_step.plugins is None assert "AMD_CI_RUNTIME" not in command_step.env - assert command_step.env["DOCKER_IMAGE_NAME"] == amd.AMD_STABLE_CI_BASE_IMAGE + assert command_step.env["DOCKER_IMAGE_NAME"] == amd.AMD_BUILD_CI_BASE_IMAGE + + assert command_step.env["VLLM_CI_BASE_IMAGE"] == amd.AMD_BUILD_CI_BASE_IMAGE assert command_step.retry == amd.AMD_RETRY assert len(command_step.retry["automatic"]) == 7 @@ -306,7 +331,7 @@ def test_untagged_mirror_defaults_to_dind( amd_command_step = amd_group.steps[1] assert isinstance(amd_command_step, buildkite_step.BuildkiteCommandStep) assert amd_command_step.plugins is None - assert amd_command_step.env["DOCKER_IMAGE_NAME"] == amd.AMD_STABLE_CI_BASE_IMAGE + assert amd_command_step.env["DOCKER_IMAGE_NAME"] == amd.AMD_BUILD_CI_BASE_IMAGE @pytest.mark.parametrize( diff --git a/buildkite/tests/pipeline_generator/test_step.py b/buildkite/tests/pipeline_generator/test_step.py index b4227e68e..511700cb8 100644 --- a/buildkite/tests/pipeline_generator/test_step.py +++ b/buildkite/tests/pipeline_generator/test_step.py @@ -319,6 +319,66 @@ def test_timeout_in_minutes_propagates_to_command_step(): assert command_step.timeout_in_minutes == 42 +def test_concurrency_fields_propagate_from_yaml_to_command_step(): + step = Step.from_yaml( + { + "label": "Serialized promotion", + "group": "Image promotion", + "key": "serialized-promotion", + "commands": ["promote-image"], + "device": "amd_cpu", + "concurrency": 1, + "concurrency_group": "rocm-stable-image-promotion", + "if_condition": ( + "build.branch == pipeline.default_branch " + "&& build.pull_request.id == null" + ), + } + ) + + group_step = _render_single_step(step) + command_step = next( + rendered_step + for rendered_step in group_step.steps + if isinstance(rendered_step, buildkite_step.BuildkiteCommandStep) + ) + rendered_group = group_step.model_dump(exclude_none=True, by_alias=True) + rendered = next( + rendered_step + for rendered_step in rendered_group["steps"] + if rendered_step.get("key") == command_step.key + ) + + assert rendered["concurrency"] == 1 + assert rendered["concurrency_group"] == "rocm-stable-image-promotion" + assert rendered["if"] == ( + "build.branch == pipeline.default_branch && build.pull_request.id == null" + ) + + +@pytest.mark.parametrize( + "yaml_fields", + [ + {"concurrency": 0}, + {"concurrency": -1}, + {"concurrency": True}, + {"concurrency": 1}, + {"concurrency_group": ""}, + {"concurrency_group": " "}, + {"concurrency_group": "missing-limit"}, + ], +) +def test_step_rejects_invalid_concurrency_fields(yaml_fields): + with pytest.raises(ValueError): + Step.from_yaml( + { + "label": "Invalid serialization", + "commands": ["true"], + **yaml_fields, + } + ) + + def test_skip_timeout_omits_timeout_from_command_step(monkeypatch): monkeypatch.setenv(buildkite_step.SKIP_TIMEOUT_ENV_VAR, "1") step = Step( @@ -337,7 +397,10 @@ def test_skip_timeout_omits_timeout_from_command_step(monkeypatch): ) assert command_step.timeout_in_minutes is None - assert "timeout_in_minutes" not in command_step.model_dump(exclude_none=True) + rendered = command_step.model_dump(exclude_none=True) + assert "timeout_in_minutes" not in rendered + assert "concurrency" not in rendered + assert "concurrency_group" not in rendered def test_missing_timeout_in_minutes_is_omitted_from_pipeline(): From 3f844500e55d2c56f4e60fb2745381827b04b60a Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 4 Aug 2026 16:31:14 +0000 Subject: [PATCH 2/5] [CI][AMD] Gate build-scoped ROCm handoff Signed-off-by: Andreas Karatzas --- buildkite/bootstrap-amd.sh | 8 +++ buildkite/pipeline_generator/amd.py | 49 +++++++++++++---- buildkite/test-template-amd.j2 | 47 ++++++++++++++--- .../tests/pipeline_generator/test_amd.py | 52 +++++++++++++++++-- pyproject.toml | 4 ++ 5 files changed, 140 insertions(+), 20 deletions(-) diff --git a/buildkite/bootstrap-amd.sh b/buildkite/bootstrap-amd.sh index f62c0265e..9af483935 100644 --- a/buildkite/bootstrap-amd.sh +++ b/buildkite/bootstrap-amd.sh @@ -136,7 +136,14 @@ get_diff_main() { # upload_pipeline: render and upload the Buildkite pipeline YAML # --------------------------------------------------------------------------- upload_pipeline() { + local rocm_build_scoped_handoff=0 + + if [[ -f ".buildkite/scripts/rocm/promote-stable-images.sh" ]]; then + rocm_build_scoped_handoff=1 + fi + echo "Uploading pipeline..." + echo "ROCm build-scoped image handoff: ${rocm_build_scoped_handoff}" # Install minijinja ls .buildkite || buildkite-agent annotate --style error 'Please merge upstream main branch for buildkite CI' curl -sSfL https://github.com/mitsuhiko/minijinja/releases/download/2.3.1/minijinja-cli-installer.sh | sh @@ -190,6 +197,7 @@ upload_pipeline() { -D rocm_base_refresh_skip="${ROCM_BASE_REFRESH_SKIP:-0}" \ -D rocm_base_refresh_force="${ROCM_BASE_REFRESH_FORCE:-0}" \ -D rocm_base_refresh_diff_unavailable="${ROCM_BASE_REFRESH_DIFF_UNAVAILABLE:-0}" \ + -D rocm_build_scoped_handoff="${rocm_build_scoped_handoff}" \ | sed '/^[[:space:]]*$/d' \ > pipeline.yaml ) diff --git a/buildkite/pipeline_generator/amd.py b/buildkite/pipeline_generator/amd.py index d57f55111..1c96dc780 100644 --- a/buildkite/pipeline_generator/amd.py +++ b/buildkite/pipeline_generator/amd.py @@ -1,13 +1,18 @@ import os from copy import deepcopy from dataclasses import dataclass +from pathlib import Path from typing import Any, Dict, List, Mapping, Optional, TypedDict from constants import AgentQueue, DeviceType AMD_TEST_COMMAND = "bash .buildkite/scripts/hardware_ci/run-amd-test.sh" +AMD_STABLE_CI_BASE_IMAGE = "rocm/vllm-dev:ci_base" +AMD_NATIVE_BASE_IMAGE = "rocm/vllm-dev:ci_base-$BUILDKITE_COMMIT" AMD_BUILD_CI_BASE_IMAGE = "rocm/vllm-dev:ci_base-build-$BUILDKITE_BUILD_ID" +AMD_ROCM_PROMOTION_SCRIPT = Path(".buildkite/scripts/rocm/promote-stable-images.sh") AMD_FALLBACK_CI_IMAGE = "rocm/vllm-ci:$BUILDKITE_COMMIT" +AMD_BUILD_FALLBACK_CI_IMAGE = "rocm/vllm-ci:build-$BUILDKITE_BUILD_ID" AMD_ARTIFACT_GLOB = "artifacts/vllm-rocm-install/vllm-rocm-install.tar.gz" AMD_ARTIFACT_CHECKSUM_GLOB = f"{AMD_ARTIFACT_GLOB}.sha256" AMD_ARTIFACT_STEP = "image-build-amd" @@ -22,6 +27,7 @@ AMD_STANDARD_TIMEOUT_MINUTES = 3 * 60 AMD_ROCM_BASE_REFRESH_STEP_KEY = "refresh-rocm-base-amd" AMD_STABLE_IMAGE_PROMOTION_STEP_KEY = "promote-stable-rocm-images-amd" +AMD_ROCM_BASE_DOCKERFILE = "docker/Dockerfile.rocm_base" AMD_ROCM_BASE_REFRESH_TIMEOUT_MINUTES = 9 * 60 AMD_ROCM_BASE_REFRESH_NOOP_TIMEOUT_MINUTES = 15 AMD_ALWAYS_RUN_STEP_KEYS = frozenset( @@ -48,6 +54,25 @@ ROCM_DEBUG_AGENT_LIB = "/opt/rocm/lib/librocm-debug-agent.so.2" +def supports_build_scoped_rocm_images() -> bool: + """Whether the checked-out vLLM revision publishes build-scoped aliases.""" + return AMD_ROCM_PROMOTION_SCRIPT.is_file() + + +def get_amd_ci_base_image(*, dind: bool) -> str: + if supports_build_scoped_rocm_images(): + return AMD_BUILD_CI_BASE_IMAGE + if dind: + return AMD_STABLE_CI_BASE_IMAGE + return AMD_NATIVE_BASE_IMAGE + + +def get_amd_fallback_ci_image() -> str: + if supports_build_scoped_rocm_images(): + return AMD_BUILD_FALLBACK_CI_IMAGE + return AMD_FALLBACK_CI_IMAGE + + def get_amd_timeout_in_minutes(timeout_in_minutes: Optional[int]) -> int: """Default and cap standard AMD jobs at three hours.""" requested_timeout = ( @@ -94,12 +119,16 @@ def ensure_amd_stack_error_retry( return retry_policy -def get_rocm_base_refresh_timeout(_list_file_diff: List[str]) -> int: +def get_rocm_base_refresh_timeout(list_file_diff: List[str]) -> int: if os.getenv("ROCM_BASE_REFRESH_SKIP", "0") == "1": return AMD_ROCM_BASE_REFRESH_NOOP_TIMEOUT_MINUTES - # Every selection performs an exact registry lookup. A missing tag or a - # changed parent digest can turn any lookup into a self-healing build. - return AMD_ROCM_BASE_REFRESH_TIMEOUT_MINUTES + if supports_build_scoped_rocm_images() or ( + os.getenv("ROCM_BASE_REFRESH_FORCE", "0") == "1" + or os.getenv("ROCM_BASE_REFRESH_DIFF_UNAVAILABLE", "0") == "1" + or AMD_ROCM_BASE_DOCKERFILE in list_file_diff + ): + return AMD_ROCM_BASE_REFRESH_TIMEOUT_MINUTES + return AMD_ROCM_BASE_REFRESH_NOOP_TIMEOUT_MINUTES def get_rocm_base_refresh_env() -> Dict[str, str]: @@ -275,6 +304,7 @@ def _get_amd_env( gpu_count: int, ) -> Dict[str, str]: env = dict(extra_env or {}) + ci_base_image = get_amd_ci_base_image(dind=dind) if not dind: # Native agents have no DinD sidecar, so Docker hook inputs must not # escape into this execution mode. @@ -289,7 +319,7 @@ def _get_amd_env( env.setdefault("HF_HUB_ETAG_TIMEOUT", "60") env.update( { - "VLLM_CI_BASE_IMAGE": AMD_BUILD_CI_BASE_IMAGE, + "VLLM_CI_BASE_IMAGE": ci_base_image, "VLLM_CI_USE_ARTIFACTS": "1", "VLLM_CI_ARTIFACT_GLOB": AMD_ARTIFACT_GLOB, "VLLM_CI_RESULTS_ROOT": AMD_RESULTS_ROOT, @@ -307,9 +337,9 @@ def _get_amd_env( env.update( { "DOCKER_BUILDKIT": "1", - "DOCKER_IMAGE_NAME": AMD_BUILD_CI_BASE_IMAGE, - "VLLM_CI_BASE_IMAGE": AMD_BUILD_CI_BASE_IMAGE, - "VLLM_CI_FALLBACK_IMAGE": AMD_FALLBACK_CI_IMAGE, + "DOCKER_IMAGE_NAME": ci_base_image, + "VLLM_CI_BASE_IMAGE": ci_base_image, + "VLLM_CI_FALLBACK_IMAGE": get_amd_fallback_ci_image(), "VLLM_CI_USE_ARTIFACTS": "1", "VLLM_CI_ARTIFACT_GLOB": AMD_ARTIFACT_GLOB, "VLLM_CI_RESULTS_ROOT": AMD_RESULTS_ROOT, @@ -412,6 +442,7 @@ def build_amd_step_options( gpu_count = resolve_amd_gpu_count(device, num_devices, no_gpu) plugins = None if not dind: + ci_base_image = get_amd_ci_base_image(dind=False) container_env = { "AMD_CI_RUNTIME": "native", "NATIVE_CI": "true", @@ -423,7 +454,7 @@ def build_amd_step_options( } plugins = [ get_amd_k8s_plugin( - image=AMD_BUILD_CI_BASE_IMAGE, + image=ci_base_image, gpu_count=gpu_count, workspace=AMD_NATIVE_WORKSPACE, workspace_volume_name=AMD_NATIVE_WORKSPACE_VOLUME, diff --git a/buildkite/test-template-amd.j2 b/buildkite/test-template-amd.j2 index 32e226323..0cc1c0362 100644 --- a/buildkite/test-template-amd.j2 +++ b/buildkite/test-template-amd.j2 @@ -2,13 +2,13 @@ {% set default_amd_timeout_in_minutes = 180 %} {% set list_file_diff = list_file_diff | replace("\r", "") | replace("\n", "|") | split("|") %} {% set rocm_base_refresh_needs_build = rocm_base_refresh_force == "1" or rocm_base_refresh_diff_unavailable == "1" or "docker/Dockerfile.rocm_base" in list_file_diff %} -{% set rocm_base_refresh_timeout_in_minutes = 15 if rocm_base_refresh_skip == "1" else (540 if rocm_base_refresh_needs_build else 15) %} +{% set rocm_base_refresh_timeout_in_minutes = 15 if rocm_base_refresh_skip == "1" else (540 if rocm_build_scoped_handoff == "1" or rocm_base_refresh_needs_build else 15) %} {# TODO: Re-enable after validating the ROCm debug agent setup. {% set rocm_debug_agent_setup = "if test -f /opt/rocm/lib/librocm-debug-agent.so.2; then export HSA_TOOLS_LIB=/opt/rocm/lib/librocm-debug-agent.so.2 && export HSA_ENABLE_DEBUG=1 && echo ROCm debug agent enabled: /opt/rocm/lib/librocm-debug-agent.so.2; else echo 'WARNING: ROCm debug agent not found at /opt/rocm/lib/librocm-debug-agent.so.2; skipping coredump setup'; fi" %} #} {% set rocm_debug_agent_setup = "echo 'ROCm debug agent disabled; re-enable rocm_debug_agent_setup in test-template-amd.j2 after validating coredump setup'" %} -{% macro amd_infra_retry(indent='', retry_exit_status_one=false) -%} +{% macro amd_infra_retry(indent='', retry_exit_status_one=false, retry_exit_status_two=false) -%} {{ indent }}retry: {{ indent }} automatic: {{ indent }} - signal_reason: stack_error # Kubernetes pod/stack provisioning failed @@ -17,6 +17,10 @@ {{ indent }} - exit_status: 1 # Transient Docker/BuildKit failure {{ indent }} limit: 1 {% endif %} +{% if retry_exit_status_two %} +{{ indent }} - exit_status: 2 # Transient git/registry identity failure +{{ indent }} limit: 1 +{% endif %} {{ indent }} - signal_reason: agent_stop # Agent host stopped unexpectedly {{ indent }} limit: 1 {{ indent }} - signal_reason: agent_refused # Agent could not accept the job @@ -34,7 +38,7 @@ {# Native in-pod ROCm CI: per-job container-0 image via kubernetes podSpecPatch. #} {% macro amd_native_base_image() -%} -rocm/vllm-dev:ci_base-$BUILDKITE_COMMIT +{{ "rocm/vllm-dev:ci_base-build-$BUILDKITE_BUILD_ID" if rocm_build_scoped_handoff == "1" else "rocm/vllm-dev:ci_base-$BUILDKITE_COMMIT" }} {%- endmacro %} {% macro amd_native_k8s_plugin(step) %} @@ -115,7 +119,7 @@ rocm/vllm-dev:ci_base-$BUILDKITE_COMMIT DOCKER_BUILDKIT: "1" BUILDKIT_PROGRESS: "tty" TERM: "xterm-256color" -{{ amd_infra_retry(' ', true) }} +{{ amd_infra_retry(' ', true, true) }} agents: queue: amd-cpu @@ -151,13 +155,34 @@ rocm/vllm-dev:ci_base-$BUILDKITE_COMMIT TERM: "xterm-256color" VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl" PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950" + {% if rocm_build_scoped_handoff == "1" %} + IMAGE_TAG: "rocm/vllm-ci:build-$BUILDKITE_BUILD_ID" + IMAGE_TAG_LATEST: "rocm/vllm-ci:$BUILDKITE_COMMIT" + VLLM_CI_SMOKE_IMAGE: "rocm/vllm-ci:build-$BUILDKITE_BUILD_ID" + {% else %} IMAGE_TAG: "rocm/vllm-ci:$BUILDKITE_COMMIT" - REMOTE_VLLM: "1" - VLLM_BRANCH: "$BUILDKITE_COMMIT" + {% endif %} + REMOTE_VLLM: "0" {{ amd_infra_retry(' ', true) }} agents: queue: amd-cpu + {% if rocm_build_scoped_handoff == "1" %} + - label: "AMD: :docker: promote stable ROCm images" + depends_on: "image-build-amd" + soft_fail: false + commands: + - bash .buildkite/scripts/rocm/promote-stable-images.sh + key: "promote-stable-rocm-images-amd" + if: build.branch == pipeline.default_branch && build.pull_request.id == null + concurrency: 1 + concurrency_group: "vllm/rocm/stable-image-promotion" + timeout_in_minutes: 30 +{{ amd_infra_retry(' ', true) }} + agents: + queue: amd-cpu + {% endif %} + {% for step in steps %} {% if step.mirror_hardwares and mirror_hw in step.mirror_hardwares %} @@ -230,7 +255,7 @@ rocm/vllm-dev:ci_base-$BUILDKITE_COMMIT NUM_NODES: "{{ step.num_nodes or 1 }}" # Agent hooks read DOCKER_IMAGE_NAME before run-amd-test.py starts. {% if step.dind == false %} - # Native: podSpecPatch sets container-0 to ci_base- from ensure-ci-base-amd. + # Native: podSpecPatch sets container-0 to the selected ci_base alias. VLLM_CI_BASE_IMAGE: "{{ amd_native_base_image() | safe }}" AMD_CI_RUNTIME: "native" NATIVE_CI: "true" @@ -246,12 +271,18 @@ rocm/vllm-dev:ci_base-$BUILDKITE_COMMIT HF_HUB_ETAG_TIMEOUT: "60" PYTORCH_ROCM_ARCH: "" {% else %} - # DinD: hook warm-up uses stable ci_base; docker run uses fallback image if needed. + # DinD: use the build-scoped handoff when the checkout publishes it. DOCKER_BUILDKIT: "1" + {% if rocm_build_scoped_handoff == "1" %} + DOCKER_IMAGE_NAME: "rocm/vllm-dev:ci_base-build-$BUILDKITE_BUILD_ID" + VLLM_CI_BASE_IMAGE: "rocm/vllm-dev:ci_base-build-$BUILDKITE_BUILD_ID" + VLLM_CI_FALLBACK_IMAGE: "rocm/vllm-ci:build-$BUILDKITE_BUILD_ID" + {% else %} DOCKER_IMAGE_NAME: "rocm/vllm-dev:ci_base" VLLM_CI_BASE_IMAGE: "rocm/vllm-dev:ci_base" VLLM_CI_FALLBACK_IMAGE: "rocm/vllm-ci:$BUILDKITE_COMMIT" {% endif %} + {% endif %} VLLM_CI_USE_ARTIFACTS: "1" VLLM_CI_ARTIFACT_GLOB: "artifacts/vllm-rocm-install/vllm-rocm-install.tar.gz" VLLM_CI_RESULTS_ROOT: "/home/buildkite-agent/huggingface/amd-ci-results" diff --git a/buildkite/tests/pipeline_generator/test_amd.py b/buildkite/tests/pipeline_generator/test_amd.py index 3b9e7ab90..5ba97e137 100644 --- a/buildkite/tests/pipeline_generator/test_amd.py +++ b/buildkite/tests/pipeline_generator/test_amd.py @@ -10,13 +10,36 @@ pytestmark = pytest.mark.usefixtures("fake_global_config") -def test_legacy_amd_template_retries_gpu_hang_abort(): +@pytest.fixture(autouse=True) +def build_scoped_rocm_capability(tmp_path, monkeypatch): + promotion_script = tmp_path / "promote-stable-images.sh" + promotion_script.touch() + monkeypatch.setattr(amd, "AMD_ROCM_PROMOTION_SCRIPT", promotion_script) + + +def test_amd_template_and_bootstrap_preserve_build_scoped_contract(): template = (Path(__file__).parents[2] / "test-template-amd.j2").read_text() + bootstrap = (Path(__file__).parents[2] / "bootstrap-amd.sh").read_text() assert ( "{{ indent }} - exit_status: 134 # ROCm/KFD GPU hang (SIGABRT)\n" "{{ indent }} limit: 1" ) in template + assert "exit_status: 2 # Transient git/registry identity failure" in template + assert "ci_base-build-$BUILDKITE_BUILD_ID" in template + assert 'key: "promote-stable-rocm-images-amd"' in template + assert "if: build.branch == pipeline.default_branch" in template + assert 'concurrency_group: "vllm/rocm/stable-image-promotion"' in template + assert 'IMAGE_TAG: "rocm/vllm-ci:build-$BUILDKITE_BUILD_ID"' in template + assert 'IMAGE_TAG_LATEST: "rocm/vllm-ci:$BUILDKITE_COMMIT"' in template + assert 'VLLM_CI_SMOKE_IMAGE: "rocm/vllm-ci:build-$BUILDKITE_BUILD_ID"' in template + assert ( + 'VLLM_CI_FALLBACK_IMAGE: "rocm/vllm-ci:build-$BUILDKITE_BUILD_ID"' in template + ) + assert 'REMOTE_VLLM: "0"' in template + assert 'REMOTE_VLLM: "1"' not in template + assert '[[ -f ".buildkite/scripts/rocm/promote-stable-images.sh" ]]' in bootstrap + assert "-D rocm_build_scoped_handoff=" in bootstrap def _render_single_step(step): @@ -38,10 +61,16 @@ def _rocm_base_refresh_step(): ) -def test_amd_ci_base_consumer_image_is_build_scoped(): +def test_amd_ci_base_consumer_image_is_capability_gated(monkeypatch, tmp_path): assert ( - amd.AMD_BUILD_CI_BASE_IMAGE == "rocm/vllm-dev:ci_base-build-$BUILDKITE_BUILD_ID" + amd.get_amd_ci_base_image(dind=False) + == "rocm/vllm-dev:ci_base-build-$BUILDKITE_BUILD_ID" ) + assert amd.get_amd_fallback_ci_image() == amd.AMD_BUILD_FALLBACK_CI_IMAGE + monkeypatch.setattr(amd, "AMD_ROCM_PROMOTION_SCRIPT", tmp_path / "missing") + assert amd.get_amd_ci_base_image(dind=False) == amd.AMD_NATIVE_BASE_IMAGE + assert amd.get_amd_ci_base_image(dind=True) == amd.AMD_STABLE_CI_BASE_IMAGE + assert amd.get_amd_fallback_ci_image() == amd.AMD_FALLBACK_CI_IMAGE @pytest.mark.parametrize( @@ -67,6 +96,19 @@ def test_rocm_base_refresh_has_self_healing_build_timeout( assert command_step.env["ROCM_BASE_REFRESH_FORCE"] == "0" +def test_legacy_rocm_base_refresh_keeps_diff_based_timeout( + fake_global_config, monkeypatch, tmp_path +): + monkeypatch.setattr(amd, "AMD_ROCM_PROMOTION_SCRIPT", tmp_path / "missing") + fake_global_config["list_file_diff"] = ["vllm/config.py"] + command_step = _render_single_step(_rocm_base_refresh_step()).steps[0] + assert command_step.timeout_in_minutes == 15 + + fake_global_config["list_file_diff"] = [amd.AMD_ROCM_BASE_DOCKERFILE] + command_step = _render_single_step(_rocm_base_refresh_step()).steps[0] + assert command_step.timeout_in_minutes == 540 + + def test_rocm_stable_promotion_always_runs(fake_global_config): fake_global_config["list_file_diff"] = ["vllm/config.py"] step = Step( @@ -155,6 +197,10 @@ def test_direct_amd_gpu_steps_use_dind_flag(device, queue, dind, expected_gpu_co assert command_step.plugins is None assert "AMD_CI_RUNTIME" not in command_step.env assert command_step.env["DOCKER_IMAGE_NAME"] == amd.AMD_BUILD_CI_BASE_IMAGE + assert ( + command_step.env["VLLM_CI_FALLBACK_IMAGE"] + == amd.AMD_BUILD_FALLBACK_CI_IMAGE + ) assert command_step.env["VLLM_CI_BASE_IMAGE"] == amd.AMD_BUILD_CI_BASE_IMAGE diff --git a/pyproject.toml b/pyproject.toml index 3cbafdbb5..a3eddef75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,3 +12,7 @@ include = ["buildkite*"] [tool.pytest.ini_options] pythonpath = [".", "buildkite/pipeline_generator"] + +[tool.typos.default.extend-identifiers] +HSA_ENABLE_DEBUG = "HSA_ENABLE_DEBUG" +HSA_TOOLS_LIB = "HSA_TOOLS_LIB" From c54f2bd3cfbbd2958fae2ccf119f101bb3c3c3bd Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Fri, 14 Aug 2026 05:08:03 +0000 Subject: [PATCH 3/5] [AMD][CI] Increase native shared-memory capacity Raise the native AMD tmpfs /dev/shm limit from 16 GiB to 32 GiB in both pipeline generators. Assisted-by: OpenAI Codex Signed-off-by: Andreas Karatzas --- buildkite/pipeline_generator/amd.py | 2 +- buildkite/test-template-amd.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/pipeline_generator/amd.py b/buildkite/pipeline_generator/amd.py index 7450bb334..8cdb05577 100644 --- a/buildkite/pipeline_generator/amd.py +++ b/buildkite/pipeline_generator/amd.py @@ -44,7 +44,7 @@ def get_amd_ci_image() -> str: AMD_HF_HOME = "/home/buildkite-agent/huggingface" AMD_NATIVE_WORKSPACE = "/vllm-workspace" AMD_NATIVE_WORKSPACE_VOLUME = "vllm-workspace" -AMD_NATIVE_SHM_SIZE = "16Gi" +AMD_NATIVE_SHM_SIZE = "32Gi" AMD_NATIVE_POD_IDENTITY_ENV = { "VLLM_CI_K8S_NODE_NAME": "spec.nodeName", } diff --git a/buildkite/test-template-amd.j2 b/buildkite/test-template-amd.j2 index d57ec4f6e..a2bcf5073 100644 --- a/buildkite/test-template-amd.j2 +++ b/buildkite/test-template-amd.j2 @@ -112,7 +112,7 @@ rocm/vllm-ci:$BUILDKITE_COMMIT - name: devshm emptyDir: medium: Memory - sizeLimit: 16Gi + sizeLimit: 32Gi - name: vllm-workspace emptyDir: {} {%- endmacro %} From 32d21086b6a54030376f9bda23b81f5ed64251c1 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Mon, 17 Aug 2026 00:31:04 -0500 Subject: [PATCH 4/5] [CI][AMD] Require build-scoped promotion capability Signed-off-by: Andreas Karatzas --- buildkite/bootstrap-amd.sh | 4 +++- buildkite/pipeline_generator/amd.py | 2 +- buildkite/test-template-amd.j2 | 1 - buildkite/tests/pipeline_generator/test_amd.py | 15 +++++++++++---- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/buildkite/bootstrap-amd.sh b/buildkite/bootstrap-amd.sh index 71ec31dc4..8cd5fc278 100644 --- a/buildkite/bootstrap-amd.sh +++ b/buildkite/bootstrap-amd.sh @@ -138,7 +138,9 @@ get_diff_main() { upload_pipeline() { local rocm_stable_image_promotion=0 - if [[ -f ".buildkite/scripts/rocm/promote-stable-images.sh" ]]; then + if [[ -f ".buildkite/scripts/rocm/promote-stable-images.sh" ]] \ + && grep -q 'CI_BASE_IMAGE_TAG_BUILD_REF' \ + .buildkite/scripts/ci-bake-rocm.sh 2>/dev/null; then rocm_stable_image_promotion=1 fi diff --git a/buildkite/pipeline_generator/amd.py b/buildkite/pipeline_generator/amd.py index 8cdb05577..39c6d5d4c 100644 --- a/buildkite/pipeline_generator/amd.py +++ b/buildkite/pipeline_generator/amd.py @@ -21,7 +21,7 @@ def supports_build_scoped_images() -> bool: def supports_stable_image_promotion() -> bool: - return AMD_ROCM_PROMOTION_SCRIPT.is_file() + return AMD_ROCM_PROMOTION_SCRIPT.is_file() and supports_build_scoped_images() def get_amd_native_base_image() -> str: diff --git a/buildkite/test-template-amd.j2 b/buildkite/test-template-amd.j2 index a2bcf5073..7222a02f8 100644 --- a/buildkite/test-template-amd.j2 +++ b/buildkite/test-template-amd.j2 @@ -175,7 +175,6 @@ rocm/vllm-ci:$BUILDKITE_COMMIT PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950" {% if rocm_stable_image_promotion == "1" %} IMAGE_TAG: "rocm/vllm-ci:build-$BUILDKITE_BUILD_ID" - IMAGE_TAG_LATEST: "rocm/vllm-ci:$BUILDKITE_COMMIT" VLLM_CI_SMOKE_IMAGE: "rocm/vllm-ci:build-$BUILDKITE_BUILD_ID" {% else %} IMAGE_TAG: "{{ amd_ci_image() | safe }}" diff --git a/buildkite/tests/pipeline_generator/test_amd.py b/buildkite/tests/pipeline_generator/test_amd.py index 320aacf15..023cd3dcd 100644 --- a/buildkite/tests/pipeline_generator/test_amd.py +++ b/buildkite/tests/pipeline_generator/test_amd.py @@ -31,13 +31,17 @@ def test_stable_image_promotion_capability_uses_build_scoped_dind_base( tmp_path, monkeypatch ): monkeypatch.chdir(tmp_path) - producer = tmp_path / ".buildkite/scripts/ci-bake-rocm.sh" - producer.parent.mkdir(parents=True) - producer.write_text("CI_BASE_IMAGE_TAG_BUILD_REF=enabled") promotion_script = tmp_path / "promote-stable-images.sh" promotion_script.touch() monkeypatch.setattr(amd, "AMD_ROCM_PROMOTION_SCRIPT", promotion_script) + assert not amd.supports_stable_image_promotion() + assert amd.get_amd_ci_base_image(dind=True) == amd.AMD_STABLE_CI_BASE_IMAGE + + producer = tmp_path / ".buildkite/scripts/ci-bake-rocm.sh" + producer.parent.mkdir(parents=True) + producer.write_text("CI_BASE_IMAGE_TAG_BUILD_REF=enabled") + assert amd.supports_build_scoped_images() assert amd.supports_stable_image_promotion() assert amd.get_amd_ci_base_image(dind=False) == amd.AMD_BUILD_NATIVE_BASE_IMAGE @@ -64,7 +68,7 @@ def test_amd_template_and_bootstrap_preserve_promotion_contract(): assert "if: build.branch == pipeline.default_branch" in template assert 'concurrency_group: "vllm/rocm/stable-image-promotion"' in template assert 'IMAGE_TAG: "rocm/vllm-ci:build-$BUILDKITE_BUILD_ID"' in template - assert 'IMAGE_TAG_LATEST: "rocm/vllm-ci:$BUILDKITE_COMMIT"' in template + assert "IMAGE_TAG_LATEST:" not in template assert 'VLLM_CI_SMOKE_IMAGE: "rocm/vllm-ci:build-$BUILDKITE_BUILD_ID"' in template assert ( 'VLLM_CI_FALLBACK_IMAGE: "rocm/vllm-ci:build-$BUILDKITE_BUILD_ID"' in template @@ -72,6 +76,9 @@ def test_amd_template_and_bootstrap_preserve_promotion_contract(): assert 'REMOTE_VLLM: "0"' in template assert 'REMOTE_VLLM: "1"' not in template assert '[[ -f ".buildkite/scripts/rocm/promote-stable-images.sh" ]]' in bootstrap + promotion_gate = bootstrap.split("local rocm_stable_image_promotion=0", 1)[1] + promotion_gate = promotion_gate.split("rocm_stable_image_promotion=1", 1)[0] + assert "CI_BASE_IMAGE_TAG_BUILD_REF" in promotion_gate assert "-D rocm_build_scoped_images=" in bootstrap assert "-D rocm_stable_image_promotion=" in bootstrap From bc8d684da00af0541a541c04e194a77b036c46f1 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Mon, 21 Sep 2026 00:09:58 +0000 Subject: [PATCH 5/5] [CI][AMD] Align selector with ROCm promotion support Recognize if_condition as an upload-time Buildkite gate so conditional steps are not force-selected as unknown fields. Allow the promotion step to be absent only on vLLM checkouts without promotion support, while retaining drift checks for established AMD steps and promotion-capable checkouts. Add regression coverage for conditional-step preflight and legacy/promotion-capable checkout contracts. Signed-off-by: Andreas Karatzas --- .../ci_selector/ci_selector/handwritten.py | 3 + buildkite/ci_selector/tests/test_buildkite.py | 101 ++++++++++++++---- buildkite/ci_selector/tests/test_guards.py | 29 +++++ 3 files changed, 112 insertions(+), 21 deletions(-) diff --git a/buildkite/ci_selector/ci_selector/handwritten.py b/buildkite/ci_selector/ci_selector/handwritten.py index e5e3c6739..7eefb476e 100644 --- a/buildkite/ci_selector/ci_selector/handwritten.py +++ b/buildkite/ci_selector/ci_selector/handwritten.py @@ -410,6 +410,9 @@ "agent_pool", "concurrency", "concurrency_group", + # Emitted as Buildkite's `if`: evaluated on upload, after step selection. + # It can only restrict execution, so it does not force-select a step. + "if_condition", # Buildkite built-in, absent from the generator's Step model, so it is # dropped before reaching a command. Governs ordering after a failure. "allow_dependency_failure", diff --git a/buildkite/ci_selector/tests/test_buildkite.py b/buildkite/ci_selector/tests/test_buildkite.py index a039d98b0..335626bdf 100644 --- a/buildkite/ci_selector/tests/test_buildkite.py +++ b/buildkite/ci_selector/tests/test_buildkite.py @@ -11,6 +11,7 @@ import pytest import regex as re +from amd import AMD_STABLE_IMAGE_PROMOTION_STEP_KEY, supports_stable_image_promotion from ci_selector.codemap.pipeline.buildkite import ( _expand_mirror, load_pipeline_configs, @@ -121,8 +122,36 @@ def test_a_mirror_publishes_the_hardware_as_a_prefix(loaded): assert plain.buildkite_key == "lora" +def _assert_amd_auto_run_semantics(rocm, promotion_supported): + # Only the promotion step is optional across companion vLLM checkouts. + # Existing keys still have to exist, and a capable checkout must include + # promotion. Whenever promotion is present, verify its always-run shortcut. + for key in AMD_ALWAYS_RUN_STEP_KEYS: + if ( + key == AMD_STABLE_IMAGE_PROMOTION_STEP_KEY + and key not in rocm + and not promotion_supported + ): + continue + assert key in rocm, drift_message( + f"AMD_ALWAYS_RUN_STEP_KEYS names a step that no longer exists: {key}", + "These steps declare no dependencies, so always_runs is the only " + "thing that ever selects them. A renamed key means we stop naming " + "the step and CI stops running it.", + "we re-export this list from the generator's amd.py, so there is " + "nothing to edit here: fix the key in amd.py, or in vLLM's rocm " + "yaml if that is where it was renamed", + ) + assert rocm[key].always_runs, drift_message( + f"{key} is in AMD_ALWAYS_RUN_STEP_KEYS but does not read as always-run.", + "It would only be selected if some rule happened to pick it, and " + "these steps have no dependencies for a rule to match on.", + "check Step.always_runs against ci-infra's _step_should_run", + ) + + @pytest.mark.drift -def test_auto_run_semantics(loaded): +def test_auto_run_semantics(loaded, vllm_repo, monkeypatch): """always_runs is the generator's key shortcut (image-build*/AMD list), NOT "has no deps": a no-deps step without the shortcut never auto-runs.""" _, steps, _ = loaded @@ -153,27 +182,12 @@ def test_auto_run_semantics(loaded): assert by_key["cpu-arm64-image-build"].source_file_dependencies is None, prefix assert not by_key["cpu-arm64-image-build"].always_runs, prefix assert not by_key["arm64-image-build"].always_runs, prefix - # Membership asserted: these steps declare no deps, so always_runs - # is the only thing that ever selects them, and this is the - # only guard tying the generator's list to vLLM's yaml. Under `if key in - # rocm` a step vanishing from the yaml would skip the check in silence. + # Read capability from the vLLM checkout, not the ci-infra working directory. rocm = {s.key: s for s in steps["vllm_rocm_ci"] if s.key} - for key in AMD_ALWAYS_RUN_STEP_KEYS: - assert key in rocm, drift_message( - f"AMD_ALWAYS_RUN_STEP_KEYS names a step that no longer exists: {key}", - "These steps declare no dependencies, so always_runs is the only " - "thing that ever selects them. A renamed key means we stop naming " - "the step and CI stops running it.", - "we re-export this list from the generator's amd.py, so there is " - "nothing to edit here: fix the key in amd.py, or in vLLM's rocm " - "yaml if that is where it was renamed", - ) - assert rocm[key].always_runs, drift_message( - f"{key} is in AMD_ALWAYS_RUN_STEP_KEYS but does not read as always-run.", - "It would only be selected if some rule happened to pick it, and " - "these steps have no dependencies for a rule to match on.", - "check Step.always_runs against ci-infra's _step_should_run", - ) + with monkeypatch.context() as checkout: + checkout.chdir(vllm_repo) + promotion_supported = supports_stable_image_promotion() + _assert_amd_auto_run_semantics(rocm, promotion_supported) no_deps_no_shortcut = [ s for s in steps["vllm_ci"] @@ -367,6 +381,51 @@ def _parent_step(**kw): return Step(**{**base, **kw}) +def _amd_always_run_steps(): + return {key: _parent_step(key=key) for key in AMD_ALWAYS_RUN_STEP_KEYS} + + +@pytest.mark.parametrize("promotion_supported", [False, True]) +@pytest.mark.parametrize("promotion_present", [False, True]) +def test_amd_promotion_membership_tracks_checkout_capability( + promotion_supported, promotion_present +): + rocm = _amd_always_run_steps() + if not promotion_present: + del rocm[AMD_STABLE_IMAGE_PROMOTION_STEP_KEY] + if promotion_supported and not promotion_present: + with pytest.raises(AssertionError, match=AMD_STABLE_IMAGE_PROMOTION_STEP_KEY): + _assert_amd_auto_run_semantics(rocm, promotion_supported) + else: + _assert_amd_auto_run_semantics(rocm, promotion_supported) + + +@pytest.mark.parametrize( + "missing_key", + sorted(AMD_ALWAYS_RUN_STEP_KEYS - {AMD_STABLE_IMAGE_PROMOTION_STEP_KEY}), +) +def test_old_checkout_still_requires_existing_amd_steps(missing_key): + rocm = _amd_always_run_steps() + del rocm[AMD_STABLE_IMAGE_PROMOTION_STEP_KEY] + del rocm[missing_key] + with pytest.raises(AssertionError, match=missing_key): + _assert_amd_auto_run_semantics(rocm, promotion_supported=False) + + +@pytest.mark.parametrize("promotion_supported", [False, True]) +def test_present_amd_promotion_must_read_as_always_run( + monkeypatch, promotion_supported +): + rocm = _amd_always_run_steps() + monkeypatch.setattr( + Step, + "always_runs", + property(lambda step: step.key != AMD_STABLE_IMAGE_PROMOTION_STEP_KEY), + ) + with pytest.raises(AssertionError, match="does not read as always-run"): + _assert_amd_auto_run_semantics(rocm, promotion_supported) + + def test_a_stray_mirror_key_is_recorded_against_the_mirror_not_the_parent(): """`_expand_mirror` files unknown keys under the variant's id, so an AMD-only typo forces the AMD step and leaves its NVIDIA parent free.""" diff --git a/buildkite/ci_selector/tests/test_guards.py b/buildkite/ci_selector/tests/test_guards.py index 93dfd8796..ba4135c11 100644 --- a/buildkite/ci_selector/tests/test_guards.py +++ b/buildkite/ci_selector/tests/test_guards.py @@ -123,6 +123,35 @@ def test_unknown_field_force_selected(): assert step.step_id in pf.force_select +@pytest.mark.parametrize( + "condition", ["true", "false", "build.branch == pipeline.default_branch"] +) +def test_buildkite_condition_does_not_force_selection(tmp_path, condition): + from ci_selector.codemap.pipeline.buildkite import load_steps + + (tmp_path / "jobs").mkdir() + (tmp_path / "jobs" / "test.yaml").write_text( + "steps:\n" + " - label: Conditional test\n" + " key: conditional-test\n" + " commands: [pytest tests/x]\n" + f" if_condition: '{condition}'\n" + ) + config = PipelineConfig("p", ".buildkite/x.yaml", ["jobs"], [], []) + report = LoadReport() + steps = load_steps(tmp_path, config, report) + (step,) = steps + pf = run_preflight( + tmp_path, + [_pipe(steps, _covered_targets(step))], + _healthy_full(), + report, + ) + + assert report.unknown_fields == {} + assert step.step_id not in pf.force_select + + def test_duplicate_id_force_selected(): a, b = _step("dup"), _step("dup") report = LoadReport()