From 1928e80288619b3ae9d60e94428f9d419af3033d Mon Sep 17 00:00:00 2001 From: ammallya Date: Tue, 30 Jun 2026 18:29:01 +0000 Subject: [PATCH 01/23] Adding reuse based on artifact availability and unaffected stages --- .github/workflows/multi_arch_ci.yml | 5 +++ .github/workflows/setup_multi_arch.yml | 7 ++++ .../github_actions/configure_multi_arch_ci.py | 38 +++++++++++++++++-- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/.github/workflows/multi_arch_ci.yml b/.github/workflows/multi_arch_ci.yml index 5f65169cedd..62e93939700 100644 --- a/.github/workflows/multi_arch_ci.yml +++ b/.github/workflows/multi_arch_ci.yml @@ -42,6 +42,10 @@ on: type: string default: "" description: "Workflow run ID to copy prebuilt stage artifacts from; required when prebuilt_stages is set" + stage_reuse_mode: + type: string + default: "dry-run" + description: "dry-run (report only) | skip-stage (actually skip unaffected stages)" changed_projects: type: string default: "" @@ -82,6 +86,7 @@ jobs: windows_test_labels: ${{ inputs.windows_test_labels || '' }} prebuilt_stages: ${{ inputs.prebuilt_stages || '' }} baseline_run_id: ${{ inputs.baseline_run_id || '' }} + stage_reuse_mode: ${{ inputs.stage_reuse_mode || 'dry-run' }} build_pytorch: ${{ github.event_name != 'workflow_dispatch' || inputs.build_pytorch }} linux_build_and_test: diff --git a/.github/workflows/setup_multi_arch.yml b/.github/workflows/setup_multi_arch.yml index 7039cf7be9b..dd13bfb9d54 100644 --- a/.github/workflows/setup_multi_arch.yml +++ b/.github/workflows/setup_multi_arch.yml @@ -43,6 +43,10 @@ on: description: "Workflow run ID to copy prebuilt stage artifacts from." type: string default: "" + stage_reuse_mode: + description: "dry-run (default) only reports which stages would be skipped; skip-stage actually skips unaffected stages." + type: string + default: "dry-run" build_pytorch: description: "Build PyTorch wheels." type: boolean @@ -154,6 +158,9 @@ jobs: WINDOWS_TEST_LABELS: ${{ inputs.windows_test_labels }} PREBUILT_STAGES: ${{ inputs.prebuilt_stages }} BASELINE_RUN_ID: ${{ inputs.baseline_run_id }} + STAGE_REUSE_MODE: ${{ inputs.stage_reuse_mode }} + STAGE_REUSE_CURRENT_SHA: ${{ steps.checkout.outputs.commit }} + STAGE_REUSE_MAX_AGE_HOURS: "72" # Skip path filtering for external repos (git sha won't exist in TheRock) SKIP_PATH_FILTERS: ${{ inputs.external_repo != '' && 'true' || '' }} CI_CONFIG_PATH: ci-config diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index 04850370425..9676c2e34fa 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -627,6 +627,12 @@ def _determine_test_type( # Default: quick tests for fast CI feedback. return "quick", "default" +def _stage_reuse_target_families(ci_inputs: "CIInputs") -> list[str]: + families = list(dict.fromkeys( + [*ci_inputs.linux_amdgpu_families, *ci_inputs.windows_amdgpu_families])) + if "generic" not in families: + families.append("generic") + return families def decide_jobs( ci_inputs: CIInputs, @@ -635,14 +641,38 @@ def decide_jobs( """Determine which job groups to run, skip, or satisfy with prebuilt files.""" # Build ROCm. - # TODO(#3399): Use changed files and build_topology.py to: - # 1. set per-stage prebuilt decisions - # 2. skip job groups that aren't reachable from the changed files - # Parse explicit prebuilt stages from workflow_dispatch input. + # Parse explicit prebuilt stages from workflow_dispatch input. These are + # the MANUAL inputs and are always honored, unchanged, in every mode. + stage_decisions: dict[str, JobAction] = {} if ci_inputs.prebuilt_stages: for stage in _parse_prebuilt_stages(ci_inputs.prebuilt_stages): stage_decisions[stage] = JobAction.PREBUILT + + # Behind STAGE_REUSE_MODE: in dry-run we only PRINT + # which stages WOULD be skipped and apply nothing; in "skip-stage" the + # eligible stages are merged into stage_decisions so the orchestrator + # skips their builds and copies artifacts instead. + from stage_reuse_decision import ( + StageReuseMode, + compute_auto_stage_reuse, + render_step_summary, + ) + target_families = _stage_reuse_target_families(ci_inputs) + auto = compute_auto_stage_reuse( + changed_files=git_context.changed_files, + mode=StageReuseMode.from_environ(), + target_families=target_families, + ) + for line in auto.report_lines: + print(line) + try: + gha_append_step_summary(render_step_summary(auto)) + except Exception as exc: # never fail CI config on a reporting + print(f"[STAGE-REUSE] could not write step summary: {exc}") + # Only skip-stage mode returns non-empty applied_reuse_stages. + for stage in auto.applied_reuse_stages: + stage_decisions.setdefault(stage, JobAction.PREBUILT) build_rocm = BuildRocmDecision( action=JobAction.RUN, stage_decisions=stage_decisions, From 1097ec277fe42b150ddcf5dc3a90b9578ad2a691 Mon Sep 17 00:00:00 2001 From: ammallya Date: Tue, 30 Jun 2026 18:29:07 +0000 Subject: [PATCH 02/23] Adding reuse based on artifact availability and unaffected stages --- .../github_actions/stage_reuse_decision.py | 204 ++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 build_tools/github_actions/stage_reuse_decision.py diff --git a/build_tools/github_actions/stage_reuse_decision.py b/build_tools/github_actions/stage_reuse_decision.py new file mode 100644 index 00000000000..368533ece3c --- /dev/null +++ b/build_tools/github_actions/stage_reuse_decision.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Auto-compute per-stage rebuild/reuse decisions for multi-arch CI. + +This script decides per build stage, whether the stage can be satisfied with prebuilt +artifacts (reuse) instead of being rebuilt. + +--------------------------------- +This module is wired into CI behind a two-way mode switch so it can be +observed before it changes anything: +* ``dry-run`` - DEFAULT. Compute the analysis and PRINT, for each stage that + *would* be skipped, a line to the console + step summary, but + return NO auto stages, so ``prebuilt_stages`` is unchanged and + every stage still builds exactly. +* ``skip-stage`` - Compute the analysis and actually return the reuse stages so + the orchestrator copies their artifacts and skips the build. + +""" + +from __future__ import annotations + +import enum +import os +from dataclasses import dataclass, field +from typing import Sequence + + +class StageReuseMode(enum.Enum): + DRY_RUN = "dry-run" + ENFORCE = "skip-stage" + + @staticmethod + def from_environ(default: "StageReuseMode" = None) -> "StageReuseMode": + """Read STAGE_REUSE_MODE; default to dry-run when unset/invalid.""" + default = default or StageReuseMode.DRY_RUN + raw = (os.environ.get("STAGE_REUSE_MODE", "") or "").strip().lower() + for mode in StageReuseMode: + if raw == mode.value: + return mode + return default + + +@dataclass(frozen=True) +class AutoStageReuse: + """Result of the auto stage-reuse analysis. + + Attributes: + mode: The mode this analysis ran under. + would_reuse_stages: Stages stage-impact says are unaffected and could + be satisfied by prebuilt artifacts. + rebuild_stages: Stages that must rebuild. + full_rebuild_required: True when the conservative fallback fired + (e.g. build tooling / workflow / topology change). + reasons: Reasons from the impact analysis. + applied_reuse_stages: Stages actually returned for application. Empty + unless mode == ENFORCE. + report_lines: Console / step-summary lines describing the decision. + """ + + mode: StageReuseMode + would_reuse_stages: tuple[str, ...] + rebuild_stages: tuple[str, ...] + full_rebuild_required: bool + reasons: tuple[str, ...] + applied_reuse_stages: tuple[str, ...] + report_lines: tuple[str, ...] = field(default_factory=tuple) + +LOG_PREFIX = "[STAGE-REUSE]" + + +def compute_auto_stage_reuse( + *, + changed_files: Sequence[str] | None, + mode: StageReuseMode, + platform: str | None = None, + topology=None, +) -> AutoStageReuse: + """Compute auto stage-reuse decisions from changed files. + + Args: + changed_files: Repo-relative changed paths. ``None`` (schedule / + workflow_dispatch with no diff) is treated as "cannot narrow scope" + -> conservative full rebuild, nothing reused. + mode: The stage-reuse mode switch. + platform: Optional platform filter passed to stage-impact. + topology: Optional BuildTopology (injected for testing). + + Returns: + An AutoStageReuse. ``applied_reuse_stages`` is non-empty only when + ``mode == ENFORCE``. + """ + # No diff available -> cannot safely narrow scope -> rebuild everything. + if changed_files is None: + lines = ( + f"{LOG_PREFIX} mode={mode.value}; no changed-file list available " + f"(schedule/dispatch). Conservatively rebuilding all stages.", + ) + return AutoStageReuse( + mode=mode, + would_reuse_stages=(), + rebuild_stages=(), + full_rebuild_required=True, + reasons=("no changed-file list available",), + applied_reuse_stages=(), + report_lines=lines, + ) + + # Import here so the rest of the module don't require the + # topology stack to be importable. + from stage_impact import analyze_stage_impact + + impact = analyze_stage_impact( + changed_inputs=list(changed_files), + topology=topology, + platform=platform, + ) + + would_reuse = tuple(impact.copy_stages) + rebuild = tuple(impact.rebuild_stages) + + applied = would_reuse if mode is StageReuseMode.ENFORCE else () + + report_lines = _format_report( + mode=mode, + would_reuse=would_reuse, + rebuild=rebuild, + full_rebuild_required=impact.full_rebuild_required, + reasons=tuple(impact.reasons), + applied=applied, + ) + + return AutoStageReuse( + mode=mode, + would_reuse_stages=would_reuse, + rebuild_stages=rebuild, + full_rebuild_required=impact.full_rebuild_required, + reasons=tuple(impact.reasons), + applied_reuse_stages=applied, + report_lines=report_lines, + ) + + +def _format_report( + *, + mode: StageReuseMode, + would_reuse: tuple[str, ...], + rebuild: tuple[str, ...], + full_rebuild_required: bool, + reasons: tuple[str, ...], + applied: tuple[str, ...], +) -> tuple[str, ...]: + lines: list[str] = [] + verb = "WILL be skipped (enforced)" if mode is StageReuseMode.ENFORCE else "WOULD be skipped (dry-run)" + lines.append(f"{LOG_PREFIX} mode={mode.value}") + if full_rebuild_required: + lines.append( + f"{LOG_PREFIX} conservative full rebuild: no stages eligible for reuse." + ) + for reason in reasons: + lines.append(f"{LOG_PREFIX} reason: {reason}") + return tuple(lines) + + if not would_reuse: + lines.append(f"{LOG_PREFIX} no stages eligible for reuse; all stages rebuild.") + return tuple(lines) + + for stage in would_reuse: + lines.append(f"{LOG_PREFIX} stage '{stage}' {verb}") + if rebuild: + lines.append(f"{LOG_PREFIX} stages rebuilding: {', '.join(rebuild)}") + if mode is StageReuseMode.DRY_RUN: + lines.append( + f"{LOG_PREFIX} dry-run: prebuilt_stages NOT modified; " + f"all stages still build. Set STAGE_REUSE_MODE=skip-stage to apply." + ) + return tuple(lines) + + +def render_step_summary(result: AutoStageReuse) -> str: + """Render a GitHub step-summary markdown block for the analysis.""" + out = ["### Stage reuse analysis", ""] + out.append(f"- mode: `{result.mode.value}`") + out.append(f"- full rebuild required: `{result.full_rebuild_required}`") + out.append( + f"- would reuse: " + + (", ".join(f"`{s}`" for s in result.would_reuse_stages) or "_none_") + ) + out.append( + f"- applied: " + + (", ".join(f"`{s}`" for s in result.applied_reuse_stages) or "_none_") + ) + if result.reasons: + out.append("- reasons:") + for reason in result.reasons: + out.append(f" - {reason}") + if result.mode is StageReuseMode.DRY_RUN and result.would_reuse_stages: + out.append("") + out.append( + "> Dry-run only: no build steps were skipped. " + "Set `STAGE_REUSE_MODE=skip-stage` to enable skipping." + ) + return "\n".join(out) From b0d8f67c68d1e905ceadafc02124e3b449fab3d2 Mon Sep 17 00:00:00 2001 From: ammallya Date: Tue, 30 Jun 2026 19:50:11 +0000 Subject: [PATCH 03/23] Adding test cases and fixing bugs --- .../github_actions/configure_multi_arch_ci.py | 13 +- .../github_actions/stage_reuse_decision.py | 308 ++++++++++++------ .../tests/stage_reuse_decision_test.py | 254 +++++++++++++++ 3 files changed, 467 insertions(+), 108 deletions(-) create mode 100644 build_tools/tests/stage_reuse_decision_test.py diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index 9676c2e34fa..6184d99c6cd 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -627,12 +627,6 @@ def _determine_test_type( # Default: quick tests for fast CI feedback. return "quick", "default" -def _stage_reuse_target_families(ci_inputs: "CIInputs") -> list[str]: - families = list(dict.fromkeys( - [*ci_inputs.linux_amdgpu_families, *ci_inputs.windows_amdgpu_families])) - if "generic" not in families: - families.append("generic") - return families def decide_jobs( ci_inputs: CIInputs, @@ -658,6 +652,7 @@ def decide_jobs( compute_auto_stage_reuse, render_step_summary, ) + from github_actions_api import gha_append_step_summary target_families = _stage_reuse_target_families(ci_inputs) auto = compute_auto_stage_reuse( changed_files=git_context.changed_files, @@ -719,6 +714,12 @@ def decide_jobs( # Step 4: Select Targets # --------------------------------------------------------------------------- +def _stage_reuse_target_families(ci_inputs: "CIInputs") -> list[str]: + families = list(dict.fromkeys( + [*ci_inputs.linux_amdgpu_families, *ci_inputs.windows_amdgpu_families])) + if "generic" not in families: + families.append("generic") + return families def _validate_family_names( names: list[str], diff --git a/build_tools/github_actions/stage_reuse_decision.py b/build_tools/github_actions/stage_reuse_decision.py index 368533ece3c..74741d3c4b5 100644 --- a/build_tools/github_actions/stage_reuse_decision.py +++ b/build_tools/github_actions/stage_reuse_decision.py @@ -24,7 +24,7 @@ import enum import os from dataclasses import dataclass, field -from typing import Sequence +from typing import Callable, Optional, Sequence class StageReuseMode(enum.Enum): @@ -42,73 +42,108 @@ def from_environ(default: "StageReuseMode" = None) -> "StageReuseMode": return default +BaselineSelector = Callable[[Sequence["object"]], Optional["object"]] + + @dataclass(frozen=True) class AutoStageReuse: - """Result of the auto stage-reuse analysis. - - Attributes: - mode: The mode this analysis ran under. - would_reuse_stages: Stages stage-impact says are unaffected and could - be satisfied by prebuilt artifacts. - rebuild_stages: Stages that must rebuild. - full_rebuild_required: True when the conservative fallback fired - (e.g. build tooling / workflow / topology change). - reasons: Reasons from the impact analysis. - applied_reuse_stages: Stages actually returned for application. Empty - unless mode == ENFORCE. - report_lines: Console / step-summary lines describing the decision. - """ + """Result of the auto stage-reuse analysis.""" mode: StageReuseMode - would_reuse_stages: tuple[str, ...] + candidate_stages: tuple[str, ...] rebuild_stages: tuple[str, ...] full_rebuild_required: bool - reasons: tuple[str, ...] + baseline_run_id: Optional[str] + baseline_html_url: Optional[str] + available_stages: tuple[str, ...] + unavailable_stages: tuple[str, ...] applied_reuse_stages: tuple[str, ...] + reasons: tuple[str, ...] report_lines: tuple[str, ...] = field(default_factory=tuple) + LOG_PREFIX = "[STAGE-REUSE]" +def required_artifacts_for_stages(topology, stage_names, target_families): + """Artifact/family pairs the given stages produce.""" + from baseline_runs import RequiredArtifact + + artifacts_by_group = topology.get_artifact_group_to_artifacts() + required: list = [] + seen: set = set() + for stage_name in stage_names: + stage = topology.build_stages.get(stage_name) + if stage is None: + continue + for group_name in stage.artifact_groups: + for artifact_name in artifacts_by_group.get(group_name, []): + for family in target_families: + req = RequiredArtifact(name=artifact_name, target_family=family) + if req not in seen: + seen.add(req) + required.append(req) + return required + + +def _matched_filenames(baseline) -> set: + if baseline is None: + return set() + return set(baseline.artifact_availability.matched_filenames) + + +def _stage_artifacts_available(topology, stage_name, target_families, available_filenames): + """True when every artifact this stage produces has an archive present.""" + from artifact_manager import ARTIFACT_COMPONENTS + from _therock_utils.artifact_backend import ARTIFACT_EXTENSIONS + + artifacts_by_group = topology.get_artifact_group_to_artifacts() + stage = topology.build_stages.get(stage_name) + if stage is None: + return False + for group_name in stage.artifact_groups: + for artifact_name in artifacts_by_group.get(group_name, []): + for family in target_families: + found = False + for component in ARTIFACT_COMPONENTS: + for extension in ARTIFACT_EXTENSIONS: + filename = f"{artifact_name}_{component}_{family}{extension}" + if filename in available_filenames: + found = True + break + if found: + break + if not found: + return False + return True + + def compute_auto_stage_reuse( *, changed_files: Sequence[str] | None, mode: StageReuseMode, platform: str | None = None, + target_families: Sequence[str] = (), topology=None, + baseline_selector: BaselineSelector | None = None, ) -> AutoStageReuse: - """Compute auto stage-reuse decisions from changed files. - - Args: - changed_files: Repo-relative changed paths. ``None`` (schedule / - workflow_dispatch with no diff) is treated as "cannot narrow scope" - -> conservative full rebuild, nothing reused. - mode: The stage-reuse mode switch. - platform: Optional platform filter passed to stage-impact. - topology: Optional BuildTopology (injected for testing). - - Returns: - An AutoStageReuse. ``applied_reuse_stages`` is non-empty only when - ``mode == ENFORCE``. - """ - # No diff available -> cannot safely narrow scope -> rebuild everything. + """Compute auto stage-reuse decisions, verified against a baseline run.""" if changed_files is None: - lines = ( - f"{LOG_PREFIX} mode={mode.value}; no changed-file list available " - f"(schedule/dispatch). Conservatively rebuilding all stages.", - ) - return AutoStageReuse( - mode=mode, - would_reuse_stages=(), - rebuild_stages=(), + return _empty_result( + mode, full_rebuild_required=True, reasons=("no changed-file list available",), - applied_reuse_stages=(), - report_lines=lines, + report_lines=( + f"{LOG_PREFIX} mode={mode.value}; no changed-file list " + f"Conservatively rebuilding all stages.", + ), ) - # Import here so the rest of the module don't require the - # topology stack to be importable. + if topology is None: + from _therock_utils.build_topology import get_topology + + topology = get_topology() + from stage_impact import analyze_stage_impact impact = analyze_stage_impact( @@ -116,65 +151,135 @@ def compute_auto_stage_reuse( topology=topology, platform=platform, ) - - would_reuse = tuple(impact.copy_stages) + candidates = tuple(impact.copy_stages) rebuild = tuple(impact.rebuild_stages) + families = tuple(target_families) or ("generic",) + + if impact.full_rebuild_required or not candidates: + lines = _format_report( + mode=mode, candidates=candidates, rebuild=rebuild, + full_rebuild_required=impact.full_rebuild_required, + reasons=tuple(impact.reasons), baseline_run_id=None, + available=(), unavailable=candidates, + ) + return AutoStageReuse( + mode=mode, candidate_stages=candidates, rebuild_stages=rebuild, + full_rebuild_required=impact.full_rebuild_required, + baseline_run_id=None, baseline_html_url=None, + available_stages=(), unavailable_stages=candidates, + applied_reuse_stages=(), reasons=tuple(impact.reasons), + report_lines=lines, + ) + + required = required_artifacts_for_stages(topology, candidates, families) + baseline = None + baseline_error: Optional[str] = None + try: + if baseline_selector is None: + baseline_selector = _default_baseline_selector(platform=platform) + baseline = baseline_selector(required) + except Exception as exc: + baseline_error = str(exc) - applied = would_reuse if mode is StageReuseMode.ENFORCE else () + available_filenames = _matched_filenames(baseline) + available: list[str] = [] + unavailable: list[str] = [] + if baseline is not None: + for stage_name in candidates: + if _stage_artifacts_available(topology, stage_name, families, available_filenames): + available.append(stage_name) + else: + unavailable.append(stage_name) + else: + unavailable = list(candidates) - report_lines = _format_report( - mode=mode, - would_reuse=would_reuse, - rebuild=rebuild, - full_rebuild_required=impact.full_rebuild_required, - reasons=tuple(impact.reasons), - applied=applied, + available_t = tuple(available) + unavailable_t = tuple(unavailable) + applied = available_t if mode is StageReuseMode.ENFORCE else () + baseline_run_id = baseline.run_id if baseline is not None else None + baseline_url = baseline.html_url if baseline is not None else None + + lines = _format_report( + mode=mode, candidates=candidates, rebuild=rebuild, + full_rebuild_required=False, reasons=tuple(impact.reasons), + baseline_run_id=baseline_run_id, available=available_t, + unavailable=unavailable_t, baseline_error=baseline_error, + ) + return AutoStageReuse( + mode=mode, candidate_stages=candidates, rebuild_stages=rebuild, + full_rebuild_required=False, baseline_run_id=baseline_run_id, + baseline_html_url=baseline_url, available_stages=available_t, + unavailable_stages=unavailable_t, applied_reuse_stages=applied, + reasons=tuple(impact.reasons), report_lines=lines, ) + +def _default_baseline_selector(*, platform: str | None) -> BaselineSelector: + """Build a selector bound to baseline_runs.select_baseline_run.""" + from baseline_runs import select_baseline_run + + github_repository = os.environ.get("GITHUB_REPOSITORY", "ROCm/TheRock") + branch = os.environ.get("STAGE_REUSE_BASELINE_BRANCH", "main") + workflow_name = os.environ.get("STAGE_REUSE_BASELINE_WORKFLOW", "multi_arch_ci.yml") + current_commit_sha = os.environ.get("STAGE_REUSE_CURRENT_SHA") or None + max_age_hours_raw = os.environ.get("STAGE_REUSE_MAX_AGE_HOURS") + max_age_hours = float(max_age_hours_raw) if max_age_hours_raw else None + + def _select(required): + return select_baseline_run( + required_artifacts=required, + github_repository=github_repository, + workflow_name=workflow_name, + branch=branch, + platform=platform or "linux", + current_commit_sha=current_commit_sha, + ordered_commit_shas=None if current_commit_sha is None else [], + max_age_hours=max_age_hours, + ) + + return _select + + +def _empty_result(mode, *, full_rebuild_required=False, reasons=(), report_lines=()): return AutoStageReuse( - mode=mode, - would_reuse_stages=would_reuse, - rebuild_stages=rebuild, - full_rebuild_required=impact.full_rebuild_required, - reasons=tuple(impact.reasons), - applied_reuse_stages=applied, - report_lines=report_lines, + mode=mode, candidate_stages=(), rebuild_stages=(), + full_rebuild_required=full_rebuild_required, + baseline_run_id=None, baseline_html_url=None, + available_stages=(), unavailable_stages=(), + applied_reuse_stages=(), reasons=reasons, report_lines=report_lines, ) -def _format_report( - *, - mode: StageReuseMode, - would_reuse: tuple[str, ...], - rebuild: tuple[str, ...], - full_rebuild_required: bool, - reasons: tuple[str, ...], - applied: tuple[str, ...], -) -> tuple[str, ...]: - lines: list[str] = [] - verb = "WILL be skipped (enforced)" if mode is StageReuseMode.ENFORCE else "WOULD be skipped (dry-run)" - lines.append(f"{LOG_PREFIX} mode={mode.value}") +def _format_report(*, mode, candidates, rebuild, full_rebuild_required, + reasons, baseline_run_id, available, unavailable, baseline_error=None): + lines: list[str] = [f"{LOG_PREFIX} mode={mode.value}"] if full_rebuild_required: - lines.append( - f"{LOG_PREFIX} conservative full rebuild: no stages eligible for reuse." - ) + lines.append(f"{LOG_PREFIX} conservative full rebuild: no stages eligible for reuse.") for reason in reasons: lines.append(f"{LOG_PREFIX} reason: {reason}") return tuple(lines) - - if not would_reuse: - lines.append(f"{LOG_PREFIX} no stages eligible for reuse; all stages rebuild.") + if not candidates: + lines.append(f"{LOG_PREFIX} no unaffected stages; all stages rebuild.") return tuple(lines) - - for stage in would_reuse: - lines.append(f"{LOG_PREFIX} stage '{stage}' {verb}") + if baseline_error: + lines.append(f"{LOG_PREFIX} baseline lookup failed ({baseline_error}); " + f"cannot verify artifacts, rebuilding all candidates.") + elif baseline_run_id: + lines.append(f"{LOG_PREFIX} baseline run for artifact check: {baseline_run_id}") + else: + lines.append(f"{LOG_PREFIX} no baseline run contains artifacts for all " + f"candidate stages; rebuilding all candidates.") + verb = ("WILL be skipped" if mode is StageReuseMode.ENFORCE + else "WOULD be skipped") + for stage in available: + lines.append(f"{LOG_PREFIX} stage '{stage}' unaffected AND available in baseline -> {verb}") + for stage in unavailable: + lines.append(f"{LOG_PREFIX} stage '{stage}' unaffected but artifacts NOT available -> rebuild") if rebuild: - lines.append(f"{LOG_PREFIX} stages rebuilding: {', '.join(rebuild)}") - if mode is StageReuseMode.DRY_RUN: - lines.append( - f"{LOG_PREFIX} dry-run: prebuilt_stages NOT modified; " - f"all stages still build. Set STAGE_REUSE_MODE=skip-stage to apply." - ) + lines.append(f"{LOG_PREFIX} stages rebuilding (impacted): {', '.join(rebuild)}") + if mode is StageReuseMode.DRY_RUN and available: + lines.append(f"{LOG_PREFIX} dry-run: prebuilt_stages NOT modified; all stages " + f"still build.") return tuple(lines) @@ -183,22 +288,21 @@ def render_step_summary(result: AutoStageReuse) -> str: out = ["### Stage reuse analysis", ""] out.append(f"- mode: `{result.mode.value}`") out.append(f"- full rebuild required: `{result.full_rebuild_required}`") - out.append( - f"- would reuse: " - + (", ".join(f"`{s}`" for s in result.would_reuse_stages) or "_none_") - ) - out.append( - f"- applied: " - + (", ".join(f"`{s}`" for s in result.applied_reuse_stages) or "_none_") - ) + out.append("- baseline run checked: " + (f"`{result.baseline_run_id}`" if result.baseline_run_id else "_none_")) + out.append("- unaffected candidates: " + (", ".join(f"`{s}`" for s in result.candidate_stages) or "_none_")) + out.append("- available in baseline: " + (", ".join(f"`{s}`" for s in result.available_stages) or "_none_")) + out.append("- applied: " + (", ".join(f"`{s}`" for s in result.applied_reuse_stages) or "_none_")) if result.reasons: out.append("- reasons:") for reason in result.reasons: out.append(f" - {reason}") - if result.mode is StageReuseMode.DRY_RUN and result.would_reuse_stages: + if result.mode is StageReuseMode.DRY_RUN and result.available_stages: out.append("") - out.append( - "> Dry-run only: no build steps were skipped. " - "Set `STAGE_REUSE_MODE=skip-stage` to enable skipping." - ) - return "\n".join(out) + out.append("> Dry-run only: no build steps were skipped. Artifacts were " + "verified against the baseline run above. Set " + "`STAGE_REUSE_MODE=skip-stage`.") + return "\n".join(out) \ No newline at end of file diff --git a/build_tools/tests/stage_reuse_decision_test.py b/build_tools/tests/stage_reuse_decision_test.py new file mode 100644 index 00000000000..9707fcf313b --- /dev/null +++ b/build_tools/tests/stage_reuse_decision_test.py @@ -0,0 +1,254 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Tests for stage_reuse_decision: impact + baseline-availability gates. +""" + +from pathlib import Path +import sys +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import stage_reuse_decision as srd +from stage_reuse_decision import StageReuseMode, compute_auto_stage_reuse +from baseline_runs import ( + BaselineRun, + WorkflowRunSummary, + ArtifactAvailability, + WorkflowJobHealth, +) + + +class _FakeStage: + def __init__(self, groups): + self.artifact_groups = groups + + +class FakeTopology: + """Minimal BuildTopology stand-in for stage_impact + artifact derivation. + + compiler-runtime produces artifact 'base'; math-libs produces 'blas'. + """ + + def __init__(self): + self.build_stages = { + "compiler-runtime": _FakeStage(["base-group"]), + "math-libs": _FakeStage(["blas-group"]), + } + self.artifact_groups = { + "base-group": type("G", (), {"source_sets": ["core"]})(), + "blas-group": type("G", (), {"source_sets": ["libs"]})(), + } + + def get_source_set_to_artifact_groups(self): + return {"core": ["base-group"], "libs": ["blas-group"]} + + def get_artifact_group_to_build_stages(self): + return {"base-group": ["compiler-runtime"], "blas-group": ["math-libs"]} + + def get_artifact_group_to_artifacts(self): + return {"base-group": ["base"], "blas-group": ["blas"]} + + def get_produced_artifacts(self, stage_name): + return set() + + def get_artifacts_in_group(self, group_name): + return {"base-group": [], "blas-group": []}.get(group_name, []) + + def get_source_set_for_submodule(self, name, platform=None): + mapping = {"rocm-libraries": "libs"} + ss = mapping.get(name) + return type("S", (), {"name": ss})() if ss else None + + def get_source_set_for_path(self, path, platform=None): + return None + + +def _baseline(run_id, matched_filenames): + summary = WorkflowRunSummary( + repository="ROCm/TheRock", + branch="main", + commit=f"sha-{run_id}", + workflow="multi_arch_ci.yml", + run_id=run_id, + status="completed", + conclusion="success", + timestamp="2026-06-17T20:00:00Z", + html_url=f"https://github.com/ROCm/TheRock/actions/runs/{run_id}", + ) + return BaselineRun( + source_ref=summary, + platform="linux", + job_health=WorkflowJobHealth( + required_name_substrings=("Build",), + matched_job_names=("Build",), + failed_job_names=(), + missing_name_substrings=(), + ), + artifact_availability=ArtifactAvailability( + required_artifacts=(), + matched_filenames=tuple(matched_filenames), + missing_artifacts=(), + ), + ) + + +def _selector(baseline): + """Return an injected baseline_selector that always yields `baseline`.""" + return lambda required: baseline + + +class ModeParsingTest(unittest.TestCase): + def test_default_is_dry_run(self): + import os + + os.environ.pop("STAGE_REUSE_MODE", None) + self.assertEqual(StageReuseMode.from_environ(), StageReuseMode.DRY_RUN) + + def test_explicit_modes(self): + import os + + for value, expected in [ + ("dry-run", StageReuseMode.DRY_RUN), + ("skip-stage", StageReuseMode.ENFORCE), + ("skip-stage", StageReuseMode.ENFORCE), + ("garbage", StageReuseMode.DRY_RUN), + ]: + os.environ["STAGE_REUSE_MODE"] = value + self.assertEqual(StageReuseMode.from_environ(), expected) + os.environ.pop("STAGE_REUSE_MODE", None) + + +class AvailabilityGateTest(unittest.TestCase): + def test_dry_run_verifies_artifacts_present(self): + # math-libs changed -> compiler-runtime unaffected (candidate). + # Baseline HAS base artifact -> reported as WOULD skip, but applied=(). + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.DRY_RUN, + target_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + self.assertIn("compiler-runtime", result.candidate_stages) + self.assertIn("compiler-runtime", result.available_stages) + self.assertEqual(result.applied_reuse_stages, ()) # dry-run applies nothing + self.assertEqual(result.baseline_run_id, "123") + joined = "\n".join(result.report_lines) + self.assertIn("available in baseline", joined) + self.assertIn("WOULD be skipped", joined) + + def test_dry_run_unaffected_but_artifacts_missing_rebuilds(self): + # compiler-runtime unaffected, but baseline only has blas (not base). + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.DRY_RUN, + target_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["blas_lib_generic.tar.zst"])), + ) + self.assertIn("compiler-runtime", result.candidate_stages) + self.assertIn("compiler-runtime", result.unavailable_stages) + self.assertEqual(result.available_stages, ()) + joined = "\n".join(result.report_lines) + self.assertIn("artifacts NOT available", joined) + + def test_no_baseline_found_rebuilds_candidates(self): + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.DRY_RUN, + target_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(None), + ) + self.assertIn("compiler-runtime", result.candidate_stages) + self.assertEqual(result.available_stages, ()) + self.assertIsNone(result.baseline_run_id) + joined = "\n".join(result.report_lines) + self.assertIn("no baseline run contains artifacts", joined) + + def test_partial_family_availability_rebuilds(self): + # Needs base for two families; baseline only has the generic archive. + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.DRY_RUN, + target_families=["generic", "gfx94X-dcgpu"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + self.assertIn("compiler-runtime", result.unavailable_stages) + self.assertEqual(result.available_stages, ()) + + def test_enforce_applies_only_available_stages(self): + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.ENFORCE, + target_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + self.assertEqual(result.applied_reuse_stages, ("compiler-runtime",)) + joined = "\n".join(result.report_lines) + self.assertIn("WILL be skipped", joined) + + def test_baseline_lookup_error_is_safe(self): + def boom(required): + raise RuntimeError("network down") + + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.ENFORCE, + target_families=["generic"], + topology=FakeTopology(), + baseline_selector=boom, + ) + self.assertEqual(result.applied_reuse_stages, ()) + joined = "\n".join(result.report_lines) + self.assertIn("baseline lookup failed", joined) + + +class GuardrailTest(unittest.TestCase): + def test_full_ci_trigger_skips_baseline_query(self): + called = {"n": 0} + + def selector(required): + called["n"] += 1 + return None + + result = compute_auto_stage_reuse( + changed_files=["build_tools/foo.py"], + mode=StageReuseMode.ENFORCE, + topology=FakeTopology(), + baseline_selector=selector, + ) + self.assertTrue(result.full_rebuild_required) + self.assertEqual(result.applied_reuse_stages, ()) + self.assertEqual(called["n"], 0) + + def test_no_diff_is_conservative(self): + result = compute_auto_stage_reuse( + changed_files=None, + mode=StageReuseMode.ENFORCE, + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + self.assertTrue(result.full_rebuild_required) + self.assertEqual(result.applied_reuse_stages, ()) + + def test_step_summary_shows_baseline_and_availability(self): + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.DRY_RUN, + target_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + summary = srd.render_step_summary(result) + self.assertIn("baseline run checked: `123`", summary) + self.assertIn("available in baseline", summary) + self.assertIn("no build steps were skipped", summary) + + +if __name__ == "__main__": + unittest.main() From 7bd631c401008ee2457bfdfa673ac4eab8eb1415 Mon Sep 17 00:00:00 2001 From: ammallya Date: Tue, 30 Jun 2026 19:54:17 +0000 Subject: [PATCH 04/23] Moving tests to right location --- .../tests/stage_reuse_decision_test.py | 254 ------------------ 1 file changed, 254 deletions(-) delete mode 100644 build_tools/tests/stage_reuse_decision_test.py diff --git a/build_tools/tests/stage_reuse_decision_test.py b/build_tools/tests/stage_reuse_decision_test.py deleted file mode 100644 index 9707fcf313b..00000000000 --- a/build_tools/tests/stage_reuse_decision_test.py +++ /dev/null @@ -1,254 +0,0 @@ -# Copyright Advanced Micro Devices, Inc. -# SPDX-License-Identifier: MIT - -"""Tests for stage_reuse_decision: impact + baseline-availability gates. -""" - -from pathlib import Path -import sys -import unittest - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -import stage_reuse_decision as srd -from stage_reuse_decision import StageReuseMode, compute_auto_stage_reuse -from baseline_runs import ( - BaselineRun, - WorkflowRunSummary, - ArtifactAvailability, - WorkflowJobHealth, -) - - -class _FakeStage: - def __init__(self, groups): - self.artifact_groups = groups - - -class FakeTopology: - """Minimal BuildTopology stand-in for stage_impact + artifact derivation. - - compiler-runtime produces artifact 'base'; math-libs produces 'blas'. - """ - - def __init__(self): - self.build_stages = { - "compiler-runtime": _FakeStage(["base-group"]), - "math-libs": _FakeStage(["blas-group"]), - } - self.artifact_groups = { - "base-group": type("G", (), {"source_sets": ["core"]})(), - "blas-group": type("G", (), {"source_sets": ["libs"]})(), - } - - def get_source_set_to_artifact_groups(self): - return {"core": ["base-group"], "libs": ["blas-group"]} - - def get_artifact_group_to_build_stages(self): - return {"base-group": ["compiler-runtime"], "blas-group": ["math-libs"]} - - def get_artifact_group_to_artifacts(self): - return {"base-group": ["base"], "blas-group": ["blas"]} - - def get_produced_artifacts(self, stage_name): - return set() - - def get_artifacts_in_group(self, group_name): - return {"base-group": [], "blas-group": []}.get(group_name, []) - - def get_source_set_for_submodule(self, name, platform=None): - mapping = {"rocm-libraries": "libs"} - ss = mapping.get(name) - return type("S", (), {"name": ss})() if ss else None - - def get_source_set_for_path(self, path, platform=None): - return None - - -def _baseline(run_id, matched_filenames): - summary = WorkflowRunSummary( - repository="ROCm/TheRock", - branch="main", - commit=f"sha-{run_id}", - workflow="multi_arch_ci.yml", - run_id=run_id, - status="completed", - conclusion="success", - timestamp="2026-06-17T20:00:00Z", - html_url=f"https://github.com/ROCm/TheRock/actions/runs/{run_id}", - ) - return BaselineRun( - source_ref=summary, - platform="linux", - job_health=WorkflowJobHealth( - required_name_substrings=("Build",), - matched_job_names=("Build",), - failed_job_names=(), - missing_name_substrings=(), - ), - artifact_availability=ArtifactAvailability( - required_artifacts=(), - matched_filenames=tuple(matched_filenames), - missing_artifacts=(), - ), - ) - - -def _selector(baseline): - """Return an injected baseline_selector that always yields `baseline`.""" - return lambda required: baseline - - -class ModeParsingTest(unittest.TestCase): - def test_default_is_dry_run(self): - import os - - os.environ.pop("STAGE_REUSE_MODE", None) - self.assertEqual(StageReuseMode.from_environ(), StageReuseMode.DRY_RUN) - - def test_explicit_modes(self): - import os - - for value, expected in [ - ("dry-run", StageReuseMode.DRY_RUN), - ("skip-stage", StageReuseMode.ENFORCE), - ("skip-stage", StageReuseMode.ENFORCE), - ("garbage", StageReuseMode.DRY_RUN), - ]: - os.environ["STAGE_REUSE_MODE"] = value - self.assertEqual(StageReuseMode.from_environ(), expected) - os.environ.pop("STAGE_REUSE_MODE", None) - - -class AvailabilityGateTest(unittest.TestCase): - def test_dry_run_verifies_artifacts_present(self): - # math-libs changed -> compiler-runtime unaffected (candidate). - # Baseline HAS base artifact -> reported as WOULD skip, but applied=(). - result = compute_auto_stage_reuse( - changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], - mode=StageReuseMode.DRY_RUN, - target_families=["generic"], - topology=FakeTopology(), - baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), - ) - self.assertIn("compiler-runtime", result.candidate_stages) - self.assertIn("compiler-runtime", result.available_stages) - self.assertEqual(result.applied_reuse_stages, ()) # dry-run applies nothing - self.assertEqual(result.baseline_run_id, "123") - joined = "\n".join(result.report_lines) - self.assertIn("available in baseline", joined) - self.assertIn("WOULD be skipped", joined) - - def test_dry_run_unaffected_but_artifacts_missing_rebuilds(self): - # compiler-runtime unaffected, but baseline only has blas (not base). - result = compute_auto_stage_reuse( - changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], - mode=StageReuseMode.DRY_RUN, - target_families=["generic"], - topology=FakeTopology(), - baseline_selector=_selector(_baseline("123", ["blas_lib_generic.tar.zst"])), - ) - self.assertIn("compiler-runtime", result.candidate_stages) - self.assertIn("compiler-runtime", result.unavailable_stages) - self.assertEqual(result.available_stages, ()) - joined = "\n".join(result.report_lines) - self.assertIn("artifacts NOT available", joined) - - def test_no_baseline_found_rebuilds_candidates(self): - result = compute_auto_stage_reuse( - changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], - mode=StageReuseMode.DRY_RUN, - target_families=["generic"], - topology=FakeTopology(), - baseline_selector=_selector(None), - ) - self.assertIn("compiler-runtime", result.candidate_stages) - self.assertEqual(result.available_stages, ()) - self.assertIsNone(result.baseline_run_id) - joined = "\n".join(result.report_lines) - self.assertIn("no baseline run contains artifacts", joined) - - def test_partial_family_availability_rebuilds(self): - # Needs base for two families; baseline only has the generic archive. - result = compute_auto_stage_reuse( - changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], - mode=StageReuseMode.DRY_RUN, - target_families=["generic", "gfx94X-dcgpu"], - topology=FakeTopology(), - baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), - ) - self.assertIn("compiler-runtime", result.unavailable_stages) - self.assertEqual(result.available_stages, ()) - - def test_enforce_applies_only_available_stages(self): - result = compute_auto_stage_reuse( - changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], - mode=StageReuseMode.ENFORCE, - target_families=["generic"], - topology=FakeTopology(), - baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), - ) - self.assertEqual(result.applied_reuse_stages, ("compiler-runtime",)) - joined = "\n".join(result.report_lines) - self.assertIn("WILL be skipped", joined) - - def test_baseline_lookup_error_is_safe(self): - def boom(required): - raise RuntimeError("network down") - - result = compute_auto_stage_reuse( - changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], - mode=StageReuseMode.ENFORCE, - target_families=["generic"], - topology=FakeTopology(), - baseline_selector=boom, - ) - self.assertEqual(result.applied_reuse_stages, ()) - joined = "\n".join(result.report_lines) - self.assertIn("baseline lookup failed", joined) - - -class GuardrailTest(unittest.TestCase): - def test_full_ci_trigger_skips_baseline_query(self): - called = {"n": 0} - - def selector(required): - called["n"] += 1 - return None - - result = compute_auto_stage_reuse( - changed_files=["build_tools/foo.py"], - mode=StageReuseMode.ENFORCE, - topology=FakeTopology(), - baseline_selector=selector, - ) - self.assertTrue(result.full_rebuild_required) - self.assertEqual(result.applied_reuse_stages, ()) - self.assertEqual(called["n"], 0) - - def test_no_diff_is_conservative(self): - result = compute_auto_stage_reuse( - changed_files=None, - mode=StageReuseMode.ENFORCE, - topology=FakeTopology(), - baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), - ) - self.assertTrue(result.full_rebuild_required) - self.assertEqual(result.applied_reuse_stages, ()) - - def test_step_summary_shows_baseline_and_availability(self): - result = compute_auto_stage_reuse( - changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], - mode=StageReuseMode.DRY_RUN, - target_families=["generic"], - topology=FakeTopology(), - baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), - ) - summary = srd.render_step_summary(result) - self.assertIn("baseline run checked: `123`", summary) - self.assertIn("available in baseline", summary) - self.assertIn("no build steps were skipped", summary) - - -if __name__ == "__main__": - unittest.main() From 58b0a1d57c5fab2003960412fad7a5d3c68ab368 Mon Sep 17 00:00:00 2001 From: ammallya Date: Tue, 30 Jun 2026 20:05:58 +0000 Subject: [PATCH 05/23] removing redundant import --- build_tools/github_actions/configure_multi_arch_ci.py | 1 - 1 file changed, 1 deletion(-) diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index 6184d99c6cd..8b98c81c68d 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -652,7 +652,6 @@ def decide_jobs( compute_auto_stage_reuse, render_step_summary, ) - from github_actions_api import gha_append_step_summary target_families = _stage_reuse_target_families(ci_inputs) auto = compute_auto_stage_reuse( changed_files=git_context.changed_files, From 9b7a0e16beed8da9d7a801bf641922fa1c1fad01 Mon Sep 17 00:00:00 2001 From: ammallya Date: Tue, 30 Jun 2026 20:29:19 +0000 Subject: [PATCH 06/23] Fixing copilot detected bug --- .github/workflows/setup_multi_arch.yml | 1 + .../github_actions/stage_reuse_decision.py | 38 ++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/.github/workflows/setup_multi_arch.yml b/.github/workflows/setup_multi_arch.yml index dd13bfb9d54..0930d9a0d2b 100644 --- a/.github/workflows/setup_multi_arch.yml +++ b/.github/workflows/setup_multi_arch.yml @@ -161,6 +161,7 @@ jobs: STAGE_REUSE_MODE: ${{ inputs.stage_reuse_mode }} STAGE_REUSE_CURRENT_SHA: ${{ steps.checkout.outputs.commit }} STAGE_REUSE_MAX_AGE_HOURS: "72" + STAGE_REUSE_COMMIT_HISTORY: "50" # Skip path filtering for external repos (git sha won't exist in TheRock) SKIP_PATH_FILTERS: ${{ inputs.external_repo != '' && 'true' || '' }} CI_CONFIG_PATH: ci-config diff --git a/build_tools/github_actions/stage_reuse_decision.py b/build_tools/github_actions/stage_reuse_decision.py index 74741d3c4b5..cccf50fb48c 100644 --- a/build_tools/github_actions/stage_reuse_decision.py +++ b/build_tools/github_actions/stage_reuse_decision.py @@ -224,6 +224,40 @@ def _default_baseline_selector(*, platform: str | None) -> BaselineSelector: current_commit_sha = os.environ.get("STAGE_REUSE_CURRENT_SHA") or None max_age_hours_raw = os.environ.get("STAGE_REUSE_MAX_AGE_HOURS") max_age_hours = float(max_age_hours_raw) if max_age_hours_raw else None + history_count_raw = os.environ.get("STAGE_REUSE_COMMIT_HISTORY", "50") + try: + history_count = max(1, int(history_count_raw)) + except ValueError: + history_count = 50 + # The commit-compatibility rule needs the branch history (newest-first) to + # establish ancestry. select_baseline_run only accepts a candidate whose + # head_sha is `same` or `ancestor` of current_commit_sha; with an EMPTY + # window every candidate resolves to `unknown` and is rejected, so reuse + # never activates. Fetch the real history here. If the SHA is set but the + # history fetch fails (or returns empty), disable the commit rule (pass both + # as None) rather than enabling it with an empty window -- recency and + # artifact availability still gate the selection. + ordered_commit_shas = None + effective_commit_sha = current_commit_sha + if current_commit_sha is not None: + try: + from github_actions_api import gha_query_recent_branch_commits + + ordered_commit_shas = gha_query_recent_branch_commits( + github_repository_name=github_repository, + branch=branch, + max_count=history_count, + ) + if not ordered_commit_shas: + effective_commit_sha = None + ordered_commit_shas = None + except Exception as exc: + print( + f"{LOG_PREFIX} could not fetch branch history " + f"({exc}); skipping commit-compatibility rule." + ) + effective_commit_sha = None + ordered_commit_shas = None def _select(required): return select_baseline_run( @@ -232,8 +266,8 @@ def _select(required): workflow_name=workflow_name, branch=branch, platform=platform or "linux", - current_commit_sha=current_commit_sha, - ordered_commit_shas=None if current_commit_sha is None else [], + current_commit_sha=effective_commit_sha, + ordered_commit_shas=ordered_commit_shas, max_age_hours=max_age_hours, ) From fc21126d8b4ded43a5046e48f43a5b4f016c1e47 Mon Sep 17 00:00:00 2001 From: ammallya Date: Tue, 30 Jun 2026 21:29:52 +0000 Subject: [PATCH 07/23] Fixing formatting --- .../github_actions/stage_reuse_decision.py | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/build_tools/github_actions/stage_reuse_decision.py b/build_tools/github_actions/stage_reuse_decision.py index cccf50fb48c..d507e51602c 100644 --- a/build_tools/github_actions/stage_reuse_decision.py +++ b/build_tools/github_actions/stage_reuse_decision.py @@ -316,27 +316,36 @@ def _format_report(*, mode, candidates, rebuild, full_rebuild_required, f"still build.") return tuple(lines) +def _format_stage_list(stages: tuple[str, ...]) -> str: + """Render a tuple of stage names as backticked, comma-separated markdown.""" + if not stages: + return "_none_" + return ", ".join(f"`{stage}`" for stage in stages) def render_step_summary(result: AutoStageReuse) -> str: - """Render a GitHub step-summary markdown block for the analysis.""" + """Render a GitHub step-summary markdown block for the analysis. + """ + baseline = f"`{result.baseline_run_id}`" if result.baseline_run_id else "_none_" + candidates = _format_stage_list(result.candidate_stages) + available = _format_stage_list(result.available_stages) + applied = _format_stage_list(result.applied_reuse_stages) + out = ["### Stage reuse analysis", ""] out.append(f"- mode: `{result.mode.value}`") out.append(f"- full rebuild required: `{result.full_rebuild_required}`") - out.append("- baseline run checked: " - (f"`{result.baseline_run_id}`" if result.baseline_run_id else "_none_")) - out.append("- unaffected candidates: " - (", ".join(f"`{s}`" for s in result.candidate_stages) or "_none_")) - out.append("- available in baseline: " - (", ".join(f"`{s}`" for s in result.available_stages) or "_none_")) - out.append("- applied: " - (", ".join(f"`{s}`" for s in result.applied_reuse_stages) or "_none_")) + out.append(f"- baseline run checked: {baseline}") + out.append(f"- unaffected candidates: {candidates}") + out.append(f"- available in baseline: {available}") + out.append(f"- applied: {applied}") if result.reasons: out.append("- reasons:") for reason in result.reasons: out.append(f" - {reason}") if result.mode is StageReuseMode.DRY_RUN and result.available_stages: out.append("") - out.append("> Dry-run only: no build steps were skipped. Artifacts were " - "verified against the baseline run above. Set " - "`STAGE_REUSE_MODE=skip-stage`.") + out.append( + "> Dry-run only: no build steps were skipped. Artifacts were " + "verified against the baseline run above. Set " + "`STAGE_REUSE_MODE=skip-stage` after review to enable skipping." + ) return "\n".join(out) \ No newline at end of file From 0648dccc6f5550f99198a888e3045dad2b2e3239 Mon Sep 17 00:00:00 2001 From: ammallya Date: Tue, 30 Jun 2026 21:42:29 +0000 Subject: [PATCH 08/23] Adding unit tests --- .../tests/stage_reuse_decision_test.py | 354 ++++++++++++++++++ 1 file changed, 354 insertions(+) create mode 100644 build_tools/github_actions/tests/stage_reuse_decision_test.py diff --git a/build_tools/github_actions/tests/stage_reuse_decision_test.py b/build_tools/github_actions/tests/stage_reuse_decision_test.py new file mode 100644 index 00000000000..b6a2c95538a --- /dev/null +++ b/build_tools/github_actions/tests/stage_reuse_decision_test.py @@ -0,0 +1,354 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Tests for stage_reuse_decision: impact + baseline-availability gates. +""" + +from pathlib import Path +import sys +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import stage_reuse_decision as srd +from stage_reuse_decision import StageReuseMode, compute_auto_stage_reuse +from baseline_runs import ( + BaselineRun, + WorkflowRunSummary, + ArtifactAvailability, + WorkflowJobHealth, +) + + +class _FakeStage: + def __init__(self, groups): + self.artifact_groups = groups + + +class FakeTopology: + """Minimal BuildTopology stand-in for stage_impact + artifact derivation. + + compiler-runtime produces artifact 'base'; math-libs produces 'blas'. + """ + + def __init__(self): + self.build_stages = { + "compiler-runtime": _FakeStage(["base-group"]), + "math-libs": _FakeStage(["blas-group"]), + } + self.artifact_groups = { + "base-group": type("G", (), {"source_sets": ["core"]})(), + "blas-group": type("G", (), {"source_sets": ["libs"]})(), + } + + def get_source_set_to_artifact_groups(self): + return {"core": ["base-group"], "libs": ["blas-group"]} + + def get_artifact_group_to_build_stages(self): + return {"base-group": ["compiler-runtime"], "blas-group": ["math-libs"]} + + def get_artifact_group_to_artifacts(self): + return {"base-group": ["base"], "blas-group": ["blas"]} + + def get_produced_artifacts(self, stage_name): + return set() + + def get_artifacts_in_group(self, group_name): + return {"base-group": [], "blas-group": []}.get(group_name, []) + + def get_source_set_for_submodule(self, name, platform=None): + mapping = {"rocm-libraries": "libs"} + ss = mapping.get(name) + return type("S", (), {"name": ss})() if ss else None + + def get_source_set_for_path(self, path, platform=None): + return None + + +def _baseline(run_id, matched_filenames): + summary = WorkflowRunSummary( + repository="ROCm/TheRock", + branch="main", + commit=f"sha-{run_id}", + workflow="multi_arch_ci.yml", + run_id=run_id, + status="completed", + conclusion="success", + timestamp="2026-06-17T20:00:00Z", + html_url=f"https://github.com/ROCm/TheRock/actions/runs/{run_id}", + ) + return BaselineRun( + source_ref=summary, + platform="linux", + job_health=WorkflowJobHealth( + required_name_substrings=("Build",), + matched_job_names=("Build",), + failed_job_names=(), + missing_name_substrings=(), + ), + artifact_availability=ArtifactAvailability( + required_artifacts=(), + matched_filenames=tuple(matched_filenames), + missing_artifacts=(), + ), + ) + + +def _selector(baseline): + """Return an injected baseline_selector that always yields `baseline`.""" + return lambda required: baseline + + +class ModeParsingTest(unittest.TestCase): + def test_default_is_dry_run(self): + import os + + os.environ.pop("STAGE_REUSE_MODE", None) + self.assertEqual(StageReuseMode.from_environ(), StageReuseMode.DRY_RUN) + + def test_explicit_modes(self): + import os + + for value, expected in [ + ("dry-run", StageReuseMode.DRY_RUN), + ("skip-stage", StageReuseMode.ENFORCE), + ("skip-stage", StageReuseMode.ENFORCE), + ("garbage", StageReuseMode.DRY_RUN), + ]: + os.environ["STAGE_REUSE_MODE"] = value + self.assertEqual(StageReuseMode.from_environ(), expected) + os.environ.pop("STAGE_REUSE_MODE", None) + + +class AvailabilityGateTest(unittest.TestCase): + def test_dry_run_verifies_artifacts_present(self): + # math-libs changed -> compiler-runtime unaffected (candidate). + # Baseline HAS base artifact -> reported as WOULD skip, but applied=(). + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.DRY_RUN, + target_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + self.assertIn("compiler-runtime", result.candidate_stages) + self.assertIn("compiler-runtime", result.available_stages) + self.assertEqual(result.applied_reuse_stages, ()) # dry-run applies nothing + self.assertEqual(result.baseline_run_id, "123") + joined = "\n".join(result.report_lines) + self.assertIn("available in baseline", joined) + self.assertIn("WOULD be skipped", joined) + + def test_dry_run_unaffected_but_artifacts_missing_rebuilds(self): + # compiler-runtime unaffected, but baseline only has blas (not base). + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.DRY_RUN, + target_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["blas_lib_generic.tar.zst"])), + ) + self.assertIn("compiler-runtime", result.candidate_stages) + self.assertIn("compiler-runtime", result.unavailable_stages) + self.assertEqual(result.available_stages, ()) + joined = "\n".join(result.report_lines) + self.assertIn("artifacts NOT available", joined) + + def test_no_baseline_found_rebuilds_candidates(self): + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.DRY_RUN, + target_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(None), + ) + self.assertIn("compiler-runtime", result.candidate_stages) + self.assertEqual(result.available_stages, ()) + self.assertIsNone(result.baseline_run_id) + joined = "\n".join(result.report_lines) + self.assertIn("no baseline run contains artifacts", joined) + + def test_partial_family_availability_rebuilds(self): + # Needs base for two families; baseline only has the generic archive. + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.DRY_RUN, + target_families=["generic", "gfx94X-dcgpu"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + self.assertIn("compiler-runtime", result.unavailable_stages) + self.assertEqual(result.available_stages, ()) + + def test_enforce_applies_only_available_stages(self): + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.ENFORCE, + target_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + self.assertEqual(result.applied_reuse_stages, ("compiler-runtime",)) + joined = "\n".join(result.report_lines) + self.assertIn("WILL be skipped", joined) + + def test_baseline_lookup_error_is_safe(self): + def boom(required): + raise RuntimeError("network down") + + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.ENFORCE, + target_families=["generic"], + topology=FakeTopology(), + baseline_selector=boom, + ) + self.assertEqual(result.applied_reuse_stages, ()) + joined = "\n".join(result.report_lines) + self.assertIn("baseline lookup failed", joined) + + +class GuardrailTest(unittest.TestCase): + def test_full_ci_trigger_skips_baseline_query(self): + called = {"n": 0} + + def selector(required): + called["n"] += 1 + return None + + result = compute_auto_stage_reuse( + changed_files=["build_tools/foo.py"], + mode=StageReuseMode.ENFORCE, + topology=FakeTopology(), + baseline_selector=selector, + ) + self.assertTrue(result.full_rebuild_required) + self.assertEqual(result.applied_reuse_stages, ()) + self.assertEqual(called["n"], 0) + + def test_no_diff_is_conservative(self): + result = compute_auto_stage_reuse( + changed_files=None, + mode=StageReuseMode.ENFORCE, + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + self.assertTrue(result.full_rebuild_required) + self.assertEqual(result.applied_reuse_stages, ()) + + def test_step_summary_shows_baseline_and_availability(self): + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.DRY_RUN, + target_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + summary = srd.render_step_summary(result) + self.assertIn("baseline run checked: `123`", summary) + self.assertIn("available in baseline", summary) + self.assertIn("no build steps were skipped", summary) +class DefaultBaselineSelectorTest(unittest.TestCase): + """_default_baseline_selector must fetch real branch history and never pass + an empty ordered_commit_shas window while current_commit_sha is set.""" + + def _run_with_env(self, env, fake_history, fake_select): + import os + import importlib + + old_env = {k: os.environ.get(k) for k in env} + os.environ.update({k: v for k, v in env.items() if v is not None}) + for k, v in env.items(): + if v is None: + os.environ.pop(k, None) + + import baseline_runs + import github_actions_api + + orig_select = baseline_runs.select_baseline_run + orig_hist = getattr( + github_actions_api, "gha_query_recent_branch_commits", None + ) + captured = {} + + def _capturing_select(**kwargs): + captured.update(kwargs) + return fake_select + + baseline_runs.select_baseline_run = _capturing_select + github_actions_api.gha_query_recent_branch_commits = fake_history + importlib.reload(srd) if False else None + try: + selector = srd._default_baseline_selector(platform="linux") + result = selector([("base", "generic")]) + finally: + baseline_runs.select_baseline_run = orig_select + if orig_hist is not None: + github_actions_api.gha_query_recent_branch_commits = orig_hist + for k, v in old_env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + return captured, result + + def test_history_is_fetched_and_threaded(self): + def fake_history(**kwargs): + return ["sha-current", "sha-old", "sha-older"] + + captured, _ = self._run_with_env( + {"STAGE_REUSE_CURRENT_SHA": "sha-current"}, + fake_history, + fake_select="baseline", + ) + # Real history passed through (NOT an empty list). + self.assertEqual( + captured["ordered_commit_shas"], ["sha-current", "sha-old", "sha-older"] + ) + self.assertEqual(captured["current_commit_sha"], "sha-current") + + def test_empty_history_disables_commit_rule(self): + def fake_history(**kwargs): + return [] + + captured, _ = self._run_with_env( + {"STAGE_REUSE_CURRENT_SHA": "sha-current"}, + fake_history, + fake_select="baseline", + ) + # Disabled rather than enabled-with-empty-window: both None. + self.assertIsNone(captured["current_commit_sha"]) + self.assertIsNone(captured["ordered_commit_shas"]) + + def test_history_fetch_error_disables_commit_rule(self): + def fake_history(**kwargs): + raise RuntimeError("api down") + + captured, _ = self._run_with_env( + {"STAGE_REUSE_CURRENT_SHA": "sha-current"}, + fake_history, + fake_select="baseline", + ) + self.assertIsNone(captured["current_commit_sha"]) + self.assertIsNone(captured["ordered_commit_shas"]) + + def test_no_sha_means_no_history_fetch(self): + calls = {"n": 0} + + def fake_history(**kwargs): + calls["n"] = 1 + return ["x"] + + captured, _ = self._run_with_env( + {"STAGE_REUSE_CURRENT_SHA": None}, + fake_history, + fake_select="baseline", + ) + self.assertEqual(calls["n"], 0) + self.assertIsNone(captured["current_commit_sha"]) + self.assertIsNone(captured["ordered_commit_shas"]) + + + +if __name__ == "__main__": + unittest.main() From 886eafc9af27a3d98dd4729983f6499836633d49 Mon Sep 17 00:00:00 2001 From: erman-gurses Date: Mon, 6 Jul 2026 14:43:28 +0000 Subject: [PATCH 09/23] Format --- .../github_actions/configure_multi_arch_ci.py | 10 +- .../github_actions/stage_reuse_decision.py | 130 ++++++++++++------ .../tests/stage_reuse_decision_test.py | 10 +- 3 files changed, 103 insertions(+), 47 deletions(-) diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index 295964cf026..eb3c41344f6 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -702,6 +702,7 @@ def decide_jobs( compute_auto_stage_reuse, render_step_summary, ) + target_families = _stage_reuse_target_families(ci_inputs) auto = compute_auto_stage_reuse( changed_files=git_context.changed_files, @@ -765,13 +766,18 @@ def decide_jobs( # Step 4: Select Targets # --------------------------------------------------------------------------- + def _stage_reuse_target_families(ci_inputs: "CIInputs") -> list[str]: - families = list(dict.fromkeys( - [*ci_inputs.linux_amdgpu_families, *ci_inputs.windows_amdgpu_families])) + families = list( + dict.fromkeys( + [*ci_inputs.linux_amdgpu_families, *ci_inputs.windows_amdgpu_families] + ) + ) if "generic" not in families: families.append("generic") return families + def _validate_family_names( names: list[str], known: dict[str, dict], diff --git a/build_tools/github_actions/stage_reuse_decision.py b/build_tools/github_actions/stage_reuse_decision.py index d507e51602c..0edeec87717 100644 --- a/build_tools/github_actions/stage_reuse_decision.py +++ b/build_tools/github_actions/stage_reuse_decision.py @@ -92,7 +92,9 @@ def _matched_filenames(baseline) -> set: return set(baseline.artifact_availability.matched_filenames) -def _stage_artifacts_available(topology, stage_name, target_families, available_filenames): +def _stage_artifacts_available( + topology, stage_name, target_families, available_filenames +): """True when every artifact this stage produces has an archive present.""" from artifact_manager import ARTIFACT_COMPONENTS from _therock_utils.artifact_backend import ARTIFACT_EXTENSIONS @@ -157,17 +159,26 @@ def compute_auto_stage_reuse( if impact.full_rebuild_required or not candidates: lines = _format_report( - mode=mode, candidates=candidates, rebuild=rebuild, + mode=mode, + candidates=candidates, + rebuild=rebuild, full_rebuild_required=impact.full_rebuild_required, - reasons=tuple(impact.reasons), baseline_run_id=None, - available=(), unavailable=candidates, + reasons=tuple(impact.reasons), + baseline_run_id=None, + available=(), + unavailable=candidates, ) return AutoStageReuse( - mode=mode, candidate_stages=candidates, rebuild_stages=rebuild, + mode=mode, + candidate_stages=candidates, + rebuild_stages=rebuild, full_rebuild_required=impact.full_rebuild_required, - baseline_run_id=None, baseline_html_url=None, - available_stages=(), unavailable_stages=candidates, - applied_reuse_stages=(), reasons=tuple(impact.reasons), + baseline_run_id=None, + baseline_html_url=None, + available_stages=(), + unavailable_stages=candidates, + applied_reuse_stages=(), + reasons=tuple(impact.reasons), report_lines=lines, ) @@ -186,7 +197,9 @@ def compute_auto_stage_reuse( unavailable: list[str] = [] if baseline is not None: for stage_name in candidates: - if _stage_artifacts_available(topology, stage_name, families, available_filenames): + if _stage_artifacts_available( + topology, stage_name, families, available_filenames + ): available.append(stage_name) else: unavailable.append(stage_name) @@ -200,17 +213,28 @@ def compute_auto_stage_reuse( baseline_url = baseline.html_url if baseline is not None else None lines = _format_report( - mode=mode, candidates=candidates, rebuild=rebuild, - full_rebuild_required=False, reasons=tuple(impact.reasons), - baseline_run_id=baseline_run_id, available=available_t, - unavailable=unavailable_t, baseline_error=baseline_error, + mode=mode, + candidates=candidates, + rebuild=rebuild, + full_rebuild_required=False, + reasons=tuple(impact.reasons), + baseline_run_id=baseline_run_id, + available=available_t, + unavailable=unavailable_t, + baseline_error=baseline_error, ) return AutoStageReuse( - mode=mode, candidate_stages=candidates, rebuild_stages=rebuild, - full_rebuild_required=False, baseline_run_id=baseline_run_id, - baseline_html_url=baseline_url, available_stages=available_t, - unavailable_stages=unavailable_t, applied_reuse_stages=applied, - reasons=tuple(impact.reasons), report_lines=lines, + mode=mode, + candidate_stages=candidates, + rebuild_stages=rebuild, + full_rebuild_required=False, + baseline_run_id=baseline_run_id, + baseline_html_url=baseline_url, + available_stages=available_t, + unavailable_stages=unavailable_t, + applied_reuse_stages=applied, + reasons=tuple(impact.reasons), + report_lines=lines, ) @@ -251,7 +275,7 @@ def _default_baseline_selector(*, platform: str | None) -> BaselineSelector: if not ordered_commit_shas: effective_commit_sha = None ordered_commit_shas = None - except Exception as exc: + except Exception as exc: print( f"{LOG_PREFIX} could not fetch branch history " f"({exc}); skipping commit-compatibility rule." @@ -276,19 +300,37 @@ def _select(required): def _empty_result(mode, *, full_rebuild_required=False, reasons=(), report_lines=()): return AutoStageReuse( - mode=mode, candidate_stages=(), rebuild_stages=(), + mode=mode, + candidate_stages=(), + rebuild_stages=(), full_rebuild_required=full_rebuild_required, - baseline_run_id=None, baseline_html_url=None, - available_stages=(), unavailable_stages=(), - applied_reuse_stages=(), reasons=reasons, report_lines=report_lines, + baseline_run_id=None, + baseline_html_url=None, + available_stages=(), + unavailable_stages=(), + applied_reuse_stages=(), + reasons=reasons, + report_lines=report_lines, ) -def _format_report(*, mode, candidates, rebuild, full_rebuild_required, - reasons, baseline_run_id, available, unavailable, baseline_error=None): +def _format_report( + *, + mode, + candidates, + rebuild, + full_rebuild_required, + reasons, + baseline_run_id, + available, + unavailable, + baseline_error=None, +): lines: list[str] = [f"{LOG_PREFIX} mode={mode.value}"] if full_rebuild_required: - lines.append(f"{LOG_PREFIX} conservative full rebuild: no stages eligible for reuse.") + lines.append( + f"{LOG_PREFIX} conservative full rebuild: no stages eligible for reuse." + ) for reason in reasons: lines.append(f"{LOG_PREFIX} reason: {reason}") return tuple(lines) @@ -296,35 +338,45 @@ def _format_report(*, mode, candidates, rebuild, full_rebuild_required, lines.append(f"{LOG_PREFIX} no unaffected stages; all stages rebuild.") return tuple(lines) if baseline_error: - lines.append(f"{LOG_PREFIX} baseline lookup failed ({baseline_error}); " - f"cannot verify artifacts, rebuilding all candidates.") + lines.append( + f"{LOG_PREFIX} baseline lookup failed ({baseline_error}); " + f"cannot verify artifacts, rebuilding all candidates." + ) elif baseline_run_id: lines.append(f"{LOG_PREFIX} baseline run for artifact check: {baseline_run_id}") else: - lines.append(f"{LOG_PREFIX} no baseline run contains artifacts for all " - f"candidate stages; rebuilding all candidates.") - verb = ("WILL be skipped" if mode is StageReuseMode.ENFORCE - else "WOULD be skipped") + lines.append( + f"{LOG_PREFIX} no baseline run contains artifacts for all " + f"candidate stages; rebuilding all candidates." + ) + verb = "WILL be skipped" if mode is StageReuseMode.ENFORCE else "WOULD be skipped" for stage in available: - lines.append(f"{LOG_PREFIX} stage '{stage}' unaffected AND available in baseline -> {verb}") + lines.append( + f"{LOG_PREFIX} stage '{stage}' unaffected AND available in baseline -> {verb}" + ) for stage in unavailable: - lines.append(f"{LOG_PREFIX} stage '{stage}' unaffected but artifacts NOT available -> rebuild") + lines.append( + f"{LOG_PREFIX} stage '{stage}' unaffected but artifacts NOT available -> rebuild" + ) if rebuild: lines.append(f"{LOG_PREFIX} stages rebuilding (impacted): {', '.join(rebuild)}") if mode is StageReuseMode.DRY_RUN and available: - lines.append(f"{LOG_PREFIX} dry-run: prebuilt_stages NOT modified; all stages " - f"still build.") + lines.append( + f"{LOG_PREFIX} dry-run: prebuilt_stages NOT modified; all stages " + f"still build." + ) return tuple(lines) + def _format_stage_list(stages: tuple[str, ...]) -> str: """Render a tuple of stage names as backticked, comma-separated markdown.""" if not stages: return "_none_" return ", ".join(f"`{stage}`" for stage in stages) + def render_step_summary(result: AutoStageReuse) -> str: - """Render a GitHub step-summary markdown block for the analysis. - """ + """Render a GitHub step-summary markdown block for the analysis.""" baseline = f"`{result.baseline_run_id}`" if result.baseline_run_id else "_none_" candidates = _format_stage_list(result.candidate_stages) available = _format_stage_list(result.available_stages) @@ -348,4 +400,4 @@ def render_step_summary(result: AutoStageReuse) -> str: "verified against the baseline run above. Set " "`STAGE_REUSE_MODE=skip-stage` after review to enable skipping." ) - return "\n".join(out) \ No newline at end of file + return "\n".join(out) diff --git a/build_tools/github_actions/tests/stage_reuse_decision_test.py b/build_tools/github_actions/tests/stage_reuse_decision_test.py index b6a2c95538a..0b4eb5f920c 100644 --- a/build_tools/github_actions/tests/stage_reuse_decision_test.py +++ b/build_tools/github_actions/tests/stage_reuse_decision_test.py @@ -1,8 +1,7 @@ # Copyright Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT -"""Tests for stage_reuse_decision: impact + baseline-availability gates. -""" +"""Tests for stage_reuse_decision: impact + baseline-availability gates.""" from pathlib import Path import sys @@ -248,6 +247,8 @@ def test_step_summary_shows_baseline_and_availability(self): self.assertIn("baseline run checked: `123`", summary) self.assertIn("available in baseline", summary) self.assertIn("no build steps were skipped", summary) + + class DefaultBaselineSelectorTest(unittest.TestCase): """_default_baseline_selector must fetch real branch history and never pass an empty ordered_commit_shas window while current_commit_sha is set.""" @@ -266,9 +267,7 @@ def _run_with_env(self, env, fake_history, fake_select): import github_actions_api orig_select = baseline_runs.select_baseline_run - orig_hist = getattr( - github_actions_api, "gha_query_recent_branch_commits", None - ) + orig_hist = getattr(github_actions_api, "gha_query_recent_branch_commits", None) captured = {} def _capturing_select(**kwargs): @@ -349,6 +348,5 @@ def fake_history(**kwargs): self.assertIsNone(captured["ordered_commit_shas"]) - if __name__ == "__main__": unittest.main() From ce75e62254d693c5a2ccf8cafc4359e89e915548 Mon Sep 17 00:00:00 2001 From: ammallya Date: Mon, 6 Jul 2026 20:13:18 +0000 Subject: [PATCH 10/23] ci(stage-reuse): verify baseline artifacts per-platform before reusing stages --- .github/workflows/multi_arch_ci.yml | 16 +- .github/workflows/setup_multi_arch.yml | 19 +- .../github_actions/configure_multi_arch_ci.py | 73 +++- .../github_actions/stage_reuse_decision.py | 377 ++++++++++++++---- .../tests/stage_reuse_decision_test.py | 98 ++++- 5 files changed, 475 insertions(+), 108 deletions(-) diff --git a/.github/workflows/multi_arch_ci.yml b/.github/workflows/multi_arch_ci.yml index 711eb5d1a51..03afd0f6b8d 100644 --- a/.github/workflows/multi_arch_ci.yml +++ b/.github/workflows/multi_arch_ci.yml @@ -45,7 +45,19 @@ on: stage_reuse_mode: type: string default: "dry-run" - description: "dry-run (report only) | skip-stage (actually skip unaffected stages)" + # This drives the AUTOMATIC stage-reuse detection layer, which is + # separate from the manual `prebuilt_stages` input above. dry-run only + # reports which unaffected stages could be reused; reuse-stage actually + # reuses them (skips the build and copies artifacts from a baseline run). + description: "dry-run (report only) | reuse-stage (auto-reuse unaffected stages)" + stage_reuse_max_age_hours: + type: string + default: "72" + description: "Automatic stage reuse: reject baseline runs older than this many hours" + stage_reuse_commit_history: + type: string + default: "50" + description: "Automatic stage reuse: number of recent branch commits to fetch for ancestry" changed_projects: type: string default: "" @@ -87,6 +99,8 @@ jobs: prebuilt_stages: ${{ inputs.prebuilt_stages || '' }} baseline_run_id: ${{ inputs.baseline_run_id || '' }} stage_reuse_mode: ${{ inputs.stage_reuse_mode || 'dry-run' }} + stage_reuse_max_age_hours: ${{ inputs.stage_reuse_max_age_hours || '72' }} + stage_reuse_commit_history: ${{ inputs.stage_reuse_commit_history || '50' }} build_pytorch: ${{ github.event_name != 'workflow_dispatch' || inputs.build_pytorch }} build_jax: false diff --git a/.github/workflows/setup_multi_arch.yml b/.github/workflows/setup_multi_arch.yml index 2b2eec53536..6c589aedf3b 100644 --- a/.github/workflows/setup_multi_arch.yml +++ b/.github/workflows/setup_multi_arch.yml @@ -44,9 +44,22 @@ on: type: string default: "" stage_reuse_mode: - description: "dry-run (default) only reports which stages would be skipped; skip-stage actually skips unaffected stages." + description: >- + Automatic stage-reuse mode (separate from the manual prebuilt_stages + input): "dry-run" (default) only reports which unaffected stages could + be reused; "reuse-stage" actually reuses them. type: string default: "dry-run" + stage_reuse_max_age_hours: + description: >- + Automatic stage reuse: reject baseline runs older than this many hours. default: "72" + type: string + default: "72" + stage_reuse_commit_history: + description: >- + Automatic stage reuse: number of recent branch commits to fetch for ancestry. default: "50" + type: string + default: "50" build_pytorch: description: "Build PyTorch wheels." type: boolean @@ -165,8 +178,8 @@ jobs: BASELINE_RUN_ID: ${{ inputs.baseline_run_id }} STAGE_REUSE_MODE: ${{ inputs.stage_reuse_mode }} STAGE_REUSE_CURRENT_SHA: ${{ steps.checkout.outputs.commit }} - STAGE_REUSE_MAX_AGE_HOURS: "72" - STAGE_REUSE_COMMIT_HISTORY: "50" + STAGE_REUSE_MAX_AGE_HOURS: ${{ inputs.stage_reuse_max_age_hours }} + STAGE_REUSE_COMMIT_HISTORY: ${{ inputs.stage_reuse_commit_history }} # Skip path filtering for external repos (git sha won't exist in TheRock) SKIP_PATH_FILTERS: ${{ inputs.external_repo != '' && 'true' || '' }} CI_CONFIG_PATH: ci-config diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index eb3c41344f6..7389c7a2279 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -681,8 +681,13 @@ def _determine_test_type( def decide_jobs( ci_inputs: CIInputs, git_context: GitContext, + targets: "TargetSelection | None" = None, ) -> JobDecisions: - """Determine which job groups to run, skip, or satisfy with prebuilt files.""" + """Determine which job groups to run, skip, or satisfy with prebuilt files. + ``targets`` (the per-platform family selection) is used to scope automatic + stage reuse to the platforms actually being built, so a stage is only + reused when its artifacts exist for every one of those platforms. + """ # Build ROCm. # Parse explicit prebuilt stages from workflow_dispatch input. These are @@ -693,29 +698,43 @@ def decide_jobs( for stage in _parse_prebuilt_stages(ci_inputs.prebuilt_stages): stage_decisions[stage] = JobAction.PREBUILT - # Behind STAGE_REUSE_MODE: in dry-run we only PRINT - # which stages WOULD be skipped and apply nothing; in "skip-stage" the - # eligible stages are merged into stage_decisions so the orchestrator - # skips their builds and copies artifacts instead. + # Automatic stage reuse, behind STAGE_REUSE_MODE: in dry-run we only report + # which stages WOULD be reused and apply nothing; in "reuse-stage" the + # eligible stages are merged into stage_decisions so the orchestrator skips + # their builds and copies artifacts instead. + # These imports are intentionally local. stage_reuse_decision imports from + # sibling CI modules (baseline_runs, stage_impact, github_actions_api) that + # in turn pull in heavier deps; importing it lazily here keeps module import + # of configure_multi_arch_ci cheap and avoids an import cycle between the two + # modules (stage_reuse plumbing conceptually depends on this pipeline). + from stage_reuse_decision import ( StageReuseMode, compute_auto_stage_reuse, render_step_summary, + log_report, + render_step_summary, + stage_reuse_target_families, ) - target_families = _stage_reuse_target_families(ci_inputs) + build_platforms = _stage_reuse_build_platforms(ci_inputs, targets) + target_families = stage_reuse_target_families( + ci_inputs.linux_amdgpu_families, + ci_inputs.windows_amdgpu_families, + ) auto = compute_auto_stage_reuse( changed_files=git_context.changed_files, mode=StageReuseMode.from_environ(), + platforms=build_platforms, target_families=target_families, ) - for line in auto.report_lines: - print(line) + log_report(auto) + try: gha_append_step_summary(render_step_summary(auto)) - except Exception as exc: # never fail CI config on a reporting + except Exception as exc: # never fail CI config on a reporting error print(f"[STAGE-REUSE] could not write step summary: {exc}") - # Only skip-stage mode returns non-empty applied_reuse_stages. + # Only reuse-stage mode returns non-empty applied_reuse_stages. for stage in auto.applied_reuse_stages: stage_decisions.setdefault(stage, JobAction.PREBUILT) build_rocm = BuildRocmDecision( @@ -767,15 +786,31 @@ def decide_jobs( # --------------------------------------------------------------------------- -def _stage_reuse_target_families(ci_inputs: "CIInputs") -> list[str]: - families = list( - dict.fromkeys( - [*ci_inputs.linux_amdgpu_families, *ci_inputs.windows_amdgpu_families] - ) - ) - if "generic" not in families: - families.append("generic") - return families +def _stage_reuse_build_platforms( + ci_inputs: "CIInputs", + targets: "TargetSelection | None", +) -> list[str]: + """Platforms whose baselines must contain a stage's artifacts to reuse it. + Automatic reuse merges into the prebuilt_stages that flow to BOTH the Linux + and Windows build configs, so a stage may only be reused when its artifacts + are available for every platform actually being built. When target selection + is known we use it directly; otherwise we fall back to the requested family + inputs, defaulting to Linux only. + """ + if targets is not None: + platforms = [] + if targets.linux_families: + platforms.append("linux") + if targets.windows_families: + platforms.append("windows") + return platforms or ["linux"] + + platforms = [] + if ci_inputs.linux_amdgpu_families: + platforms.append("linux") + if ci_inputs.windows_amdgpu_families: + platforms.append("windows") + return platforms or ["linux"] def _validate_family_names( diff --git a/build_tools/github_actions/stage_reuse_decision.py b/build_tools/github_actions/stage_reuse_decision.py index 0edeec87717..213edad5ce3 100644 --- a/build_tools/github_actions/stage_reuse_decision.py +++ b/build_tools/github_actions/stage_reuse_decision.py @@ -4,42 +4,99 @@ """Auto-compute per-stage rebuild/reuse decisions for multi-arch CI. -This script decides per build stage, whether the stage can be satisfied with prebuilt -artifacts (reuse) instead of being rebuilt. - ---------------------------------- -This module is wired into CI behind a two-way mode switch so it can be -observed before it changes anything: -* ``dry-run`` - DEFAULT. Compute the analysis and PRINT, for each stage that - *would* be skipped, a line to the console + step summary, but - return NO auto stages, so ``prebuilt_stages`` is unchanged and - every stage still builds exactly. -* ``skip-stage`` - Compute the analysis and actually return the reuse stages so - the orchestrator copies their artifacts and skips the build. - +This module decides, per build stage, whether the stage can be satisfied with +prebuilt artifacts (reuse) instead of being rebuilt. It reuses the existing +``stage_impact`` and ``baseline_runs`` tooling and adds two gates before a stage +is reported reusable: + +* Impact gate (``stage_impact.analyze_stage_impact``) - the change does not + affect the stage, so it is a *candidate* for reuse. +* Availability gate (``baseline_runs.select_baseline_run``) - a single healthy, + commit-compatible baseline run actually contains the artifacts that stage + would produce, verified independently for every platform being built. + +Mode switch +----------- +The module is wired into CI behind a two-way ``STAGE_REUSE_MODE`` switch so it +can be observed before it changes anything: + +* ``dry-run`` - DEFAULT. Compute the analysis and LOG, for each stage that + *would* be reused, a line to the console + step summary, but + return NO auto stages, so ``prebuilt_stages`` is unchanged + and every stage still builds exactly. +* ``reuse-stage`` - Compute the analysis and actually return the reuse stages so + the orchestrator copies their artifacts and skips the build. + +Note: this ``STAGE_REUSE_MODE`` switch drives the *automatic* detection layer. +It is orthogonal to the ``prebuilt_stages`` workflow input, which is the +explicit, manual list of stages to reuse and is always honored regardless of +mode. + +Environment contract +-------------------- +``_default_baseline_selector`` reads the following environment variables. These +are set by ``setup_multi_arch.yml`` and form the stage-reuse interface: + +* ``STAGE_REUSE_MODE`` - ``dry-run`` (default) or ``reuse-stage``. +* ``GITHUB_REPOSITORY`` - ``owner/repo`` (default ``ROCm/TheRock``). +* ``STAGE_REUSE_BASELINE_BRANCH`` - baseline branch to search (default ``main``). +* ``STAGE_REUSE_BASELINE_WORKFLOW`` - baseline workflow file + (default ``multi_arch_ci.yml``). +* ``STAGE_REUSE_CURRENT_SHA`` - current commit SHA; enables the + commit-compatibility rule when set. +* ``STAGE_REUSE_MAX_AGE_HOURS`` - recency window in hours; disables the recency + rule when unset. +* ``STAGE_REUSE_COMMIT_HISTORY`` - number of branch commits to fetch for + ancestry (default ``50``). """ + from __future__ import annotations import enum import os +import logging +import functools from dataclasses import dataclass, field from typing import Callable, Optional, Sequence +logger = logging.getLogger(__name__) + class StageReuseMode(enum.Enum): DRY_RUN = "dry-run" - ENFORCE = "skip-stage" + ENFORCE = "reuse-stage" @staticmethod def from_environ(default: "StageReuseMode" = None) -> "StageReuseMode": """Read STAGE_REUSE_MODE; default to dry-run when unset/invalid.""" default = default or StageReuseMode.DRY_RUN raw = (os.environ.get("STAGE_REUSE_MODE", "") or "").strip().lower() - for mode in StageReuseMode: - if raw == mode.value: - return mode - return default + return _STAGE_REUSE_MODE_ALIASES.get(raw, default) + + +# Accepted spellings for each mode. ``skip-stage`` is the legacy alias that +# predates the ``reuse-stage`` rename and is kept for backwards compatibility +# with any callers still passing the old value. +_STAGE_REUSE_MODE_ALIASES = { + "dry-run": StageReuseMode.DRY_RUN, + "reuse-stage": StageReuseMode.ENFORCE, + "skip-stage": StageReuseMode.ENFORCE, +} + + +@dataclass(frozen=True) +class StageReusePlan: + """Result of the *planning* step: which stages are reuse candidates. + This is a pure function of the changed files and topology -- no baseline + selection, artifact verification, or reporting. Keeping it separate lets + callers reuse the impact decision without triggering any network/API work. + """ + + candidate_stages: tuple[str, ...] + rebuild_stages: tuple[str, ...] + full_rebuild_required: bool + reasons: tuple[str, ...] BaselineSelector = Callable[[Sequence["object"]], Optional["object"]] @@ -47,7 +104,7 @@ def from_environ(default: "StageReuseMode" = None) -> "StageReuseMode": @dataclass(frozen=True) class AutoStageReuse: - """Result of the auto stage-reuse analysis.""" + """Result of the auto stage-reuse analysis (planning + verification).""" mode: StageReuseMode candidate_stages: tuple[str, ...] @@ -60,11 +117,27 @@ class AutoStageReuse: applied_reuse_stages: tuple[str, ...] reasons: tuple[str, ...] report_lines: tuple[str, ...] = field(default_factory=tuple) + platform_available: dict[str, tuple[str, ...]] = field(default_factory=dict) LOG_PREFIX = "[STAGE-REUSE]" +def stage_reuse_target_families( + linux_amdgpu_families: Sequence[str] = (), + windows_amdgpu_families: Sequence[str] = (), +) -> list[str]: + """Family list whose artifacts must be verified for stage reuse. + Lives here (next to the reuse implementation) rather than in the caller, + since it only exists to prepare inputs for ``compute_auto_stage_reuse``. + Always includes ``generic`` because every stage produces a generic archive. + """ + families = list(dict.fromkeys([*linux_amdgpu_families, *windows_amdgpu_families])) + if "generic" not in families: + families.append("generic") + return families + + def required_artifacts_for_stages(topology, stage_names, target_families): """Artifact/family pairs the given stages produce.""" from baseline_runs import RequiredArtifact @@ -120,25 +193,24 @@ def _stage_artifacts_available( return True -def compute_auto_stage_reuse( +def plan_stage_reuse( *, changed_files: Sequence[str] | None, - mode: StageReuseMode, platform: str | None = None, - target_families: Sequence[str] = (), topology=None, - baseline_selector: BaselineSelector | None = None, -) -> AutoStageReuse: - """Compute auto stage-reuse decisions, verified against a baseline run.""" +) -> StageReusePlan: + """Planning step: which stages are unaffected reuse candidates. + + Pure decision logic -- no baseline selection, artifact verification, or + reporting -- so it can be reused independently of the CI plumbing. + """ + if changed_files is None: - return _empty_result( - mode, + return StageReusePlan( + candidate_stages=(), + rebuild_stages=(), full_rebuild_required=True, reasons=("no changed-file list available",), - report_lines=( - f"{LOG_PREFIX} mode={mode.value}; no changed-file list " - f"Conservatively rebuilding all stages.", - ), ) if topology is None: @@ -148,22 +220,78 @@ def compute_auto_stage_reuse( from stage_impact import analyze_stage_impact + # TODO(#3399): thread build flags/variant through here for superrepos so a + # baseline built with different flags is not considered reusable. Not needed + # for the current single-variant multi-arch CI, but required before enabling + # reuse across superrepo builds that vary build configuration. impact = analyze_stage_impact( changed_inputs=list(changed_files), topology=topology, platform=platform, ) - candidates = tuple(impact.copy_stages) - rebuild = tuple(impact.rebuild_stages) + return StageReusePlan( + candidate_stages=tuple(impact.copy_stages), + rebuild_stages=tuple(impact.rebuild_stages), + full_rebuild_required=impact.full_rebuild_required, + reasons=tuple(impact.reasons), + ) + + +def compute_auto_stage_reuse( + *, + changed_files: Sequence[str] | None, + mode: StageReuseMode, + platform: str | None = None, + platforms: Sequence[str] | None = None, + target_families: Sequence[str] = (), + topology=None, + baseline_selector: BaselineSelector | None = None, + baseline_selector_factory: Callable[[str], BaselineSelector] | None = None, +) -> AutoStageReuse: + """Compute auto stage-reuse decisions, verified against a baseline run. + A stage is only reusable when it is unaffected by the change AND its + artifacts are present in a healthy baseline run for *every* platform in + ``platforms``. This guards against the case flagged in review where the + resulting ``prebuilt_stages`` are applied to both Linux and Windows build + configs: a stage available only in the Linux baseline must not be skipped + on Windows. ``platform`` remains accepted as a single-platform shorthand + for backwards compatibility. + """ + # Resolve the platform set. ``platforms`` wins; fall back to the legacy + # single ``platform`` arg; default to linux only when neither is given. + resolved_platforms = _resolve_platforms(platforms, platform) + + if topology is None and changed_files is not None: + from _therock_utils.build_topology import get_topology + + topology = get_topology() + + plan = plan_stage_reuse( + changed_files=changed_files, + platform=resolved_platforms[0], + topology=topology, + ) + candidates = plan.candidate_stages + rebuild = plan.rebuild_stages families = tuple(target_families) or ("generic",) + if changed_files is None: + return _empty_result( + mode, + full_rebuild_required=True, + reasons=plan.reasons, + report_lines=( + f"{LOG_PREFIX} mode={mode.value}; no changed-file list. " + f"Conservatively rebuilding all stages.", + ), + ) - if impact.full_rebuild_required or not candidates: + if plan.full_rebuild_required or not candidates: lines = _format_report( mode=mode, candidates=candidates, rebuild=rebuild, - full_rebuild_required=impact.full_rebuild_required, - reasons=tuple(impact.reasons), + full_rebuild_required=plan.full_rebuild_required, + reasons=plan.reasons, baseline_run_id=None, available=(), unavailable=candidates, @@ -172,53 +300,80 @@ def compute_auto_stage_reuse( mode=mode, candidate_stages=candidates, rebuild_stages=rebuild, - full_rebuild_required=impact.full_rebuild_required, + full_rebuild_required=plan.full_rebuild_required, baseline_run_id=None, baseline_html_url=None, available_stages=(), unavailable_stages=candidates, applied_reuse_stages=(), - reasons=tuple(impact.reasons), + reasons=plan.reasons, report_lines=lines, ) required = required_artifacts_for_stages(topology, candidates, families) - baseline = None + # Verify artifact availability independently for each platform. A single + # ``baseline_selector`` (used by tests) applies to all platforms; otherwise + # a per-platform selector is built so each platform is checked against a + # baseline that actually produced that platform's artifacts. baseline_error: Optional[str] = None - try: - if baseline_selector is None: - baseline_selector = _default_baseline_selector(platform=platform) - baseline = baseline_selector(required) - except Exception as exc: - baseline_error = str(exc) - - available_filenames = _matched_filenames(baseline) + per_platform_available: dict[str, tuple[str, ...]] = {} + # The reported baseline run id/url comes from the first platform's baseline + # (they share commit/recency gates); per-platform detail lives in + # platform_available. + reported_baseline_run_id: Optional[str] = None + reported_baseline_url: Optional[str] = None + + for plat in resolved_platforms: + baseline = None + try: + if baseline_selector is not None: + selector = baseline_selector + elif baseline_selector_factory is not None: + selector = baseline_selector_factory(plat) + else: + selector = _default_baseline_selector(platform=plat) + baseline = selector(required) + except Exception as exc: + baseline_error = str(exc) + + if reported_baseline_run_id is None and baseline is not None: + reported_baseline_run_id = baseline.run_id + reported_baseline_url = baseline.html_url + + available_filenames = _matched_filenames(baseline) + avail_here: list[str] = [] + if baseline is not None: + for stage_name in candidates: + if _stage_artifacts_available( + topology, stage_name, families, available_filenames + ): + avail_here.append(stage_name) + per_platform_available[plat] = tuple(avail_here) + + # A stage is available only when present on EVERY platform being built. available: list[str] = [] unavailable: list[str] = [] - if baseline is not None: - for stage_name in candidates: - if _stage_artifacts_available( - topology, stage_name, families, available_filenames - ): - available.append(stage_name) - else: - unavailable.append(stage_name) - else: - unavailable = list(candidates) + + for stage_name in candidates: + if all( + stage_name in per_platform_available.get(plat, ()) + for plat in resolved_platforms + ): + available.append(stage_name) + else: + unavailable.append(stage_name) available_t = tuple(available) unavailable_t = tuple(unavailable) applied = available_t if mode is StageReuseMode.ENFORCE else () - baseline_run_id = baseline.run_id if baseline is not None else None - baseline_url = baseline.html_url if baseline is not None else None lines = _format_report( mode=mode, candidates=candidates, rebuild=rebuild, full_rebuild_required=False, - reasons=tuple(impact.reasons), - baseline_run_id=baseline_run_id, + reasons=plan.reasons, + baseline_run_id=reported_baseline_run_id, available=available_t, unavailable=unavailable_t, baseline_error=baseline_error, @@ -228,18 +383,38 @@ def compute_auto_stage_reuse( candidate_stages=candidates, rebuild_stages=rebuild, full_rebuild_required=False, - baseline_run_id=baseline_run_id, - baseline_html_url=baseline_url, + baseline_run_id=reported_baseline_run_id, + baseline_html_url=reported_baseline_url, available_stages=available_t, unavailable_stages=unavailable_t, applied_reuse_stages=applied, - reasons=tuple(impact.reasons), + reasons=plan.reasons, report_lines=lines, + platform_available=per_platform_available, ) +def _resolve_platforms( + platforms: Sequence[str] | None, platform: str | None +) -> tuple[str, ...]: + """Normalize the platform inputs to a de-duplicated, ordered tuple.""" + if platforms: + resolved = tuple(dict.fromkeys(platforms)) + elif platform: + resolved = (platform,) + else: + resolved = ("linux",) + return resolved + + def _default_baseline_selector(*, platform: str | None) -> BaselineSelector: - """Build a selector bound to baseline_runs.select_baseline_run.""" + """Build a selector bound to baseline_runs.select_baseline_run. + ``select_baseline_run`` already requires each candidate run to have + healthy build jobs (``required_successful_job_name_substrings=("Build",)``) + AND to contain all requested artifacts. A run with no artifacts (e.g. a + docs-only change) therefore fails the availability gate and is never + selected, so no extra "passing build" check is needed here. + """ from baseline_runs import select_baseline_run github_repository = os.environ.get("GITHUB_REPOSITORY", "ROCm/TheRock") @@ -276,26 +451,34 @@ def _default_baseline_selector(*, platform: str | None) -> BaselineSelector: effective_commit_sha = None ordered_commit_shas = None except Exception as exc: - print( - f"{LOG_PREFIX} could not fetch branch history " - f"({exc}); skipping commit-compatibility rule." + logger.warning( + "%s could not fetch branch history (%s); " + "skipping commit-compatibility rule.", + LOG_PREFIX, + exc, ) effective_commit_sha = None ordered_commit_shas = None - def _select(required): - return select_baseline_run( - required_artifacts=required, - github_repository=github_repository, - workflow_name=workflow_name, - branch=branch, - platform=platform or "linux", - current_commit_sha=effective_commit_sha, - ordered_commit_shas=ordered_commit_shas, - max_age_hours=max_age_hours, - ) + # A functools.partial binds the resolved configuration to + # select_baseline_run; the only free argument is required_artifacts, which + # matches the BaselineSelector signature. + return functools.partial( + _invoke_select_baseline_run, + select_baseline_run=select_baseline_run, + github_repository=github_repository, + workflow_name=workflow_name, + branch=branch, + platform=platform or "linux", + current_commit_sha=effective_commit_sha, + ordered_commit_shas=ordered_commit_shas, + max_age_hours=max_age_hours, + ) + - return _select +def _invoke_select_baseline_run(required, *, select_baseline_run, **kwargs): + """Adapter so a partial can present the BaselineSelector(required) shape.""" + return select_baseline_run(required_artifacts=required, **kwargs) def _empty_result(mode, *, full_rebuild_required=False, reasons=(), report_lines=()): @@ -325,8 +508,13 @@ def _format_report( available, unavailable, baseline_error=None, + platforms: Sequence[str] = (), + platform_available: dict[str, tuple[str, ...]] | None = None, ): + platform_available = platform_available or {} lines: list[str] = [f"{LOG_PREFIX} mode={mode.value}"] + if platforms: + lines.append(f"{LOG_PREFIX} platforms verified: {', '.join(platforms)}") if full_rebuild_required: lines.append( f"{LOG_PREFIX} conservative full rebuild: no stages eligible for reuse." @@ -352,11 +540,24 @@ def _format_report( verb = "WILL be skipped" if mode is StageReuseMode.ENFORCE else "WOULD be skipped" for stage in available: lines.append( - f"{LOG_PREFIX} stage '{stage}' unaffected AND available in baseline -> {verb}" + f"{LOG_PREFIX} stage '{stage}' unaffected AND available in " + f"baseline on all platforms -> {verb}" ) for stage in unavailable: + # Call out WHICH platforms are missing the artifacts so the mismatch + # (e.g. present on linux, absent on windows) is visible in the log. + if len(platforms) > 1: + missing = [ + plat + for plat in platforms + if stage not in platform_available.get(plat, ()) + ] + where = f" (missing on: {', '.join(missing)})" if missing else "" + else: + where = "" lines.append( - f"{LOG_PREFIX} stage '{stage}' unaffected but artifacts NOT available -> rebuild" + f"{LOG_PREFIX} stage '{stage}' unaffected but artifacts " + f"NOT available -> rebuild{where}" ) if rebuild: lines.append(f"{LOG_PREFIX} stages rebuilding (impacted): {', '.join(rebuild)}") @@ -389,6 +590,10 @@ def render_step_summary(result: AutoStageReuse) -> str: out.append(f"- unaffected candidates: {candidates}") out.append(f"- available in baseline: {available}") out.append(f"- applied: {applied}") + if result.platform_available: + out.append("- available per platform:") + for plat, stages in result.platform_available.items(): + out.append(f" - {plat}: {_format_stage_list(stages)}") if result.reasons: out.append("- reasons:") for reason in result.reasons: @@ -398,6 +603,12 @@ def render_step_summary(result: AutoStageReuse) -> str: out.append( "> Dry-run only: no build steps were skipped. Artifacts were " "verified against the baseline run above. Set " - "`STAGE_REUSE_MODE=skip-stage` after review to enable skipping." + "`STAGE_REUSE_MODE=reuse-stage` after review to enable skipping." ) return "\n".join(out) + + +def log_report(result: AutoStageReuse) -> None: + """Emit the analysis report lines to the module logger.""" + for line in result.report_lines: + logger.info(line) diff --git a/build_tools/github_actions/tests/stage_reuse_decision_test.py b/build_tools/github_actions/tests/stage_reuse_decision_test.py index 0b4eb5f920c..3d51d5ef6c8 100644 --- a/build_tools/github_actions/tests/stage_reuse_decision_test.py +++ b/build_tools/github_actions/tests/stage_reuse_decision_test.py @@ -110,8 +110,8 @@ def test_explicit_modes(self): for value, expected in [ ("dry-run", StageReuseMode.DRY_RUN), - ("skip-stage", StageReuseMode.ENFORCE), - ("skip-stage", StageReuseMode.ENFORCE), + ("reuse-stage", StageReuseMode.ENFORCE), + ("reuse-stage", StageReuseMode.ENFORCE), ("garbage", StageReuseMode.DRY_RUN), ]: os.environ["STAGE_REUSE_MODE"] = value @@ -348,5 +348,99 @@ def fake_history(**kwargs): self.assertIsNone(captured["ordered_commit_shas"]) +class PlatformAwareAvailabilityTest(unittest.TestCase): + """A stage is only reusable when its artifacts exist for EVERY platform. + + Guards the review concern: prebuilt_stages flow to both the Linux and + Windows build configs, so a stage available only in the Linux baseline must + not be reused when Windows is also being built. + """ + + def _selector_factory(self, per_platform): + """Return a factory yielding a per-platform baseline selector.""" + + def factory(platform): + baseline = per_platform.get(platform) + return lambda required: baseline + + return factory + + def test_stage_reused_only_when_available_on_all_platforms(self): + # compiler-runtime unaffected. Linux baseline HAS base; Windows does NOT. + per_platform = { + "linux": _baseline("L1", ["base_lib_generic.tar.zst"]), + "windows": _baseline("W1", ["blas_lib_generic.tar.zst"]), + } + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.ENFORCE, + platforms=["linux", "windows"], + target_families=["generic"], + topology=FakeTopology(), + baseline_selector_factory=self._selector_factory(per_platform), + ) + # Available on linux but missing on windows -> NOT applied. + self.assertIn("compiler-runtime", result.candidate_stages) + self.assertIn("compiler-runtime", result.unavailable_stages) + self.assertEqual(result.applied_reuse_stages, ()) + self.assertEqual(result.platform_available["linux"], ("compiler-runtime",)) + self.assertEqual(result.platform_available["windows"], ()) + joined = "\n".join(result.report_lines) + self.assertIn("missing on: windows", joined) + + def test_stage_reused_when_present_on_both_platforms(self): + per_platform = { + "linux": _baseline("L1", ["base_lib_generic.tar.zst"]), + "windows": _baseline("W1", ["base_lib_generic.tar.zst"]), + } + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.ENFORCE, + platforms=["linux", "windows"], + target_families=["generic"], + topology=FakeTopology(), + baseline_selector_factory=self._selector_factory(per_platform), + ) + self.assertEqual(result.applied_reuse_stages, ("compiler-runtime",)) + self.assertIn("compiler-runtime", result.available_stages) + + def test_single_platform_default_is_linux(self): + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.ENFORCE, + target_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + self.assertEqual(result.applied_reuse_stages, ("compiler-runtime",)) + self.assertIn("linux", result.platform_available) + + +class PlanStageReuseTest(unittest.TestCase): + """The pure planning step is independent of baseline/reporting.""" + + def test_plan_returns_candidates_without_baseline(self): + plan = srd.plan_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + topology=FakeTopology(), + ) + self.assertIn("compiler-runtime", plan.candidate_stages) + self.assertFalse(plan.full_rebuild_required) + + def test_plan_none_changed_files_is_full_rebuild(self): + plan = srd.plan_stage_reuse(changed_files=None, topology=FakeTopology()) + self.assertTrue(plan.full_rebuild_required) + self.assertEqual(plan.candidate_stages, ()) + + +class TargetFamiliesTest(unittest.TestCase): + def test_always_includes_generic(self): + self.assertEqual(srd.stage_reuse_target_families(), ["generic"]) + + def test_dedupes_and_appends_generic(self): + fams = srd.stage_reuse_target_families(["gfx94x"], ["gfx94x", "gfx120x"]) + self.assertEqual(fams, ["gfx94x", "gfx120x", "generic"]) + + if __name__ == "__main__": unittest.main() From 76397f3980024a412ecaaddb45189489a0ba6221 Mon Sep 17 00:00:00 2001 From: ammallya Date: Mon, 6 Jul 2026 20:56:55 +0000 Subject: [PATCH 11/23] Adding missing debug outputs --- build_tools/github_actions/stage_reuse_decision.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build_tools/github_actions/stage_reuse_decision.py b/build_tools/github_actions/stage_reuse_decision.py index 213edad5ce3..ff8025aeb36 100644 --- a/build_tools/github_actions/stage_reuse_decision.py +++ b/build_tools/github_actions/stage_reuse_decision.py @@ -377,6 +377,8 @@ def compute_auto_stage_reuse( available=available_t, unavailable=unavailable_t, baseline_error=baseline_error, + platforms=resolved_platforms, + platform_available=per_platform_available, ) return AutoStageReuse( mode=mode, From c6fb12778eb6bc9dd3bb92ea86741eecf59b73b0 Mon Sep 17 00:00:00 2001 From: ammallya Date: Mon, 6 Jul 2026 22:48:12 +0000 Subject: [PATCH 12/23] Removing reduandant import --- build_tools/github_actions/configure_multi_arch_ci.py | 1 - 1 file changed, 1 deletion(-) diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index 7389c7a2279..dfb4b0ca3b5 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -713,7 +713,6 @@ def decide_jobs( compute_auto_stage_reuse, render_step_summary, log_report, - render_step_summary, stage_reuse_target_families, ) From fe28af9d82f652a7bffb2d232544e713e0d88f27 Mon Sep 17 00:00:00 2001 From: erman-gurses Date: Tue, 7 Jul 2026 17:33:30 +0000 Subject: [PATCH 13/23] stage-reuse: disable automatic reuse when no platforms are selected --- .../github_actions/configure_multi_arch_ci.py | 4 ++-- build_tools/github_actions/stage_reuse_decision.py | 11 ++++++++++- .../tests/stage_reuse_decision_test.py | 12 ++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index dfb4b0ca3b5..0792e75f394 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -802,14 +802,14 @@ def _stage_reuse_build_platforms( platforms.append("linux") if targets.windows_families: platforms.append("windows") - return platforms or ["linux"] + return platforms platforms = [] if ci_inputs.linux_amdgpu_families: platforms.append("linux") if ci_inputs.windows_amdgpu_families: platforms.append("windows") - return platforms or ["linux"] + return platforms def _validate_family_names( diff --git a/build_tools/github_actions/stage_reuse_decision.py b/build_tools/github_actions/stage_reuse_decision.py index ff8025aeb36..a0c01bc90c7 100644 --- a/build_tools/github_actions/stage_reuse_decision.py +++ b/build_tools/github_actions/stage_reuse_decision.py @@ -50,7 +50,6 @@ ancestry (default ``50``). """ - from __future__ import annotations import enum @@ -257,6 +256,16 @@ def compute_auto_stage_reuse( on Windows. ``platform`` remains accepted as a single-platform shorthand for backwards compatibility. """ + if platforms is not None and len(platforms) == 0 and platform is None: + return _empty_result( + mode, + full_rebuild_required=True, + reasons=("no build platforms selected",), + report_lines=( + f"{LOG_PREFIX} no build platforms selected; automatic stage reuse disabled.", + ), + ) + # Resolve the platform set. ``platforms`` wins; fall back to the legacy # single ``platform`` arg; default to linux only when neither is given. resolved_platforms = _resolve_platforms(platforms, platform) diff --git a/build_tools/github_actions/tests/stage_reuse_decision_test.py b/build_tools/github_actions/tests/stage_reuse_decision_test.py index 3d51d5ef6c8..e2374f1f2de 100644 --- a/build_tools/github_actions/tests/stage_reuse_decision_test.py +++ b/build_tools/github_actions/tests/stage_reuse_decision_test.py @@ -415,6 +415,18 @@ def test_single_platform_default_is_linux(self): self.assertEqual(result.applied_reuse_stages, ("compiler-runtime",)) self.assertIn("linux", result.platform_available) + def test_no_platforms_selected_disables_auto_reuse(self): + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.ENFORCE, + platforms=[], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + self.assertTrue(result.full_rebuild_required) + self.assertEqual(result.applied_reuse_stages, ()) + self.assertIn("no build platforms selected", "\n".join(result.report_lines)) + class PlanStageReuseTest(unittest.TestCase): """The pure planning step is independent of baseline/reporting.""" From 4259e859b94099d6592e83e52f8faec4b42a95ff Mon Sep 17 00:00:00 2001 From: erman-gurses Date: Tue, 7 Jul 2026 18:22:47 +0000 Subject: [PATCH 14/23] stage-reuse: simplify build platform selection --- .../github_actions/configure_multi_arch_ci.py | 27 ++++++++++------- .../tests/configure_multi_arch_ci_test.py | 29 +++++++++++++++++++ 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index 0792e75f394..64f870f3df6 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -796,19 +796,24 @@ def _stage_reuse_build_platforms( is known we use it directly; otherwise we fall back to the requested family inputs, defaulting to Linux only. """ - if targets is not None: - platforms = [] - if targets.linux_families: - platforms.append("linux") - if targets.windows_families: - platforms.append("windows") - return platforms - - platforms = [] - if ci_inputs.linux_amdgpu_families: + platforms: list[str] = [] + + linux_families = ( + targets.linux_families + if targets is not None + else ci_inputs.linux_amdgpu_families + ) + windows_families = ( + targets.windows_families + if targets is not None + else ci_inputs.windows_amdgpu_families + ) + + if linux_families: platforms.append("linux") - if ci_inputs.windows_amdgpu_families: + if windows_families: platforms.append("windows") + return platforms diff --git a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py index c1bb1303267..6abdfdd283b 100644 --- a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py +++ b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py @@ -491,6 +491,35 @@ def test_explicit_prebuilt_stages(self): ) self.assertEqual(result.build_rocm.rebuild_stages, []) + def test_stage_reuse_build_platforms_prefers_targets(self): + """Resolved target selection takes precedence over workflow inputs.""" + inputs = self._inputs( + linux_amdgpu_families=["gfx94x"], + windows_amdgpu_families=["gfx110x"], + ) + + targets = cm.TargetSelection( + linux_families=["gfx94x"], + windows_families=[], + ) + + self.assertEqual( + cm._stage_reuse_build_platforms(inputs, targets), + ["linux"], + ) + + def test_stage_reuse_build_platforms_falls_back_to_inputs(self): + """Workflow inputs are used before target selection is available.""" + inputs = self._inputs( + linux_amdgpu_families=["gfx94x"], + windows_amdgpu_families=["gfx110x"], + ) + + self.assertEqual( + cm._stage_reuse_build_platforms(inputs, None), + ["linux", "windows"], + ) + def test_no_prebuilt_stages_by_default(self): """Without explicit prebuilt_stages, no stage decisions are set.""" result = cm.decide_jobs(self._inputs(), git_context=cm.GitContext()) From da6adb41a9d2b64652c7a5301953027457f7cc48 Mon Sep 17 00:00:00 2001 From: erman-gurses Date: Tue, 7 Jul 2026 18:27:23 +0000 Subject: [PATCH 15/23] Update docstring --- build_tools/github_actions/stage_reuse_decision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/github_actions/stage_reuse_decision.py b/build_tools/github_actions/stage_reuse_decision.py index a0c01bc90c7..65613ae8df9 100644 --- a/build_tools/github_actions/stage_reuse_decision.py +++ b/build_tools/github_actions/stage_reuse_decision.py @@ -32,7 +32,7 @@ explicit, manual list of stages to reuse and is always honored regardless of mode. -Environment contract +Environment Variables -------------------- ``_default_baseline_selector`` reads the following environment variables. These are set by ``setup_multi_arch.yml`` and form the stage-reuse interface: From 6851c0659bbb4bbb3d7e8a39c4b1f56448b3b80f Mon Sep 17 00:00:00 2001 From: erman-gurses Date: Tue, 7 Jul 2026 19:28:38 +0000 Subject: [PATCH 16/23] stage-reuse: propagate baseline run id for auto reuse --- .../github_actions/configure_multi_arch_ci.py | 6 ++- .../tests/configure_multi_arch_ci_test.py | 42 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index 64f870f3df6..83bc025c4c2 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -736,10 +736,14 @@ def decide_jobs( # Only reuse-stage mode returns non-empty applied_reuse_stages. for stage in auto.applied_reuse_stages: stage_decisions.setdefault(stage, JobAction.PREBUILT) + baseline_run_id = ci_inputs.baseline_run_id + if auto.applied_reuse_stages and auto.baseline_run_id: + baseline_run_id = auto.baseline_run_id + build_rocm = BuildRocmDecision( action=JobAction.RUN, stage_decisions=stage_decisions, - baseline_run_id=ci_inputs.baseline_run_id, + baseline_run_id=baseline_run_id, ) # Test ROCm. diff --git a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py index 6abdfdd283b..ee26f8f63d5 100644 --- a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py +++ b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py @@ -1442,7 +1442,49 @@ def test_no_test_labels_has_empty_outputs(self): self.assertEqual(outputs.linux_test_labels, []) self.assertEqual(outputs.windows_test_labels, []) + @patch("configure_multi_arch_ci.decide_jobs") + def test_configure_propagates_auto_reuse_baseline_run_id(self, mock_decide_jobs): + """When auto reuse applies, the selected baseline run id reaches BuildConfig.""" + mock_decide_jobs.return_value = cm.JobDecisions( + build_rocm=cm.BuildRocmDecision( + action=cm.JobAction.RUN, + stage_decisions={ + "compiler-runtime": cm.JobAction.PREBUILT, + "math-libs": cm.JobAction.RUN, + }, + baseline_run_id="123", + ), + test_rocm=cm.TestRocmDecision( + action=cm.JobAction.RUN, + test_type="quick", + ), + build_rocm_python=cm.JobGroupDecision(action=cm.JobAction.RUN), + build_pytorch=cm.JobGroupDecision(action=cm.JobAction.RUN), + test_pytorch=cm.JobGroupDecision(action=cm.JobAction.RUN), + build_jax=cm.JobGroupDecision(action=cm.JobAction.SKIP), + ) + + inputs = cm.CIInputs( + run_id="12345", + event_name="push", + commit_ref="main", + base_ref="HEAD^1", + build_variant="release", + ) + + outputs = cm.configure(inputs, cm.GitContext.empty()) + + self.assertIsNotNone(outputs.builds.linux) + self.assertIsNotNone(outputs.builds.windows) + + self.assertEqual(outputs.builds.linux.baseline_run_id, "123") + self.assertEqual(outputs.builds.windows.baseline_run_id, "123") + self.assertIn("compiler-runtime", outputs.builds.linux.prebuilt_stages) + self.assertIn("compiler-runtime", outputs.builds.windows.prebuilt_stages) + linux_payload = outputs.builds.linux.to_dict() + self.assertEqual(linux_payload["baseline_run_id"], "123") + self.assertEqual(linux_payload["prebuilt_stages"], "compiler-runtime") # --------------------------------------------------------------------------- # Contract: BuildConfig fields match workflow YAML references # --------------------------------------------------------------------------- From 397e824332cf21856bd9e540026d94bd225a135e Mon Sep 17 00:00:00 2001 From: erman-gurses Date: Tue, 7 Jul 2026 20:33:01 +0000 Subject: [PATCH 17/23] stage-reuse: disable reuse when platforms select different baselines --- .../github_actions/stage_reuse_decision.py | 25 ++++++++++++++--- .../tests/stage_reuse_decision_test.py | 27 +++++++++++++++++-- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/build_tools/github_actions/stage_reuse_decision.py b/build_tools/github_actions/stage_reuse_decision.py index 65613ae8df9..a63ebf3608a 100644 --- a/build_tools/github_actions/stage_reuse_decision.py +++ b/build_tools/github_actions/stage_reuse_decision.py @@ -332,6 +332,11 @@ def compute_auto_stage_reuse( reported_baseline_run_id: Optional[str] = None reported_baseline_url: Optional[str] = None + platform_baseline_run_ids: dict[str, Optional[str]] = {} + platform_baseline_urls: dict[str, Optional[str]] = {} + per_platform_available: dict[str, tuple[str, ...]] = {} + baseline_error: Optional[str] = None + for plat in resolved_platforms: baseline = None try: @@ -345,9 +350,8 @@ def compute_auto_stage_reuse( except Exception as exc: baseline_error = str(exc) - if reported_baseline_run_id is None and baseline is not None: - reported_baseline_run_id = baseline.run_id - reported_baseline_url = baseline.html_url + platform_baseline_run_ids[plat] = baseline.run_id if baseline is not None else None + platform_baseline_urls[plat] = baseline.html_url if baseline is not None else None available_filenames = _matched_filenames(baseline) avail_here: list[str] = [] @@ -359,6 +363,21 @@ def compute_auto_stage_reuse( avail_here.append(stage_name) per_platform_available[plat] = tuple(avail_here) + selected_run_ids = {run_id for run_id in platform_baseline_run_ids.values() if run_id is not None} + + if len(selected_run_ids) > 1: + return _empty_result( + mode, + full_rebuild_required=True, + reasons=("automatic reuse resolved different baseline runs per platform",), + report_lines=( + f"{LOG_PREFIX} multiple baseline runs selected across platforms; disabling automatic reuse.", + ), + ) + + reported_baseline_run_id = next(iter(selected_run_ids), None) + reported_baseline_url = next((url for url in platform_baseline_urls.values() if url), None) + # A stage is available only when present on EVERY platform being built. available: list[str] = [] unavailable: list[str] = [] diff --git a/build_tools/github_actions/tests/stage_reuse_decision_test.py b/build_tools/github_actions/tests/stage_reuse_decision_test.py index e2374f1f2de..7cd6a8d4d4d 100644 --- a/build_tools/github_actions/tests/stage_reuse_decision_test.py +++ b/build_tools/github_actions/tests/stage_reuse_decision_test.py @@ -369,7 +369,7 @@ def test_stage_reused_only_when_available_on_all_platforms(self): # compiler-runtime unaffected. Linux baseline HAS base; Windows does NOT. per_platform = { "linux": _baseline("L1", ["base_lib_generic.tar.zst"]), - "windows": _baseline("W1", ["blas_lib_generic.tar.zst"]), + "windows": _baseline("L1", ["blas_lib_generic.tar.zst"]), } result = compute_auto_stage_reuse( changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], @@ -391,7 +391,7 @@ def test_stage_reused_only_when_available_on_all_platforms(self): def test_stage_reused_when_present_on_both_platforms(self): per_platform = { "linux": _baseline("L1", ["base_lib_generic.tar.zst"]), - "windows": _baseline("W1", ["base_lib_generic.tar.zst"]), + "windows": _baseline("L1", ["base_lib_generic.tar.zst"]), } result = compute_auto_stage_reuse( changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], @@ -427,6 +427,29 @@ def test_no_platforms_selected_disables_auto_reuse(self): self.assertEqual(result.applied_reuse_stages, ()) self.assertIn("no build platforms selected", "\n".join(result.report_lines)) + def test_different_platform_baselines_disable_reuse(self): + """Automatic reuse is disabled when platforms resolve to different baseline runs.""" + per_platform = { + "linux": _baseline("L1", ["base_lib_generic.tar.zst"]), + "windows": _baseline("W1", ["base_lib_generic.tar.zst"]), + } + + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.ENFORCE, + platforms=["linux", "windows"], + target_families=["generic"], + topology=FakeTopology(), + baseline_selector_factory=self._selector_factory(per_platform), + ) + + self.assertTrue(result.full_rebuild_required) + self.assertEqual(result.applied_reuse_stages, ()) + self.assertIn( + "automatic reuse resolved different baseline runs per platform", + result.reasons, + ) + class PlanStageReuseTest(unittest.TestCase): """The pure planning step is independent of baseline/reporting.""" From e7086a2ca9233c606e961e40e401dbf1e0c3baba Mon Sep 17 00:00:00 2001 From: erman-gurses Date: Tue, 7 Jul 2026 20:35:57 +0000 Subject: [PATCH 18/23] Black --- .../github_actions/stage_reuse_decision.py | 16 ++++++++++++---- .../tests/configure_multi_arch_ci_test.py | 2 ++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/build_tools/github_actions/stage_reuse_decision.py b/build_tools/github_actions/stage_reuse_decision.py index a63ebf3608a..ec56ab822fe 100644 --- a/build_tools/github_actions/stage_reuse_decision.py +++ b/build_tools/github_actions/stage_reuse_decision.py @@ -350,8 +350,12 @@ def compute_auto_stage_reuse( except Exception as exc: baseline_error = str(exc) - platform_baseline_run_ids[plat] = baseline.run_id if baseline is not None else None - platform_baseline_urls[plat] = baseline.html_url if baseline is not None else None + platform_baseline_run_ids[plat] = ( + baseline.run_id if baseline is not None else None + ) + platform_baseline_urls[plat] = ( + baseline.html_url if baseline is not None else None + ) available_filenames = _matched_filenames(baseline) avail_here: list[str] = [] @@ -363,7 +367,9 @@ def compute_auto_stage_reuse( avail_here.append(stage_name) per_platform_available[plat] = tuple(avail_here) - selected_run_ids = {run_id for run_id in platform_baseline_run_ids.values() if run_id is not None} + selected_run_ids = { + run_id for run_id in platform_baseline_run_ids.values() if run_id is not None + } if len(selected_run_ids) > 1: return _empty_result( @@ -376,7 +382,9 @@ def compute_auto_stage_reuse( ) reported_baseline_run_id = next(iter(selected_run_ids), None) - reported_baseline_url = next((url for url in platform_baseline_urls.values() if url), None) + reported_baseline_url = next( + (url for url in platform_baseline_urls.values() if url), None + ) # A stage is available only when present on EVERY platform being built. available: list[str] = [] diff --git a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py index ee26f8f63d5..adfe7ec1d82 100644 --- a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py +++ b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py @@ -1485,6 +1485,8 @@ def test_configure_propagates_auto_reuse_baseline_run_id(self, mock_decide_jobs) linux_payload = outputs.builds.linux.to_dict() self.assertEqual(linux_payload["baseline_run_id"], "123") self.assertEqual(linux_payload["prebuilt_stages"], "compiler-runtime") + + # --------------------------------------------------------------------------- # Contract: BuildConfig fields match workflow YAML references # --------------------------------------------------------------------------- From cb85bc78773a9df147ad36e1372f617a005be99a Mon Sep 17 00:00:00 2001 From: ammallya Date: Wed, 8 Jul 2026 16:17:11 +0000 Subject: [PATCH 19/23] Adding documentation --- docs/development/README.md | 1 + docs/development/ci_behavior_manipulation.md | 4 +- docs/development/ci_overview.md | 1 + docs/development/stage_reuse.md | 119 +++++++++++++++++++ 4 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 docs/development/stage_reuse.md diff --git a/docs/development/README.md b/docs/development/README.md index 22bfab7aa4d..49c5b68388c 100644 --- a/docs/development/README.md +++ b/docs/development/README.md @@ -30,6 +30,7 @@ - [S3 Buckets](s3_buckets.md) - [GitHub Actions Debugging](github_actions_debugging.md) - [CI Behavior Manipulation](ci_behavior_manipulation.md) +- [Stage Reuse](stage_reuse.md) - [CCache Troubleshooting](ccache_troubleshooting.md) - [WSL ROCDXG CI Stage](wsl_rocdxg.md) diff --git a/docs/development/ci_behavior_manipulation.md b/docs/development/ci_behavior_manipulation.md index 1d700d84e0e..da7aa6212c0 100644 --- a/docs/development/ci_behavior_manipulation.md +++ b/docs/development/ci_behavior_manipulation.md @@ -54,7 +54,9 @@ stage configuration. > This feature is under active development and will evolve as > automatic stage selection and baseline run lookup are added. > -> See https://github.com/ROCm/TheRock/issues/3399 for details. +> See https://github.com/ROCm/TheRock/issues/3399 for details and +> [`stage_reuse.md`](stage_reuse.md) for the automatic stage-reuse layer +> that builds on the manual inputs described below. The [Multi-Arch CI](https://github.com/ROCm/TheRock/actions/workflows/multi_arch_ci.yml) workflow supports skipping individual build stages by copying their artifacts diff --git a/docs/development/ci_overview.md b/docs/development/ci_overview.md index 5408a1deae3..8d6e1d06b7b 100644 --- a/docs/development/ci_overview.md +++ b/docs/development/ci_overview.md @@ -167,6 +167,7 @@ See [workflow_outputs.md](workflow_outputs.md) for the S3 layout structure and [ - [workflow_outputs.md](workflow_outputs.md) - CI output directory structure - [github_actions_debugging.md](github_actions_debugging.md) - Debugging GitHub Actions - [ci_behavior_manipulation.md](ci_behavior_manipulation.md) - Controlling CI behavior with labels and inputs +- [stage_reuse.md](stage_reuse.md) - Reusing unaffected build stages from a baseline run - [manifest_diff.md](manifest_diff.md) - Manifest diff report (submodule SHA changes between two commits) ## Getting Help diff --git a/docs/development/stage_reuse.md b/docs/development/stage_reuse.md new file mode 100644 index 00000000000..3a6e3e115ea --- /dev/null +++ b/docs/development/stage_reuse.md @@ -0,0 +1,119 @@ +# Stage Reuse + +Stage reuse lets a multi-arch CI run **skip build stages that a change does not +affect** and instead copy those stages' artifacts from a previous, compatible +workflow run. On a change that only touches one component, the unaffected +lower/parallel stages (for example `compiler-runtime`) can be satisfied from a +baseline run instead of rebuilt, cutting build time. + +There are two independent mechanisms: + +- **Manual reuse** (`prebuilt_stages` + `baseline_run_id`) — you explicitly name + the stages to reuse and the run to copy them from. +- **Automatic reuse** (`stage_reuse_mode`) — the setup job computes which stages + are unaffected, verifies their artifacts exist in a healthy baseline run, and + reuses them for you. + +Both feed the same underlying `prebuilt_stages` decision consumed by the build +jobs; automatic reuse simply *adds* to whatever you set manually. + +## How it works + +`setup_multi_arch.yml` runs `build_tools/github_actions/configure_multi_arch_ci.py`, +which calls into `build_tools/github_actions/stage_reuse_decision.py`. For every +stage the change leaves unaffected, two gates must both pass before the stage is +reused: + +1. **Impact gate** (`stage_impact.analyze_stage_impact`) — the changed files do + not affect the stage, so it is a *candidate* for reuse. +1. **Availability gate** (`baseline_runs.select_baseline_run`) — a single + healthy, commit-compatible baseline run actually contains the artifacts that + stage would produce. Availability is verified **independently for every + platform being built** (Linux and/or Windows): a stage is only reused when + its artifacts are present for *all* of those platforms. A stage available in + the Linux baseline but missing from Windows is rebuilt. + +The availability gate reuses the existing baseline-selection logic, so a +candidate run must have healthy `Build` jobs *and* contain all required +artifacts. A run with no artifacts is never +selected. + +### Modes + +`stage_reuse_mode` controls the automatic layer: + +- `dry-run` (default) — compute the analysis and **report** which stages would + be reused (console log + job step summary), but change nothing. Every stage + still builds. Use this to observe decisions. +- `reuse-stage` — actually reuse the eligible stages: they are merged into + `prebuilt_stages`, so the orchestrator copies their artifacts and skips the + build. + +Automatic reuse is **disabled** when no build platform is selected. + +### Inputs + +| Input | Default | Purpose | +| ---------------------------- | --------- | ------------------------------------------------------------------------------------------------------ | +| `stage_reuse_mode` | `dry-run` | `dry-run` (report only) or `reuse-stage` (auto-reuse unaffected stages). | +| `stage_reuse_max_age_hours` | `72` | Reject baseline runs older than this many hours (recency window). | +| `stage_reuse_commit_history` | `50` | Number of recent branch commits to fetch when establishing ancestry for the commit-compatibility rule. | +| `prebuilt_stages` | `""` | Manual, comma-separated stages to reuse (or `all`). Always honored, independent of `stage_reuse_mode`. | +| `baseline_run_id` | `""` | Run ID to copy manually-listed `prebuilt_stages` artifacts from. | + +Example `workflow_call` from a component CI: + +```yaml +jobs: + therock: + uses: ROCm/TheRock/.github/workflows/setup_multi_arch.yml@main + with: + linux_amdgpu_families: "gfx94X" + stage_reuse_mode: "dry-run" + # stage_reuse_mode: "reuse-stage" + # Optional tuning: + # stage_reuse_max_age_hours: "72" + # stage_reuse_commit_history: "50" +``` + +### What "compatible baseline" means + +A baseline run is only used when it is: + +- on the configured baseline branch (default `main`) of the configured workflow + (default `multi_arch_ci.yml`), +- recent enough (`stage_reuse_max_age_hours`), +- commit-compatible — its commit is the same as, or an ancestor of, the current + commit (established from the last `stage_reuse_commit_history` commits), and +- healthy — its `Build` jobs succeeded and it contains every required artifact + for every platform being built. + +If no run satisfies all of these, nothing is reused and every candidate stage is +rebuilt — reuse fails safe toward a full build. + +## Reading the report + +Both modes emit a `Stage reuse analysis` section to the job step summary and +`[STAGE-REUSE]` lines to the console, including: + +- the platforms verified, +- unaffected candidate stages, +- which stages are available in the baseline (per platform), and for + unavailable ones, which platform is missing the artifacts, +- the baseline run used, and +- in `dry-run`, an explicit note that no build steps were skipped. + +## Further features + +- Reuse is scoped to a single build configuration/variant. Threading build + flags through for superrepo builds that vary configuration is tracked in + [#3399](https://github.com/ROCm/TheRock/issues/3399); until then, do not rely + on reuse across builds with differing build flags. + +## Related + +- [ci_overview.md](ci_overview.md) — overall CI pipeline and stages. +- [ci_behavior_manipulation.md](ci_behavior_manipulation.md) — other CI inputs + and labels. +- `build_tools/github_actions/stage_reuse_decision.py` — implementation and the + full `STAGE_REUSE_*` environment contract. From 07f7f7ff09b42e53a4442bbcd831c04f7607c992 Mon Sep 17 00:00:00 2001 From: ammallya Date: Wed, 8 Jul 2026 17:45:13 +0000 Subject: [PATCH 20/23] fixing ordering regression --- .../github_actions/configure_multi_arch_ci.py | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index 8c25fecdaad..30680aea526 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -717,10 +717,20 @@ def decide_jobs( ) build_platforms = _stage_reuse_build_platforms(ci_inputs, targets) - target_families = stage_reuse_target_families( - ci_inputs.linux_amdgpu_families, - ci_inputs.windows_amdgpu_families, - ) + # Derive the families to verify from the resolved target selection when we + # have it, so normal CI still checks the real gfx + # families rather than collapsing to just "generic". Fall back to the raw + # inputs only for the workflow_dispatch path where targets are not threaded. + if targets is not None: + target_families = stage_reuse_target_families( + targets.linux_families, + targets.windows_families, + ) + else: + target_families = stage_reuse_target_families( + ci_inputs.linux_amdgpu_families, + ci_inputs.windows_amdgpu_families, + ) auto = compute_auto_stage_reuse( changed_files=git_context.changed_files, mode=StageReuseMode.from_environ(), @@ -798,7 +808,8 @@ def _stage_reuse_build_platforms( and Windows build configs, so a stage may only be reused when its artifacts are available for every platform actually being built. When target selection is known we use it directly; otherwise we fall back to the requested family - inputs, defaulting to Linux only. + inputs. Returns an empty list when no platform has any families, which + disables automatic reuse. """ platforms: list[str] = [] @@ -1332,14 +1343,18 @@ def configure(ci_inputs: CIInputs, git_context: GitContext) -> CIOutputs: return CIOutputs.skipped() print("Result: CI will run") - print("\n=== Deciding job configuration ===") - jobs = decide_jobs(ci_inputs=ci_inputs, git_context=git_context) - jobs.log() - print("\n=== Selecting GPU target families ===") targets = select_targets(ci_inputs) targets.log() + print("\n=== Deciding job configuration ===") + # Target selection runs first so automatic stage reuse scopes its + # platform/family verification to what is actually being built. On normal + # PR/push/schedule runs the raw workflow inputs are empty; the resolved + # `targets` are what determine the real platforms and families. + jobs = decide_jobs(ci_inputs=ci_inputs, git_context=git_context, targets=targets) + jobs.log() + print("\n=== Building per-platform configs ===") builds = expand_build_configs( ci_inputs=ci_inputs, From b1f3c37721aef485e0ac66b6b523fef590a50957 Mon Sep 17 00:00:00 2001 From: ammallya Date: Thu, 16 Jul 2026 16:39:14 +0000 Subject: [PATCH 21/23] Refactor and clean up based on Scott's review --- .github/workflows/multi_arch_ci.yml | 5 +- .../github_actions/configure_multi_arch_ci.py | 386 +++++++---------- .../github_actions/stage_reuse_decision.py | 410 ++++++++++-------- .../tests/configure_multi_arch_ci_test.py | 94 ++-- .../tests/stage_reuse_decision_test.py | 94 ++-- docs/development/stage_reuse.md | 10 +- 6 files changed, 515 insertions(+), 484 deletions(-) diff --git a/.github/workflows/multi_arch_ci.yml b/.github/workflows/multi_arch_ci.yml index 03afd0f6b8d..0d308e01d0c 100644 --- a/.github/workflows/multi_arch_ci.yml +++ b/.github/workflows/multi_arch_ci.yml @@ -43,8 +43,11 @@ on: default: "" description: "Workflow run ID to copy prebuilt stage artifacts from; required when prebuilt_stages is set" stage_reuse_mode: - type: string + type: choice default: "dry-run" + options: + - "dry-run" + - "reuse-stage" # This drives the AUTOMATIC stage-reuse detection layer, which is # separate from the manual `prebuilt_stages` input above. dry-run only # reports which unaffected stages could be reused; reuse-stage actually diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index 30680aea526..51c2a2baf08 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -8,8 +8,9 @@ 1. Parse Inputs — read GitHub event context → CIInputs, GitContext 2. Check Skip CI — gate: should we skip CI entirely? - 3. Decide Jobs — changed files + topology → per-job-group decisions - 4. Select Targets — trigger type + labels → per-platform GPU families + 3. Select Targets — trigger type + labels → per-platform GPU families + 4. Decide Jobs — changed files + topology + targets → per-job-group + decisions 5. Build Configs — families × variant → per-platform build configs 6. Write Outputs — JSON → GITHUB_OUTPUT + GITHUB_STEP_SUMMARY @@ -74,6 +75,12 @@ gha_load_github_event, gha_set_output, ) +from stage_reuse_decision import ( + AutoStageReuse, + StageReuseMode, + compute_auto_stage_reuse, + render_step_summary, +) _NULL_GIT_SHA = "0" * 40 @@ -434,6 +441,9 @@ class JobDecisions: build_pytorch: JobGroupDecision test_pytorch: JobGroupDecision build_jax: JobGroupDecision + # Automatic stage-reuse analysis, carried so its report can be appended to + # the step summary after the main CI summary. + auto_stage_reuse: AutoStageReuse def log(self) -> None: """Log job decisions for CI diagnostics.""" @@ -580,7 +590,141 @@ def should_skip_ci( # --------------------------------------------------------------------------- -# Step 3: Decide Jobs +# Step 3: Select Targets +# --------------------------------------------------------------------------- + + +def _validate_family_names( + names: list[str], + known: dict[str, dict], +) -> None: + """Raise ValueError if any family name is not in the known matrix.""" + unknown = [name for name in names if name not in known] + if unknown: + raise ValueError( + f"Unknown GPU families: {unknown}. " + f"Known families: {sorted(known.keys())}" + ) + + +def _filter_families_by_platform( + family_names: list[str], + platform: str, + all_families: dict[str, dict], +) -> list[str]: + """Return only the family names that have an entry for the given platform.""" + return [ + name + for name in family_names + if name in all_families and platform in all_families[name] + ] + + +def select_targets(ci_inputs: CIInputs) -> TargetSelection: + """Determine GPU families per platform based on trigger type and inputs. + + Trigger types run progressively larger sets of builds and tests: + + - pull_request: Smallest default set (presubmit families). Designed for + fast feedback on proposed changes. PR labels can opt in to additional + families (gfx* labels) or the full set (ci:run-all-archs). + - push: Broader coverage (presubmit + postsubmit families). Runs on + code that has landed, so we want more thorough validation than PRs + without paying the full nightly cost. + - schedule: Full coverage (all families including nightly-only). Catches + regressions on targets that are too slow or expensive for every push. + - workflow_dispatch: Full manual control. Per-platform family inputs are + taken directly from the workflow inputs, giving the caller the ability + to either replicate what CI does on PRs/push or build/test a narrow + set of targets for investigation. + + Returns per-platform family lists, filtered to only include families + that have a platform entry in amdgpu_family_matrix.py. + """ + all_families = get_all_families_for_trigger_types( + ["presubmit", "postsubmit", "nightly"] + ) + + # Select family names per platform based on trigger type. + # Ordered from most-specific (workflow_dispatch) to broadest (schedule). + if ci_inputs.is_workflow_dispatch: + # Manual trigger: caller specifies exact families per platform. + # "all" = all known families. "none" or empty = skip platform. + linux_names = list(ci_inputs.linux_amdgpu_families) + windows_names = list(ci_inputs.windows_amdgpu_families) + if linux_names == ["all"]: + linux_names = list(all_families.keys()) + print(" linux_amdgpu_families='all' -> all Linux families") + elif linux_names == ["none"]: + linux_names = [] + if windows_names == ["all"]: + windows_names = list(all_families.keys()) + print(" windows_amdgpu_families='all' -> all Windows families") + elif windows_names == ["none"]: + windows_names = [] + elif ci_inputs.is_pull_request: + # Smallest default set for fast PR feedback. PR labels can extend + # the set below (gfx* for individual families, ci:run-all-archs + # for everything). + defaults = list(get_all_families_for_trigger_types(["presubmit"]).keys()) + linux_names = list(defaults) + windows_names = list(defaults) + elif ci_inputs.is_push: + # Broader than PR: presubmit + postsubmit. Code has landed, so + # we validate on more targets (e.g. gfx950) without paying full + # nightly cost. + defaults = list( + get_all_families_for_trigger_types(["presubmit", "postsubmit"]).keys() + ) + linux_names = list(defaults) + windows_names = list(defaults) + elif ci_inputs.is_schedule: + # Full nightly coverage: every known family, including targets + # that are too slow or expensive for per-push CI. + linux_names = list(all_families.keys()) + windows_names = list(all_families.keys()) + else: + raise ValueError(f"Unsupported event type: {ci_inputs.event_name!r}") + + # PR labels can extend the family set (both platforms) + if ci_inputs.is_pull_request: + for label in ci_inputs.pr_labels: + if label == "ci:run-all-archs": + # Override to all families. + linux_names = list(all_families.keys()) + windows_names = list(all_families.keys()) + print(" Label 'ci:run-all-archs' -> all families") + break + if label.startswith("gfx"): + # Trim suffixes from labels since amdgpu_family_matrix.py + # specifies families with no suffix (e.g. `gfx94x`) but + # we have some labels like `gfx94X-dcgpu` or `gfx103X-linux`. + # Note: labels are normalized to lowercase during parsing. + target = label.split("-")[0] + linux_names.append(target) + windows_names.append(target) + print(f" Label '{label}' -> adding target {target}") + + # De-dup, validate, then filter by platform availability. + linux_names = list(dict.fromkeys(linux_names)) + windows_names = list(dict.fromkeys(windows_names)) + _validate_family_names(linux_names, all_families) + _validate_family_names(windows_names, all_families) + # TODO: For workflow_dispatch, a family requested for a specific platform + # but not available there (e.g. gfx94x on windows) is silently dropped. + # Consider validating per-platform and reporting the mismatch. + # We could also filter per-platform in get_all_families_for_trigger_types. + linux_names = _filter_families_by_platform(linux_names, "linux", all_families) + windows_names = _filter_families_by_platform(windows_names, "windows", all_families) + + return TargetSelection( + linux_families=linux_names, + windows_families=windows_names, + ) + + +# --------------------------------------------------------------------------- +# Step 4: Decide Jobs # --------------------------------------------------------------------------- @@ -681,12 +825,13 @@ def _determine_test_type( def decide_jobs( ci_inputs: CIInputs, git_context: GitContext, - targets: "TargetSelection | None" = None, + targets: TargetSelection, ) -> JobDecisions: """Determine which job groups to run, skip, or satisfy with prebuilt files. - ``targets`` (the per-platform family selection) is used to scope automatic - stage reuse to the platforms actually being built, so a stage is only - reused when its artifacts exist for every one of those platforms. + ``targets`` (the per-platform family selection from ``select_targets()``) + scopes automatic stage reuse to the platforms and families actually being + built, so a stage is only reused when its artifacts exist for every one of + those platforms. """ # Build ROCm. @@ -701,54 +846,22 @@ def decide_jobs( # Automatic stage reuse, behind STAGE_REUSE_MODE: in dry-run we only report # which stages WOULD be reused and apply nothing; in "reuse-stage" the # eligible stages are merged into stage_decisions so the orchestrator skips - # their builds and copies artifacts instead. - # These imports are intentionally local. stage_reuse_decision imports from - # sibling CI modules (baseline_runs, stage_impact, github_actions_api) that - # in turn pull in heavier deps; importing it lazily here keeps module import - # of configure_multi_arch_ci cheap and avoids an import cycle between the two - # modules (stage_reuse plumbing conceptually depends on this pipeline). - - from stage_reuse_decision import ( - StageReuseMode, - compute_auto_stage_reuse, - render_step_summary, - log_report, - stage_reuse_target_families, - ) + # their builds and copies artifacts instead. The platforms and families to + # verify come straight from the resolved target selection. - build_platforms = _stage_reuse_build_platforms(ci_inputs, targets) - # Derive the families to verify from the resolved target selection when we - # have it, so normal CI still checks the real gfx - # families rather than collapsing to just "generic". Fall back to the raw - # inputs only for the workflow_dispatch path where targets are not threaded. - if targets is not None: - target_families = stage_reuse_target_families( - targets.linux_families, - targets.windows_families, - ) - else: - target_families = stage_reuse_target_families( - ci_inputs.linux_amdgpu_families, - ci_inputs.windows_amdgpu_families, - ) - auto = compute_auto_stage_reuse( + auto_stage_reuse = compute_auto_stage_reuse( changed_files=git_context.changed_files, mode=StageReuseMode.from_environ(), - platforms=build_platforms, - target_families=target_families, + linux_amdgpu_families=targets.linux_families, + windows_amdgpu_families=targets.windows_families, ) - log_report(auto) - try: - gha_append_step_summary(render_step_summary(auto)) - except Exception as exc: # never fail CI config on a reporting error - print(f"[STAGE-REUSE] could not write step summary: {exc}") # Only reuse-stage mode returns non-empty applied_reuse_stages. - for stage in auto.applied_reuse_stages: + for stage in auto_stage_reuse.applied_reuse_stages: stage_decisions.setdefault(stage, JobAction.PREBUILT) baseline_run_id = ci_inputs.baseline_run_id - if auto.applied_reuse_stages and auto.baseline_run_id: - baseline_run_id = auto.baseline_run_id + if auto_stage_reuse.applied_reuse_stages and auto_stage_reuse.baseline_run_id: + baseline_run_id = auto_stage_reuse.baseline_run_id build_rocm = BuildRocmDecision( action=JobAction.RUN, @@ -791,173 +904,7 @@ def decide_jobs( build_pytorch=JobGroupDecision(action=build_pytorch_action), test_pytorch=JobGroupDecision(action=build_pytorch_action), build_jax=JobGroupDecision(action=build_jax_action), - ) - - -# --------------------------------------------------------------------------- -# Step 4: Select Targets -# --------------------------------------------------------------------------- - - -def _stage_reuse_build_platforms( - ci_inputs: "CIInputs", - targets: "TargetSelection | None", -) -> list[str]: - """Platforms whose baselines must contain a stage's artifacts to reuse it. - Automatic reuse merges into the prebuilt_stages that flow to BOTH the Linux - and Windows build configs, so a stage may only be reused when its artifacts - are available for every platform actually being built. When target selection - is known we use it directly; otherwise we fall back to the requested family - inputs. Returns an empty list when no platform has any families, which - disables automatic reuse. - """ - platforms: list[str] = [] - - linux_families = ( - targets.linux_families - if targets is not None - else ci_inputs.linux_amdgpu_families - ) - windows_families = ( - targets.windows_families - if targets is not None - else ci_inputs.windows_amdgpu_families - ) - - if linux_families: - platforms.append("linux") - if windows_families: - platforms.append("windows") - - return platforms - - -def _validate_family_names( - names: list[str], - known: dict[str, dict], -) -> None: - """Raise ValueError if any family name is not in the known matrix.""" - unknown = [name for name in names if name not in known] - if unknown: - raise ValueError( - f"Unknown GPU families: {unknown}. " - f"Known families: {sorted(known.keys())}" - ) - - -def _filter_families_by_platform( - family_names: list[str], - platform: str, - all_families: dict[str, dict], -) -> list[str]: - """Return only the family names that have an entry for the given platform.""" - return [ - name - for name in family_names - if name in all_families and platform in all_families[name] - ] - - -def select_targets(ci_inputs: CIInputs) -> TargetSelection: - """Determine GPU families per platform based on trigger type and inputs. - - Trigger types run progressively larger sets of builds and tests: - - - pull_request: Smallest default set (presubmit families). Designed for - fast feedback on proposed changes. PR labels can opt in to additional - families (gfx* labels) or the full set (ci:run-all-archs). - - push: Broader coverage (presubmit + postsubmit families). Runs on - code that has landed, so we want more thorough validation than PRs - without paying the full nightly cost. - - schedule: Full coverage (all families including nightly-only). Catches - regressions on targets that are too slow or expensive for every push. - - workflow_dispatch: Full manual control. Per-platform family inputs are - taken directly from the workflow inputs, giving the caller the ability - to either replicate what CI does on PRs/push or build/test a narrow - set of targets for investigation. - - Returns per-platform family lists, filtered to only include families - that have a platform entry in amdgpu_family_matrix.py. - """ - all_families = get_all_families_for_trigger_types( - ["presubmit", "postsubmit", "nightly"] - ) - - # Select family names per platform based on trigger type. - # Ordered from most-specific (workflow_dispatch) to broadest (schedule). - if ci_inputs.is_workflow_dispatch: - # Manual trigger: caller specifies exact families per platform. - # "all" = all known families. "none" or empty = skip platform. - linux_names = list(ci_inputs.linux_amdgpu_families) - windows_names = list(ci_inputs.windows_amdgpu_families) - if linux_names == ["all"]: - linux_names = list(all_families.keys()) - print(" linux_amdgpu_families='all' -> all Linux families") - elif linux_names == ["none"]: - linux_names = [] - if windows_names == ["all"]: - windows_names = list(all_families.keys()) - print(" windows_amdgpu_families='all' -> all Windows families") - elif windows_names == ["none"]: - windows_names = [] - elif ci_inputs.is_pull_request: - # Smallest default set for fast PR feedback. PR labels can extend - # the set below (gfx* for individual families, ci:run-all-archs - # for everything). - defaults = list(get_all_families_for_trigger_types(["presubmit"]).keys()) - linux_names = list(defaults) - windows_names = list(defaults) - elif ci_inputs.is_push: - # Broader than PR: presubmit + postsubmit. Code has landed, so - # we validate on more targets (e.g. gfx950) without paying full - # nightly cost. - defaults = list( - get_all_families_for_trigger_types(["presubmit", "postsubmit"]).keys() - ) - linux_names = list(defaults) - windows_names = list(defaults) - elif ci_inputs.is_schedule: - # Full nightly coverage: every known family, including targets - # that are too slow or expensive for per-push CI. - linux_names = list(all_families.keys()) - windows_names = list(all_families.keys()) - else: - raise ValueError(f"Unsupported event type: {ci_inputs.event_name!r}") - - # PR labels can extend the family set (both platforms) - if ci_inputs.is_pull_request: - for label in ci_inputs.pr_labels: - if label == "ci:run-all-archs": - # Override to all families. - linux_names = list(all_families.keys()) - windows_names = list(all_families.keys()) - print(" Label 'ci:run-all-archs' -> all families") - break - if label.startswith("gfx"): - # Trim suffixes from labels since amdgpu_family_matrix.py - # specifies families with no suffix (e.g. `gfx94x`) but - # we have some labels like `gfx94X-dcgpu` or `gfx103X-linux`. - # Note: labels are normalized to lowercase during parsing. - target = label.split("-")[0] - linux_names.append(target) - windows_names.append(target) - print(f" Label '{label}' -> adding target {target}") - - # De-dup, validate, then filter by platform availability. - linux_names = list(dict.fromkeys(linux_names)) - windows_names = list(dict.fromkeys(windows_names)) - _validate_family_names(linux_names, all_families) - _validate_family_names(windows_names, all_families) - # TODO: For workflow_dispatch, a family requested for a specific platform - # but not available there (e.g. gfx94x on windows) is silently dropped. - # Consider validating per-platform and reporting the mismatch. - # We could also filter per-platform in get_all_families_for_trigger_types. - linux_names = _filter_families_by_platform(linux_names, "linux", all_families) - windows_names = _filter_families_by_platform(windows_names, "windows", all_families) - - return TargetSelection( - linux_families=linux_names, - windows_families=windows_names, + auto_stage_reuse=auto_stage_reuse, ) @@ -1321,6 +1268,11 @@ def write_outputs( ) ) + # Append the automatic stage-reuse analysis after the main summary so the + # two read top-to-bottom in the job step summary. + if outputs.jobs is not None and outputs.jobs.auto_stage_reuse is not None: + gha_append_step_summary(render_step_summary(outputs.jobs.auto_stage_reuse)) + # --------------------------------------------------------------------------- # Pipeline orchestration @@ -1348,10 +1300,8 @@ def configure(ci_inputs: CIInputs, git_context: GitContext) -> CIOutputs: targets.log() print("\n=== Deciding job configuration ===") - # Target selection runs first so automatic stage reuse scopes its - # platform/family verification to what is actually being built. On normal - # PR/push/schedule runs the raw workflow inputs are empty; the resolved - # `targets` are what determine the real platforms and families. + # Target selection runs first so automatic stage reuse can scope its + # prebuilt artifact checks to those targets. jobs = decide_jobs(ci_inputs=ci_inputs, git_context=git_context, targets=targets) jobs.log() diff --git a/build_tools/github_actions/stage_reuse_decision.py b/build_tools/github_actions/stage_reuse_decision.py index ec56ab822fe..6722adcac1e 100644 --- a/build_tools/github_actions/stage_reuse_decision.py +++ b/build_tools/github_actions/stage_reuse_decision.py @@ -50,38 +50,50 @@ ancestry (default ``50``). """ -from __future__ import annotations - import enum import os import logging import functools +import sys +import baseline_runs +import github_actions_api +from pathlib import Path from dataclasses import dataclass, field from typing import Callable, Optional, Sequence +# Add build_tools to the path so sibling CI modules and _therock_utils import +# cleanly regardless of the current working directory. +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from _therock_utils.artifact_backend import ARTIFACT_EXTENSIONS +from _therock_utils.build_topology import BuildTopology, get_topology +from artifact_manager import ARTIFACT_COMPONENTS +from baseline_runs import BaselineRun, RequiredArtifact +from github_actions_api import GitHubAPIError +from stage_impact import analyze_stage_impact + logger = logging.getLogger(__name__) +LOG_PREFIX = "[STAGE-REUSE]" +GENERIC_FAMILY = "generic" + class StageReuseMode(enum.Enum): DRY_RUN = "dry-run" - ENFORCE = "reuse-stage" + REUSE_STAGE = "reuse-stage" @staticmethod - def from_environ(default: "StageReuseMode" = None) -> "StageReuseMode": + def from_environ(default: "StageReuseMode | None" = None) -> "StageReuseMode": """Read STAGE_REUSE_MODE; default to dry-run when unset/invalid.""" default = default or StageReuseMode.DRY_RUN raw = (os.environ.get("STAGE_REUSE_MODE", "") or "").strip().lower() - return _STAGE_REUSE_MODE_ALIASES.get(raw, default) + for mode in StageReuseMode: + if raw == mode.value: + return mode + return default -# Accepted spellings for each mode. ``skip-stage`` is the legacy alias that -# predates the ``reuse-stage`` rename and is kept for backwards compatibility -# with any callers still passing the old value. -_STAGE_REUSE_MODE_ALIASES = { - "dry-run": StageReuseMode.DRY_RUN, - "reuse-stage": StageReuseMode.ENFORCE, - "skip-stage": StageReuseMode.ENFORCE, -} +BaselineSelector = Callable[[Sequence[RequiredArtifact]], BaselineRun | None] @dataclass(frozen=True) @@ -98,9 +110,6 @@ class StageReusePlan: reasons: tuple[str, ...] -BaselineSelector = Callable[[Sequence["object"]], Optional["object"]] - - @dataclass(frozen=True) class AutoStageReuse: """Result of the auto stage-reuse analysis (planning + verification).""" @@ -109,8 +118,8 @@ class AutoStageReuse: candidate_stages: tuple[str, ...] rebuild_stages: tuple[str, ...] full_rebuild_required: bool - baseline_run_id: Optional[str] - baseline_html_url: Optional[str] + baseline_run_id: str | None + baseline_html_url: str | None available_stages: tuple[str, ...] unavailable_stages: tuple[str, ...] applied_reuse_stages: tuple[str, ...] @@ -119,31 +128,30 @@ class AutoStageReuse: platform_available: dict[str, tuple[str, ...]] = field(default_factory=dict) -LOG_PREFIX = "[STAGE-REUSE]" - - -def stage_reuse_target_families( - linux_amdgpu_families: Sequence[str] = (), - windows_amdgpu_families: Sequence[str] = (), -) -> list[str]: +def _target_families( + linux_amdgpu_families: Sequence[str], + windows_amdgpu_families: Sequence[str], +) -> tuple[str, ...]: """Family list whose artifacts must be verified for stage reuse. - Lives here (next to the reuse implementation) rather than in the caller, - since it only exists to prepare inputs for ``compute_auto_stage_reuse``. - Always includes ``generic`` because every stage produces a generic archive. + De-duplicates the requested Linux and Windows families and always appends + the generic pseudo-family, since every stage produces a generic archive. """ families = list(dict.fromkeys([*linux_amdgpu_families, *windows_amdgpu_families])) - if "generic" not in families: - families.append("generic") - return families + if GENERIC_FAMILY not in families: + families.append(GENERIC_FAMILY) + return tuple(families) -def required_artifacts_for_stages(topology, stage_names, target_families): +def _required_artifacts_for_stages( + topology: BuildTopology, + stage_names: Sequence[str], + target_families: Sequence[str], +) -> list[RequiredArtifact]: """Artifact/family pairs the given stages produce.""" - from baseline_runs import RequiredArtifact artifacts_by_group = topology.get_artifact_group_to_artifacts() - required: list = [] - seen: set = set() + required: list[RequiredArtifact] = [] + seen: set[RequiredArtifact] = set() for stage_name in stage_names: stage = topology.build_stages.get(stage_name) if stage is None: @@ -158,18 +166,19 @@ def required_artifacts_for_stages(topology, stage_names, target_families): return required -def _matched_filenames(baseline) -> set: +def _matched_filenames(baseline: BaselineRun | None) -> set[str]: if baseline is None: return set() return set(baseline.artifact_availability.matched_filenames) def _stage_artifacts_available( - topology, stage_name, target_families, available_filenames -): + topology: BuildTopology, + stage_name: str, + target_families: Sequence[str], + available_filenames: set[str], +) -> bool: """True when every artifact this stage produces has an archive present.""" - from artifact_manager import ARTIFACT_COMPONENTS - from _therock_utils.artifact_backend import ARTIFACT_EXTENSIONS artifacts_by_group = topology.get_artifact_group_to_artifacts() stage = topology.build_stages.get(stage_name) @@ -196,7 +205,7 @@ def plan_stage_reuse( *, changed_files: Sequence[str] | None, platform: str | None = None, - topology=None, + topology: BuildTopology | None = None, ) -> StageReusePlan: """Planning step: which stages are unaffected reuse candidates. @@ -213,12 +222,8 @@ def plan_stage_reuse( ) if topology is None: - from _therock_utils.build_topology import get_topology - topology = get_topology() - from stage_impact import analyze_stage_impact - # TODO(#3399): thread build flags/variant through here for superrepos so a # baseline built with different flags is not considered reusable. Not needed # for the current single-variant multi-arch CI, but required before enabling @@ -240,58 +245,59 @@ def compute_auto_stage_reuse( *, changed_files: Sequence[str] | None, mode: StageReuseMode, - platform: str | None = None, - platforms: Sequence[str] | None = None, - target_families: Sequence[str] = (), - topology=None, + linux_amdgpu_families: Sequence[str] = (), + windows_amdgpu_families: Sequence[str] = (), + topology: BuildTopology | None = None, baseline_selector: BaselineSelector | None = None, baseline_selector_factory: Callable[[str], BaselineSelector] | None = None, ) -> AutoStageReuse: """Compute auto stage-reuse decisions, verified against a baseline run. A stage is only reusable when it is unaffected by the change AND its - artifacts are present in a healthy baseline run for *every* platform in - ``platforms``. This guards against the case flagged in review where the - resulting ``prebuilt_stages`` are applied to both Linux and Windows build - configs: a stage available only in the Linux baseline must not be skipped - on Windows. ``platform`` remains accepted as a single-platform shorthand - for backwards compatibility. + artifacts are present in a healthy baseline run for *every* platform being + built. The platforms are derived from which family lists are non-empty: + ``linux_amdgpu_families`` implies the ``linux`` platform, and + ``windows_amdgpu_families`` implies ``windows``. This guards against the + case where a stage available only in the Linux baseline is skipped on + Windows. The report lines are logged before returning. """ - if platforms is not None and len(platforms) == 0 and platform is None: - return _empty_result( - mode, - full_rebuild_required=True, - reasons=("no build platforms selected",), - report_lines=( - f"{LOG_PREFIX} no build platforms selected; automatic stage reuse disabled.", - ), + platforms = _build_platforms(linux_amdgpu_families, windows_amdgpu_families) + if not platforms: + return _log_and_return( + _empty_result( + mode, + full_rebuild_required=True, + reasons=("no build platforms selected",), + report_lines=( + f"{LOG_PREFIX} no build platforms selected; " + f"automatic stage reuse disabled.", + ), + ) ) - # Resolve the platform set. ``platforms`` wins; fall back to the legacy - # single ``platform`` arg; default to linux only when neither is given. - resolved_platforms = _resolve_platforms(platforms, platform) - if topology is None and changed_files is not None: - from _therock_utils.build_topology import get_topology - topology = get_topology() + families = _target_families(linux_amdgpu_families, windows_amdgpu_families) + plan = plan_stage_reuse( changed_files=changed_files, - platform=resolved_platforms[0], + platform=platforms[0], topology=topology, ) candidates = plan.candidate_stages rebuild = plan.rebuild_stages - families = tuple(target_families) or ("generic",) + if changed_files is None: - return _empty_result( - mode, - full_rebuild_required=True, - reasons=plan.reasons, - report_lines=( - f"{LOG_PREFIX} mode={mode.value}; no changed-file list. " - f"Conservatively rebuilding all stages.", - ), + return _log_and_return( + _empty_result( + mode, + full_rebuild_required=True, + reasons=plan.reasons, + report_lines=( + f"{LOG_PREFIX} mode={mode.value}; no changed-file list. " + f"Conservatively rebuilding all stages.", + ), + ) ) if plan.full_rebuild_required or not candidates: @@ -305,80 +311,87 @@ def compute_auto_stage_reuse( available=(), unavailable=candidates, ) - return AutoStageReuse( - mode=mode, - candidate_stages=candidates, - rebuild_stages=rebuild, - full_rebuild_required=plan.full_rebuild_required, - baseline_run_id=None, - baseline_html_url=None, - available_stages=(), - unavailable_stages=candidates, - applied_reuse_stages=(), - reasons=plan.reasons, - report_lines=lines, + return _log_and_return( + AutoStageReuse( + mode=mode, + candidate_stages=candidates, + rebuild_stages=rebuild, + full_rebuild_required=plan.full_rebuild_required, + baseline_run_id=None, + baseline_html_url=None, + available_stages=(), + unavailable_stages=candidates, + applied_reuse_stages=(), + reasons=plan.reasons, + report_lines=lines, + ) ) - required = required_artifacts_for_stages(topology, candidates, families) + required = _required_artifacts_for_stages(topology, candidates, families) # Verify artifact availability independently for each platform. A single # ``baseline_selector`` (used by tests) applies to all platforms; otherwise # a per-platform selector is built so each platform is checked against a # baseline that actually produced that platform's artifacts. - baseline_error: Optional[str] = None - per_platform_available: dict[str, tuple[str, ...]] = {} - # The reported baseline run id/url comes from the first platform's baseline - # (they share commit/recency gates); per-platform detail lives in - # platform_available. - reported_baseline_run_id: Optional[str] = None - reported_baseline_url: Optional[str] = None - - platform_baseline_run_ids: dict[str, Optional[str]] = {} - platform_baseline_urls: dict[str, Optional[str]] = {} + platform_baseline_run_ids: dict[str, str | None] = {} + platform_baseline_urls: dict[str, str | None] = {} per_platform_available: dict[str, tuple[str, ...]] = {} - baseline_error: Optional[str] = None + baseline_error: str | None = None - for plat in resolved_platforms: - baseline = None + for platform in platforms: + if baseline_selector is not None: + selector = baseline_selector + elif baseline_selector_factory is not None: + selector = baseline_selector_factory(platform) + else: + selector = _default_baseline_selector(platform=platform) + + # Only transient GitHub API / network failures are tolerated here: a + # failed baseline lookup falls back to a full rebuild, which is safe. + # Configuration errors (e.g. a bad required-artifacts request) indicate + # a bug and must surface, so they are left to propagate. try: - if baseline_selector is not None: - selector = baseline_selector - elif baseline_selector_factory is not None: - selector = baseline_selector_factory(plat) - else: - selector = _default_baseline_selector(platform=plat) baseline = selector(required) - except Exception as exc: + except GitHubAPIError as exc: baseline_error = str(exc) + baseline = None - platform_baseline_run_ids[plat] = ( + platform_baseline_run_ids[platform] = ( baseline.run_id if baseline is not None else None ) - platform_baseline_urls[plat] = ( + platform_baseline_urls[platform] = ( baseline.html_url if baseline is not None else None ) available_filenames = _matched_filenames(baseline) - avail_here: list[str] = [] + available_here: list[str] = [] if baseline is not None: for stage_name in candidates: if _stage_artifacts_available( topology, stage_name, families, available_filenames ): - avail_here.append(stage_name) - per_platform_available[plat] = tuple(avail_here) + available_here.append(stage_name) + per_platform_available[platform] = tuple(available_here) selected_run_ids = { run_id for run_id in platform_baseline_run_ids.values() if run_id is not None } + # The workflow can only copy artifacts from a single baseline_run_id, so if + # platforms resolve to different baseline runs we cannot safely reuse. + if len(selected_run_ids) > 1: - return _empty_result( - mode, - full_rebuild_required=True, - reasons=("automatic reuse resolved different baseline runs per platform",), - report_lines=( - f"{LOG_PREFIX} multiple baseline runs selected across platforms; disabling automatic reuse.", - ), + return _log_and_return( + _empty_result( + mode, + full_rebuild_required=True, + reasons=( + "automatic reuse resolved different baseline runs per platform", + ), + report_lines=( + f"{LOG_PREFIX} multiple baseline runs selected across " + f"platforms; disabling automatic reuse.", + ), + ) ) reported_baseline_run_id = next(iter(selected_run_ids), None) @@ -392,8 +405,8 @@ def compute_auto_stage_reuse( for stage_name in candidates: if all( - stage_name in per_platform_available.get(plat, ()) - for plat in resolved_platforms + stage_name in per_platform_available.get(platform, ()) + for platform in platforms ): available.append(stage_name) else: @@ -401,7 +414,7 @@ def compute_auto_stage_reuse( available_t = tuple(available) unavailable_t = tuple(unavailable) - applied = available_t if mode is StageReuseMode.ENFORCE else () + applied = available_t if mode is StageReuseMode.REUSE_STAGE else () lines = _format_report( mode=mode, @@ -413,47 +426,53 @@ def compute_auto_stage_reuse( available=available_t, unavailable=unavailable_t, baseline_error=baseline_error, - platforms=resolved_platforms, + platforms=platforms, platform_available=per_platform_available, ) - return AutoStageReuse( - mode=mode, - candidate_stages=candidates, - rebuild_stages=rebuild, - full_rebuild_required=False, - baseline_run_id=reported_baseline_run_id, - baseline_html_url=reported_baseline_url, - available_stages=available_t, - unavailable_stages=unavailable_t, - applied_reuse_stages=applied, - reasons=plan.reasons, - report_lines=lines, - platform_available=per_platform_available, + return _log_and_return( + AutoStageReuse( + mode=mode, + candidate_stages=candidates, + rebuild_stages=rebuild, + full_rebuild_required=False, + baseline_run_id=reported_baseline_run_id, + baseline_html_url=reported_baseline_url, + available_stages=available_t, + unavailable_stages=unavailable_t, + applied_reuse_stages=applied, + reasons=plan.reasons, + report_lines=lines, + platform_available=per_platform_available, + ) ) -def _resolve_platforms( - platforms: Sequence[str] | None, platform: str | None +def _build_platforms( + linux_amdgpu_families: Sequence[str], + windows_amdgpu_families: Sequence[str], ) -> tuple[str, ...]: - """Normalize the platform inputs to a de-duplicated, ordered tuple.""" - if platforms: - resolved = tuple(dict.fromkeys(platforms)) - elif platform: - resolved = (platform,) - else: - resolved = ("linux",) - return resolved + """Platforms whose baselines must contain a stage's artifacts to reuse it. + Automatic reuse merges into the prebuilt_stages that flow to BOTH the Linux + and Windows build configs, so a stage may only be reused when its artifacts + are available for every platform actually being built. + """ + platforms: list[str] = [] + if linux_amdgpu_families: + platforms.append("linux") + if windows_amdgpu_families: + platforms.append("windows") + return tuple(platforms) -def _default_baseline_selector(*, platform: str | None) -> BaselineSelector: + +def _default_baseline_selector(*, platform: str) -> BaselineSelector: """Build a selector bound to baseline_runs.select_baseline_run. - ``select_baseline_run`` already requires each candidate run to have - healthy build jobs (``required_successful_job_name_substrings=("Build",)``) - AND to contain all requested artifacts. A run with no artifacts (e.g. a - docs-only change) therefore fails the availability gate and is never - selected, so no extra "passing build" check is needed here. + ``select_baseline_run`` already requires each candidate run to have healthy + build jobs (``required_successful_job_name_substrings=("Build",)``) AND to + contain all requested artifacts. A run with no artifacts (e.g. a docs-only + change) therefore fails the availability gate and is never selected, so no + extra "passing build" check is needed here. """ - from baseline_runs import select_baseline_run github_repository = os.environ.get("GITHUB_REPOSITORY", "ROCm/TheRock") branch = os.environ.get("STAGE_REUSE_BASELINE_BRANCH", "main") @@ -478,23 +497,20 @@ def _default_baseline_selector(*, platform: str | None) -> BaselineSelector: effective_commit_sha = current_commit_sha if current_commit_sha is not None: try: - from github_actions_api import gha_query_recent_branch_commits - - ordered_commit_shas = gha_query_recent_branch_commits( + ordered_commit_shas = github_actions_api.gha_query_recent_branch_commits( github_repository_name=github_repository, branch=branch, max_count=history_count, ) - if not ordered_commit_shas: - effective_commit_sha = None - ordered_commit_shas = None - except Exception as exc: + except GitHubAPIError as exc: logger.warning( "%s could not fetch branch history (%s); " "skipping commit-compatibility rule.", LOG_PREFIX, exc, ) + ordered_commit_shas = None + if not ordered_commit_shas: effective_commit_sha = None ordered_commit_shas = None @@ -503,23 +519,34 @@ def _default_baseline_selector(*, platform: str | None) -> BaselineSelector: # matches the BaselineSelector signature. return functools.partial( _invoke_select_baseline_run, - select_baseline_run=select_baseline_run, github_repository=github_repository, workflow_name=workflow_name, branch=branch, - platform=platform or "linux", + platform=platform, current_commit_sha=effective_commit_sha, ordered_commit_shas=ordered_commit_shas, max_age_hours=max_age_hours, ) -def _invoke_select_baseline_run(required, *, select_baseline_run, **kwargs): - """Adapter so a partial can present the BaselineSelector(required) shape.""" - return select_baseline_run(required_artifacts=required, **kwargs) +def _invoke_select_baseline_run( + required: Sequence[RequiredArtifact], **kwargs +) -> BaselineRun | None: + """Adapter so a partial can present the BaselineSelector(required) shape. + + Calls through the ``baseline_runs`` module attribute (rather than a bound + reference) so tests can monkeypatch ``select_baseline_run``. + """ + return baseline_runs.select_baseline_run(required_artifacts=required, **kwargs) -def _empty_result(mode, *, full_rebuild_required=False, reasons=(), report_lines=()): +def _empty_result( + mode: StageReuseMode, + *, + full_rebuild_required: bool = False, + reasons: tuple[str, ...] = (), + report_lines: tuple[str, ...] = (), +) -> AutoStageReuse: return AutoStageReuse( mode=mode, candidate_stages=(), @@ -537,18 +564,18 @@ def _empty_result(mode, *, full_rebuild_required=False, reasons=(), report_lines def _format_report( *, - mode, - candidates, - rebuild, - full_rebuild_required, - reasons, - baseline_run_id, - available, - unavailable, - baseline_error=None, + mode: StageReuseMode, + candidates: Sequence[str], + rebuild: Sequence[str], + full_rebuild_required: bool, + reasons: Sequence[str], + baseline_run_id: str | None, + available: Sequence[str], + unavailable: Sequence[str], + baseline_error: str | None = None, platforms: Sequence[str] = (), platform_available: dict[str, tuple[str, ...]] | None = None, -): +) -> tuple[str, ...]: platform_available = platform_available or {} lines: list[str] = [f"{LOG_PREFIX} mode={mode.value}"] if platforms: @@ -575,7 +602,9 @@ def _format_report( f"{LOG_PREFIX} no baseline run contains artifacts for all " f"candidate stages; rebuilding all candidates." ) - verb = "WILL be skipped" if mode is StageReuseMode.ENFORCE else "WOULD be skipped" + verb = ( + "WILL be skipped" if mode is StageReuseMode.REUSE_STAGE else "WOULD be skipped" + ) for stage in available: lines.append( f"{LOG_PREFIX} stage '{stage}' unaffected AND available in " @@ -586,9 +615,9 @@ def _format_report( # (e.g. present on linux, absent on windows) is visible in the log. if len(platforms) > 1: missing = [ - plat - for plat in platforms - if stage not in platform_available.get(plat, ()) + platform + for platform in platforms + if stage not in platform_available.get(platform, ()) ] where = f" (missing on: {', '.join(missing)})" if missing else "" else: @@ -607,7 +636,7 @@ def _format_report( return tuple(lines) -def _format_stage_list(stages: tuple[str, ...]) -> str: +def _format_stage_list(stages: Sequence[str]) -> str: """Render a tuple of stage names as backticked, comma-separated markdown.""" if not stages: return "_none_" @@ -630,8 +659,8 @@ def render_step_summary(result: AutoStageReuse) -> str: out.append(f"- applied: {applied}") if result.platform_available: out.append("- available per platform:") - for plat, stages in result.platform_available.items(): - out.append(f" - {plat}: {_format_stage_list(stages)}") + for platform, stages in result.platform_available.items(): + out.append(f" - {platform}: {_format_stage_list(stages)}") if result.reasons: out.append("- reasons:") for reason in result.reasons: @@ -646,7 +675,8 @@ def render_step_summary(result: AutoStageReuse) -> str: return "\n".join(out) -def log_report(result: AutoStageReuse) -> None: - """Emit the analysis report lines to the module logger.""" +def _log_and_return(result: AutoStageReuse) -> AutoStageReuse: + """Emit the analysis report lines to the module logger and return result.""" for line in result.report_lines: logger.info(line) + return result diff --git a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py index adfe7ec1d82..8edaacbfcc7 100644 --- a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py +++ b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py @@ -343,7 +343,9 @@ def _inputs(self, **kwargs): def test_all_job_groups_run(self): """All job groups are set to run (subgraph selection is Phase 4).""" - result = cm.decide_jobs(self._inputs(), git_context=cm.GitContext()) + result = cm.decide_jobs( + self._inputs(), git_context=cm.GitContext(), targets=cm.TargetSelection() + ) self.assertIsInstance(result, cm.JobDecisions) self.assertEqual(result.build_rocm.action, cm.JobAction.RUN) self.assertEqual(result.test_rocm.action, cm.JobAction.RUN) @@ -356,6 +358,7 @@ def test_build_pytorch_input_skips_pytorch_jobs(self): result = cm.decide_jobs( self._inputs(build_pytorch=False), git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual(result.build_pytorch.action, cm.JobAction.SKIP) @@ -365,6 +368,7 @@ def test_build_jax_input_runs_jax_job(self): result = cm.decide_jobs( self._inputs(build_jax=True), git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual(result.build_jax.action, cm.JobAction.RUN) @@ -372,13 +376,17 @@ def test_build_jax_input_runs_jax_job(self): def test_default_test_type_is_quick(self): """Default test_type for PR/push with no special conditions.""" git = cm.GitContext(changed_files=["CMakeLists.txt"]) - result = cm.decide_jobs(self._inputs(), git_context=git) + result = cm.decide_jobs( + self._inputs(), git_context=git, targets=cm.TargetSelection() + ) self.assertEqual(result.test_rocm.test_type, "quick") def test_schedule_is_comprehensive(self): """Schedule trigger → comprehensive tests.""" result = cm.decide_jobs( - self._inputs(event_name="schedule"), git_context=cm.GitContext() + self._inputs(event_name="schedule"), + git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual(result.test_rocm.test_type, "comprehensive") @@ -388,7 +396,9 @@ def test_submodule_change_is_full(self): changed_files=["rocm-libraries", "CMakeLists.txt"], submodule_paths=["rocm-libraries", "rocm-systems"], ) - result = cm.decide_jobs(self._inputs(), git_context=git) + result = cm.decide_jobs( + self._inputs(), git_context=git, targets=cm.TargetSelection() + ) self.assertEqual(result.test_rocm.test_type, "standard") self.assertIn("submodule", result.test_rocm.test_type_reason) @@ -398,14 +408,18 @@ def test_no_submodule_change_stays_quick(self): changed_files=["CMakeLists.txt"], submodule_paths=["rocm-libraries", "rocm-systems"], ) - result = cm.decide_jobs(self._inputs(), git_context=git) + result = cm.decide_jobs( + self._inputs(), git_context=git, targets=cm.TargetSelection() + ) self.assertEqual(result.test_rocm.test_type, "quick") def test_pr_test_label_is_full(self): """PR with test:* label → full tests.""" git = cm.GitContext(changed_files=["CMakeLists.txt"]) result = cm.decide_jobs( - self._inputs(pr_labels=["test:rocprim"]), git_context=git + self._inputs(pr_labels=["test:rocprim"]), + git_context=git, + targets=cm.TargetSelection(), ) self.assertEqual(result.test_rocm.test_type, "full") @@ -417,13 +431,16 @@ def test_workflow_dispatch_test_labels_is_full(self): linux_test_labels=["test:rocprim"], ), git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual(result.test_rocm.test_type, "full") def test_nightly_release_is_comprehensive(self): """Nightly release → comprehensive tests.""" result = cm.decide_jobs( - self._inputs(release_type="nightly"), git_context=cm.GitContext() + self._inputs(release_type="nightly"), + git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual(result.test_rocm.test_type, "comprehensive") self.assertIn("release", result.test_rocm.test_type_reason) @@ -431,7 +448,9 @@ def test_nightly_release_is_comprehensive(self): def test_prerelease_is_full(self): """Prerelease → full tests.""" result = cm.decide_jobs( - self._inputs(release_type="prerelease"), git_context=cm.GitContext() + self._inputs(release_type="prerelease"), + git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual(result.test_rocm.test_type, "full") self.assertIn("release", result.test_rocm.test_type_reason) @@ -439,7 +458,11 @@ def test_prerelease_is_full(self): def test_dev_release_falls_through_to_default(self): """Dev release without other signals → quick (falls through).""" git = cm.GitContext(changed_files=["CMakeLists.txt"]) - result = cm.decide_jobs(self._inputs(release_type="dev"), git_context=git) + result = cm.decide_jobs( + self._inputs(release_type="dev"), + git_context=git, + targets=cm.TargetSelection(), + ) self.assertEqual(result.test_rocm.test_type, "quick") def test_test_filter_label_overrides(self): @@ -459,7 +482,9 @@ def test_test_filter_invalid_raises(self): git = cm.GitContext(changed_files=["CMakeLists.txt"]) with self.assertRaises(ValueError, msg="Unrecognized test_filter"): cm.decide_jobs( - self._inputs(pr_labels=["test_filter:bogus"]), git_context=git + self._inputs(pr_labels=["test_filter:bogus"]), + git_context=git, + targets=cm.TargetSelection(), ) def test_workflow_dispatch_test_filter_label_overrides(self): @@ -491,38 +516,28 @@ def test_explicit_prebuilt_stages(self): ) self.assertEqual(result.build_rocm.rebuild_stages, []) - def test_stage_reuse_build_platforms_prefers_targets(self): - """Resolved target selection takes precedence over workflow inputs.""" - inputs = self._inputs( - linux_amdgpu_families=["gfx94x"], - windows_amdgpu_families=["gfx110x"], - ) - - targets = cm.TargetSelection( - linux_families=["gfx94x"], - windows_families=[], - ) - - self.assertEqual( - cm._stage_reuse_build_platforms(inputs, targets), - ["linux"], - ) - - def test_stage_reuse_build_platforms_falls_back_to_inputs(self): - """Workflow inputs are used before target selection is available.""" - inputs = self._inputs( - linux_amdgpu_families=["gfx94x"], - windows_amdgpu_families=["gfx110x"], - ) - - self.assertEqual( - cm._stage_reuse_build_platforms(inputs, None), - ["linux", "windows"], + def test_reuse_scoped_to_selected_targets(self): + """decide_jobs threads the resolved targets into automatic reuse. + With no families selected there are no build platforms, so automatic + reuse is disabled and reports a full rebuild -- confirming the decision + is scoped to the passed-in target selection. + """ + result = cm.decide_jobs( + self._inputs(), + git_context=cm.GitContext( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"] + ), + targets=cm.TargetSelection(), ) + self.assertIsNotNone(result.auto_stage_reuse) + self.assertTrue(result.auto_stage_reuse.full_rebuild_required) + self.assertEqual(result.auto_stage_reuse.applied_reuse_stages, ()) def test_no_prebuilt_stages_by_default(self): """Without explicit prebuilt_stages, no stage decisions are set.""" - result = cm.decide_jobs(self._inputs(), git_context=cm.GitContext()) + result = cm.decide_jobs( + self._inputs(), git_context=cm.GitContext(), targets=cm.TargetSelection() + ) self.assertEqual(result.build_rocm.prebuilt_stages, []) self.assertEqual(result.build_rocm.stage_decisions, {}) @@ -552,6 +567,7 @@ def test_asan_tests_only_run_on_nightly_triggers(self): result = cm.decide_jobs( self._inputs(event_name=event, build_variant="asan"), git_context=git_context, + targets=cm.TargetSelection(), ) self.assertEqual( result.test_rocm.action, @@ -564,6 +580,7 @@ def test_asan_tests_only_run_on_nightly_triggers(self): result = cm.decide_jobs( self._inputs(event_name=event, build_variant="asan"), git_context=git_context, + targets=cm.TargetSelection(), ) self.assertEqual( result.test_rocm.action, @@ -575,6 +592,7 @@ def test_asan_tests_only_run_on_nightly_triggers(self): result = cm.decide_jobs( self._inputs(event_name="pull_request", build_variant="release"), git_context=git_context, + targets=cm.TargetSelection(), ) self.assertEqual(result.test_rocm.action, cm.JobAction.RUN) diff --git a/build_tools/github_actions/tests/stage_reuse_decision_test.py b/build_tools/github_actions/tests/stage_reuse_decision_test.py index 7cd6a8d4d4d..92a07953678 100644 --- a/build_tools/github_actions/tests/stage_reuse_decision_test.py +++ b/build_tools/github_actions/tests/stage_reuse_decision_test.py @@ -17,6 +17,7 @@ ArtifactAvailability, WorkflowJobHealth, ) +from github_actions_api import GitHubAPIError class _FakeStage: @@ -110,8 +111,7 @@ def test_explicit_modes(self): for value, expected in [ ("dry-run", StageReuseMode.DRY_RUN), - ("reuse-stage", StageReuseMode.ENFORCE), - ("reuse-stage", StageReuseMode.ENFORCE), + ("reuse-stage", StageReuseMode.REUSE_STAGE), ("garbage", StageReuseMode.DRY_RUN), ]: os.environ["STAGE_REUSE_MODE"] = value @@ -126,7 +126,7 @@ def test_dry_run_verifies_artifacts_present(self): result = compute_auto_stage_reuse( changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], mode=StageReuseMode.DRY_RUN, - target_families=["generic"], + linux_amdgpu_families=["generic"], topology=FakeTopology(), baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), ) @@ -143,7 +143,7 @@ def test_dry_run_unaffected_but_artifacts_missing_rebuilds(self): result = compute_auto_stage_reuse( changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], mode=StageReuseMode.DRY_RUN, - target_families=["generic"], + linux_amdgpu_families=["generic"], topology=FakeTopology(), baseline_selector=_selector(_baseline("123", ["blas_lib_generic.tar.zst"])), ) @@ -157,7 +157,7 @@ def test_no_baseline_found_rebuilds_candidates(self): result = compute_auto_stage_reuse( changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], mode=StageReuseMode.DRY_RUN, - target_families=["generic"], + linux_amdgpu_families=["generic"], topology=FakeTopology(), baseline_selector=_selector(None), ) @@ -168,22 +168,23 @@ def test_no_baseline_found_rebuilds_candidates(self): self.assertIn("no baseline run contains artifacts", joined) def test_partial_family_availability_rebuilds(self): - # Needs base for two families; baseline only has the generic archive. + # Needs base for a real family + generic; baseline only has the generic + # archive, so the real family's artifact is missing -> rebuild. result = compute_auto_stage_reuse( changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], mode=StageReuseMode.DRY_RUN, - target_families=["generic", "gfx94X-dcgpu"], + linux_amdgpu_families=["gfx94X-dcgpu"], topology=FakeTopology(), baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), ) self.assertIn("compiler-runtime", result.unavailable_stages) self.assertEqual(result.available_stages, ()) - def test_enforce_applies_only_available_stages(self): + def test_reuse_stage_applies_only_available_stages(self): result = compute_auto_stage_reuse( changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], - mode=StageReuseMode.ENFORCE, - target_families=["generic"], + mode=StageReuseMode.REUSE_STAGE, + linux_amdgpu_families=["generic"], topology=FakeTopology(), baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), ) @@ -193,12 +194,12 @@ def test_enforce_applies_only_available_stages(self): def test_baseline_lookup_error_is_safe(self): def boom(required): - raise RuntimeError("network down") + raise GitHubAPIError("network down") result = compute_auto_stage_reuse( changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], - mode=StageReuseMode.ENFORCE, - target_families=["generic"], + mode=StageReuseMode.REUSE_STAGE, + linux_amdgpu_families=["generic"], topology=FakeTopology(), baseline_selector=boom, ) @@ -206,6 +207,19 @@ def boom(required): joined = "\n".join(result.report_lines) self.assertIn("baseline lookup failed", joined) + def test_non_api_selector_error_propagates(self): + def boom(required): + raise ValueError("bad required-artifacts request") + + with self.assertRaises(ValueError): + compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.REUSE_STAGE, + linux_amdgpu_families=["generic"], + topology=FakeTopology(), + baseline_selector=boom, + ) + class GuardrailTest(unittest.TestCase): def test_full_ci_trigger_skips_baseline_query(self): @@ -217,7 +231,7 @@ def selector(required): result = compute_auto_stage_reuse( changed_files=["build_tools/foo.py"], - mode=StageReuseMode.ENFORCE, + mode=StageReuseMode.REUSE_STAGE, topology=FakeTopology(), baseline_selector=selector, ) @@ -228,7 +242,7 @@ def selector(required): def test_no_diff_is_conservative(self): result = compute_auto_stage_reuse( changed_files=None, - mode=StageReuseMode.ENFORCE, + mode=StageReuseMode.REUSE_STAGE, topology=FakeTopology(), baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), ) @@ -239,7 +253,7 @@ def test_step_summary_shows_baseline_and_availability(self): result = compute_auto_stage_reuse( changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], mode=StageReuseMode.DRY_RUN, - target_families=["generic"], + linux_amdgpu_families=["generic"], topology=FakeTopology(), baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), ) @@ -255,7 +269,6 @@ class DefaultBaselineSelectorTest(unittest.TestCase): def _run_with_env(self, env, fake_history, fake_select): import os - import importlib old_env = {k: os.environ.get(k) for k in env} os.environ.update({k: v for k, v in env.items() if v is not None}) @@ -276,7 +289,6 @@ def _capturing_select(**kwargs): baseline_runs.select_baseline_run = _capturing_select github_actions_api.gha_query_recent_branch_commits = fake_history - importlib.reload(srd) if False else None try: selector = srd._default_baseline_selector(platform="linux") result = selector([("base", "generic")]) @@ -321,7 +333,7 @@ def fake_history(**kwargs): def test_history_fetch_error_disables_commit_rule(self): def fake_history(**kwargs): - raise RuntimeError("api down") + raise GitHubAPIError("api down") captured, _ = self._run_with_env( {"STAGE_REUSE_CURRENT_SHA": "sha-current"}, @@ -373,9 +385,9 @@ def test_stage_reused_only_when_available_on_all_platforms(self): } result = compute_auto_stage_reuse( changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], - mode=StageReuseMode.ENFORCE, - platforms=["linux", "windows"], - target_families=["generic"], + mode=StageReuseMode.REUSE_STAGE, + windows_amdgpu_families=["generic"], + linux_amdgpu_families=["generic"], topology=FakeTopology(), baseline_selector_factory=self._selector_factory(per_platform), ) @@ -395,9 +407,9 @@ def test_stage_reused_when_present_on_both_platforms(self): } result = compute_auto_stage_reuse( changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], - mode=StageReuseMode.ENFORCE, - platforms=["linux", "windows"], - target_families=["generic"], + mode=StageReuseMode.REUSE_STAGE, + windows_amdgpu_families=["generic"], + linux_amdgpu_families=["generic"], topology=FakeTopology(), baseline_selector_factory=self._selector_factory(per_platform), ) @@ -407,8 +419,8 @@ def test_stage_reused_when_present_on_both_platforms(self): def test_single_platform_default_is_linux(self): result = compute_auto_stage_reuse( changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], - mode=StageReuseMode.ENFORCE, - target_families=["generic"], + mode=StageReuseMode.REUSE_STAGE, + linux_amdgpu_families=["generic"], topology=FakeTopology(), baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), ) @@ -418,7 +430,7 @@ def test_single_platform_default_is_linux(self): def test_no_platforms_selected_disables_auto_reuse(self): result = compute_auto_stage_reuse( changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], - mode=StageReuseMode.ENFORCE, + mode=StageReuseMode.REUSE_STAGE, platforms=[], topology=FakeTopology(), baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), @@ -436,9 +448,9 @@ def test_different_platform_baselines_disable_reuse(self): result = compute_auto_stage_reuse( changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], - mode=StageReuseMode.ENFORCE, - platforms=["linux", "windows"], - target_families=["generic"], + mode=StageReuseMode.REUSE_STAGE, + windows_amdgpu_families=["generic"], + linux_amdgpu_families=["generic"], topology=FakeTopology(), baseline_selector_factory=self._selector_factory(per_platform), ) @@ -470,11 +482,27 @@ def test_plan_none_changed_files_is_full_rebuild(self): class TargetFamiliesTest(unittest.TestCase): def test_always_includes_generic(self): - self.assertEqual(srd.stage_reuse_target_families(), ["generic"]) + self.assertEqual(srd._target_families((), ()), ("generic",)) def test_dedupes_and_appends_generic(self): fams = srd.stage_reuse_target_families(["gfx94x"], ["gfx94x", "gfx120x"]) - self.assertEqual(fams, ["gfx94x", "gfx120x", "generic"]) + self.assertEqual(fams, ("gfx94x", "gfx120x", "generic")) + + +class BuildPlatformsTest(unittest.TestCase): + def test_no_families_is_empty(self): + self.assertEqual(srd._build_platforms((), ()), ()) + + def test_linux_only(self): + self.assertEqual(srd._build_platforms(["gfx94x"], ()), ("linux",)) + + def test_windows_only(self): + self.assertEqual(srd._build_platforms((), ["gfx110x"]), ("windows",)) + + def test_both_platforms(self): + self.assertEqual( + srd._build_platforms(["gfx94x"], ["gfx110x"]), ("linux", "windows") + ) if __name__ == "__main__": diff --git a/docs/development/stage_reuse.md b/docs/development/stage_reuse.md index 3a6e3e115ea..ac0855224e3 100644 --- a/docs/development/stage_reuse.md +++ b/docs/development/stage_reuse.md @@ -19,10 +19,12 @@ jobs; automatic reuse simply *adds* to whatever you set manually. ## How it works -`setup_multi_arch.yml` runs `build_tools/github_actions/configure_multi_arch_ci.py`, -which calls into `build_tools/github_actions/stage_reuse_decision.py`. For every -stage the change leaves unaffected, two gates must both pass before the stage is -reused: +[`setup_multi_arch.yml`](https://github.com/ROCm/TheRock/.github/workflows/setup_multi_arch.yml) runs +[`configure_multi_arch_ci.py`](https://github.com/ROCm/TheRock/build_tools/github_actions/configure_multi_arch_ci.py), +which calls into +[`stage_reuse_decision.py`](https://github.com/ROCm/TheRock/build_tools/github_actions/stage_reuse_decision.py). +For every stage the change leaves unaffected, two gates must both pass before the +stage is reused: 1. **Impact gate** (`stage_impact.analyze_stage_impact`) — the changed files do not affect the stage, so it is a *candidate* for reuse. From 5dc07b491e6f282bafec9dc2609bc682e48b11bd Mon Sep 17 00:00:00 2001 From: ammallya Date: Thu, 16 Jul 2026 21:09:24 +0000 Subject: [PATCH 22/23] Fix after new JAX functionality merge --- .../github_actions/configure_multi_arch_ci.py | 25 +++++++++++++++---- .../tests/configure_multi_arch_ci_test.py | 3 +++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index e0018e4cba8..3598ed66255 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -455,7 +455,7 @@ class JobDecisions: build_jax: JobGroupDecision # Automatic stage-reuse analysis, carried so its report can be appended to # the step summary after the main CI summary. - auto_stage_reuse: AutoStageReuse + auto_stage_reuse: AutoStageReuse | None = None def log(self) -> None: """Log job decisions for CI diagnostics.""" @@ -688,10 +688,25 @@ def select_targets(ci_inputs: CIInputs) -> TargetSelection: linux_names = list(defaults) windows_names = list(defaults) elif ci_inputs.is_schedule: - # Full nightly coverage: every known family, including targets - # that are too slow or expensive for per-push CI. - linux_names = list(all_families.keys()) - windows_names = list(all_families.keys()) + # Schedule trigger: use explicit inputs if provided, else all families. + # "all" or empty = all known families. "none" = skip platform. + linux_names = list(ci_inputs.linux_amdgpu_families) + windows_names = list(ci_inputs.windows_amdgpu_families) + if linux_names == ["all"]: + linux_names = list(all_families.keys()) + print(" linux_amdgpu_families='all' -> all Linux families") + elif not linux_names: + linux_names = list(all_families.keys()) + elif linux_names == ["none"]: + linux_names = [] + + if windows_names == ["all"]: + windows_names = list(all_families.keys()) + print(" windows_amdgpu_families='all' -> all Windows families") + elif not windows_names: + windows_names = list(all_families.keys()) + elif windows_names == ["none"]: + windows_names = [] else: raise ValueError(f"Unsupported event type: {ci_inputs.event_name!r}") diff --git a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py index ea5d64d80b6..96cb11f5e86 100644 --- a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py +++ b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py @@ -474,6 +474,7 @@ def test_test_filter_label_overrides(self): pr_labels=["test_filter:standard"], ), git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual(result.test_rocm.test_type, "standard") @@ -495,6 +496,7 @@ def test_workflow_dispatch_test_filter_label_overrides(self): self._inputs( event_name="workflow_dispatch", linux_test_labels=["test_filter:comprehensive"], + targets=cm.TargetSelection(), ), git_context=cm.GitContext(), ) @@ -509,6 +511,7 @@ def test_explicit_prebuilt_stages(self): prebuilt_stages="compiler-runtime,runtime-tests", ), git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual( sorted(result.build_rocm.prebuilt_stages), From 1c19f0b2cc0a04edf5dbfd4eaa00123edc463f65 Mon Sep 17 00:00:00 2001 From: ammallya Date: Fri, 17 Jul 2026 00:32:32 +0000 Subject: [PATCH 23/23] Fix for tests --- .../github_actions/tests/configure_multi_arch_ci_test.py | 2 +- build_tools/github_actions/tests/stage_reuse_decision_test.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py index 96cb11f5e86..624c3f4c611 100644 --- a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py +++ b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py @@ -496,9 +496,9 @@ def test_workflow_dispatch_test_filter_label_overrides(self): self._inputs( event_name="workflow_dispatch", linux_test_labels=["test_filter:comprehensive"], - targets=cm.TargetSelection(), ), git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual(result.test_rocm.test_type, "comprehensive") self.assertIn("test_filter", result.test_rocm.test_type_reason) diff --git a/build_tools/github_actions/tests/stage_reuse_decision_test.py b/build_tools/github_actions/tests/stage_reuse_decision_test.py index 92a07953678..3e6fedc88ae 100644 --- a/build_tools/github_actions/tests/stage_reuse_decision_test.py +++ b/build_tools/github_actions/tests/stage_reuse_decision_test.py @@ -431,7 +431,6 @@ def test_no_platforms_selected_disables_auto_reuse(self): result = compute_auto_stage_reuse( changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], mode=StageReuseMode.REUSE_STAGE, - platforms=[], topology=FakeTopology(), baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), ) @@ -485,7 +484,7 @@ def test_always_includes_generic(self): self.assertEqual(srd._target_families((), ()), ("generic",)) def test_dedupes_and_appends_generic(self): - fams = srd.stage_reuse_target_families(["gfx94x"], ["gfx94x", "gfx120x"]) + fams = srd._target_families(["gfx94x"], ["gfx94x", "gfx120x"]) self.assertEqual(fams, ("gfx94x", "gfx120x", "generic"))