diff --git a/.github/workflows/multi_arch_ci.yml b/.github/workflows/multi_arch_ci.yml index a26f8663ce8..3cbd35eb25e 100644 --- a/.github/workflows/multi_arch_ci.yml +++ b/.github/workflows/multi_arch_ci.yml @@ -42,6 +42,25 @@ on: type: string default: "" description: "Workflow run ID to copy prebuilt stage artifacts from; required when prebuilt_stages is set" + stage_reuse_mode: + type: choice + default: "dry-run" + options: + - "dry-run" + - "reuse-stage" + # This drives the AUTOMATIC stage-reuse detection layer, which is + # separate from the manual `prebuilt_stages` input above. dry-run only + # reports which unaffected stages could be reused; reuse-stage actually + # reuses them (skips the build and copies artifacts from a baseline run). + description: "dry-run (report only) | reuse-stage (auto-reuse unaffected stages)" + stage_reuse_max_age_hours: + type: string + default: "72" + description: "Automatic stage reuse: reject baseline runs older than this many hours" + stage_reuse_commit_history: + type: string + default: "50" + description: "Automatic stage reuse: number of recent branch commits to fetch for ancestry" changed_projects: type: string default: "" @@ -86,6 +105,9 @@ jobs: windows_test_labels: ${{ inputs.windows_test_labels || '' }} prebuilt_stages: ${{ inputs.prebuilt_stages || '' }} baseline_run_id: ${{ inputs.baseline_run_id || '' }} + stage_reuse_mode: ${{ inputs.stage_reuse_mode || 'dry-run' }} + stage_reuse_max_age_hours: ${{ inputs.stage_reuse_max_age_hours || '72' }} + stage_reuse_commit_history: ${{ inputs.stage_reuse_commit_history || '50' }} build_pytorch: ${{ github.event_name != 'workflow_dispatch' || inputs.build_pytorch }} build_jax: ${{ github.event_name != 'workflow_dispatch' || inputs.build_jax }} diff --git a/.github/workflows/setup_multi_arch.yml b/.github/workflows/setup_multi_arch.yml index 8d30bfc1583..6c589aedf3b 100644 --- a/.github/workflows/setup_multi_arch.yml +++ b/.github/workflows/setup_multi_arch.yml @@ -43,6 +43,23 @@ on: description: "Workflow run ID to copy prebuilt stage artifacts from." type: string default: "" + stage_reuse_mode: + description: >- + Automatic stage-reuse mode (separate from the manual prebuilt_stages + input): "dry-run" (default) only reports which unaffected stages could + be reused; "reuse-stage" actually reuses them. + type: string + default: "dry-run" + stage_reuse_max_age_hours: + description: >- + Automatic stage reuse: reject baseline runs older than this many hours. default: "72" + type: string + default: "72" + stage_reuse_commit_history: + description: >- + Automatic stage reuse: number of recent branch commits to fetch for ancestry. default: "50" + type: string + default: "50" build_pytorch: description: "Build PyTorch wheels." type: boolean @@ -159,6 +176,10 @@ jobs: WINDOWS_TEST_LABELS: ${{ inputs.windows_test_labels }} PREBUILT_STAGES: ${{ inputs.prebuilt_stages }} BASELINE_RUN_ID: ${{ inputs.baseline_run_id }} + STAGE_REUSE_MODE: ${{ inputs.stage_reuse_mode }} + STAGE_REUSE_CURRENT_SHA: ${{ steps.checkout.outputs.commit }} + STAGE_REUSE_MAX_AGE_HOURS: ${{ inputs.stage_reuse_max_age_hours }} + STAGE_REUSE_COMMIT_HISTORY: ${{ inputs.stage_reuse_commit_history }} # Skip path filtering for external repos (git sha won't exist in TheRock) SKIP_PATH_FILTERS: ${{ inputs.external_repo != '' && 'true' || '' }} CI_CONFIG_PATH: ci-config diff --git a/build_tools/github_actions/configure_multi_arch_ci.py b/build_tools/github_actions/configure_multi_arch_ci.py index 52f55153e88..3598ed66255 100755 --- a/build_tools/github_actions/configure_multi_arch_ci.py +++ b/build_tools/github_actions/configure_multi_arch_ci.py @@ -8,8 +8,9 @@ 1. Parse Inputs — read GitHub event context → CIInputs, GitContext 2. Check Skip CI — gate: should we skip CI entirely? - 3. Decide Jobs — changed files + topology → per-job-group decisions - 4. Select Targets — trigger type + labels → per-platform GPU families + 3. Select Targets — trigger type + labels → per-platform GPU families + 4. Decide Jobs — changed files + topology + targets → per-job-group + decisions 5. Build Configs — families × variant → per-platform build configs 6. Write Outputs — JSON → GITHUB_OUTPUT + GITHUB_STEP_SUMMARY @@ -74,6 +75,12 @@ gha_load_github_event, gha_set_output, ) +from stage_reuse_decision import ( + AutoStageReuse, + StageReuseMode, + compute_auto_stage_reuse, + render_step_summary, +) _NULL_GIT_SHA = "0" * 40 @@ -446,6 +453,9 @@ class JobDecisions: build_pytorch: JobGroupDecision test_pytorch: JobGroupDecision build_jax: JobGroupDecision + # Automatic stage-reuse analysis, carried so its report can be appended to + # the step summary after the main CI summary. + auto_stage_reuse: AutoStageReuse | None = None def log(self) -> None: """Log job decisions for CI diagnostics.""" @@ -589,161 +599,7 @@ def should_skip_ci( # --------------------------------------------------------------------------- -# Step 3: Decide Jobs -# --------------------------------------------------------------------------- - - -_VALID_TEST_FILTER_TYPES = {"quick", "standard", "comprehensive", "full"} - - -def _has_test_labels(ci_inputs: CIInputs) -> bool: - """Check whether any test labels were specified (workflow_dispatch or PR). - - Note: test_filter: labels are not test labels - they control test_type, - not which tests to run. - """ - # Filter out test_filter: labels - those control test_type, not test selection - linux_tests = [ - l for l in ci_inputs.linux_test_labels if not l.startswith("test_filter:") - ] - windows_tests = [ - l for l in ci_inputs.windows_test_labels if not l.startswith("test_filter:") - ] - if linux_tests or windows_tests: - return True - return any(label.startswith("test:") for label in ci_inputs.pr_labels) - - -def _determine_test_type( - ci_inputs: CIInputs, - git_context: GitContext, -) -> tuple[str, str]: - """Determine test_type and reason based on trigger, labels, and changed files. - - This code implements the policies from docs/development/test_filtering.md - and docs/development/ci_behavior_manipulation.md: - - * Available filter types: ["quick", "standard", "comprehensive", "full"] - * Workflow runs choose a filter type automatically but PRs can override - with labels like `test_filter:comprehensive` - - Returns (test_type, reason). - """ - - # Check in priority order - highest priority returns early. - - # Priority 1: test_filter: label is an explicit manual override. - # This is the escape hatch: run comprehensive on a PR before merge, - # or downgrade to quick if you know the change is safe. - # Check both PR labels and workflow_dispatch test labels. - all_labels = ( - ci_inputs.pr_labels - + ci_inputs.linux_test_labels - + ci_inputs.windows_test_labels - ) - for label in all_labels: - if not label.startswith("test_filter:"): - continue - filter_type = label.split(":")[1] - if filter_type not in _VALID_TEST_FILTER_TYPES: - raise ValueError( - f"Unrecognized test_filter value: {filter_type!r}. " - f"Valid values: {sorted(_VALID_TEST_FILTER_TYPES)}" - ) - return filter_type, f"test_filter label: {label}" - - # Priority 2: test:* labels request specific component tests (e.g. - # test:rocprim). When someone explicitly asks for tests, run the full - # suite — they're investigating something specific. - if _has_test_labels(ci_inputs): - return "full", "test labels specified" - - # Priority 3: release builds run deeper test suites than regular CI. - # * 'nightly' gets comprehensive (deeper than standard, on a daily cadence) - # * 'prerelease' gets full (exhaustive pre-release validation) - # * 'dev' falls through to later priorities so changes can be tested quickly - if ci_inputs.release_type == "nightly": - return "comprehensive", "release build (nightly)" - if ci_inputs.release_type == "prerelease": - return "full", "release build (prerelease)" - - # Priority 4: schedule runs the full nightly suite — comprehensive - # coverage on a cadence, catching regressions that quick tests miss. - if ci_inputs.is_schedule: - return "comprehensive", "scheduled run" - - # Priority 5: a submodule change means actual library code changed - # (e.g. rocBLAS, MIOpen). These need full testing since the change - # could affect any downstream consumer. - if git_context.has_submodule_changes is True: - matching = set(git_context.submodule_paths) & set(git_context.changed_files) - return "standard", f"submodule(s) changed: {sorted(matching)}" - - # Default: quick tests for fast CI feedback. - return "quick", "default" - - -def decide_jobs( - ci_inputs: CIInputs, - git_context: GitContext, -) -> JobDecisions: - """Determine which job groups to run, skip, or satisfy with prebuilt files.""" - - # Build ROCm. - # TODO(#3399): Use changed files and build_topology.py to: - # 1. set per-stage prebuilt decisions - # 2. skip job groups that aren't reachable from the changed files - # Parse explicit prebuilt stages from workflow_dispatch input. - stage_decisions: dict[str, JobAction] = {} - if ci_inputs.prebuilt_stages: - for stage in _parse_prebuilt_stages(ci_inputs.prebuilt_stages): - stage_decisions[stage] = JobAction.PREBUILT - build_rocm = BuildRocmDecision( - action=JobAction.RUN, - stage_decisions=stage_decisions, - baseline_run_id=ci_inputs.baseline_run_id, - ) - - # Test ROCm. - test_type, test_type_reason = _determine_test_type( - ci_inputs=ci_inputs, - git_context=git_context, - ) - test_rocm = TestRocmDecision( - action=JobAction.RUN, - test_type=test_type, - test_type_reason=test_type_reason, - ) - - # TODO(#3433): Plumb test_rocm.action through workflow outputs. Until then, - # the skip is enforced in _expand_build_config_for_platform() via test_runs_on. - if ci_inputs.build_variant == "asan": - # Only run ASAN tests on scheduled or workflow_dispatch runs, to avoid impact on submodule bumps - if not (ci_inputs.is_schedule or ci_inputs.is_workflow_dispatch): - test_rocm = TestRocmDecision( - action=JobAction.SKIP, - test_type=test_type, - test_type_reason="ASAN tests skipped due to non-nightly trigger", - ) - - build_pytorch_action = JobAction.RUN if ci_inputs.build_pytorch else JobAction.SKIP - build_jax_action = JobAction.RUN if ci_inputs.build_jax else JobAction.SKIP - - # Other jobs run unconditionally with no configuration. - # TODO: job pruning: skip pytorch if only JAX has been edited, etc. - - return JobDecisions( - build_rocm=build_rocm, - test_rocm=test_rocm, - build_rocm_python=JobGroupDecision(action=JobAction.RUN), - build_pytorch=JobGroupDecision(action=build_pytorch_action), - test_pytorch=JobGroupDecision(action=build_pytorch_action), - build_jax=JobGroupDecision(action=build_jax_action), - ) - - -# --------------------------------------------------------------------------- -# Step 4: Select Targets +# Step 3: Select Targets # --------------------------------------------------------------------------- @@ -891,6 +747,187 @@ def select_targets(ci_inputs: CIInputs) -> TargetSelection: ) +# --------------------------------------------------------------------------- +# Step 4: Decide Jobs +# --------------------------------------------------------------------------- + + +_VALID_TEST_FILTER_TYPES = {"quick", "standard", "comprehensive", "full"} + + +def _has_test_labels(ci_inputs: CIInputs) -> bool: + """Check whether any test labels were specified (workflow_dispatch or PR). + + Note: test_filter: labels are not test labels - they control test_type, + not which tests to run. + """ + # Filter out test_filter: labels - those control test_type, not test selection + linux_tests = [ + l for l in ci_inputs.linux_test_labels if not l.startswith("test_filter:") + ] + windows_tests = [ + l for l in ci_inputs.windows_test_labels if not l.startswith("test_filter:") + ] + if linux_tests or windows_tests: + return True + return any(label.startswith("test:") for label in ci_inputs.pr_labels) + + +def _determine_test_type( + ci_inputs: CIInputs, + git_context: GitContext, +) -> tuple[str, str]: + """Determine test_type and reason based on trigger, labels, and changed files. + + This code implements the policies from docs/development/test_filtering.md + and docs/development/ci_behavior_manipulation.md: + + * Available filter types: ["quick", "standard", "comprehensive", "full"] + * Workflow runs choose a filter type automatically but PRs can override + with labels like `test_filter:comprehensive` + + Returns (test_type, reason). + """ + + # Check in priority order - highest priority returns early. + + # Priority 1: test_filter: label is an explicit manual override. + # This is the escape hatch: run comprehensive on a PR before merge, + # or downgrade to quick if you know the change is safe. + # Check both PR labels and workflow_dispatch test labels. + all_labels = ( + ci_inputs.pr_labels + + ci_inputs.linux_test_labels + + ci_inputs.windows_test_labels + ) + for label in all_labels: + if not label.startswith("test_filter:"): + continue + filter_type = label.split(":")[1] + if filter_type not in _VALID_TEST_FILTER_TYPES: + raise ValueError( + f"Unrecognized test_filter value: {filter_type!r}. " + f"Valid values: {sorted(_VALID_TEST_FILTER_TYPES)}" + ) + return filter_type, f"test_filter label: {label}" + + # Priority 2: test:* labels request specific component tests (e.g. + # test:rocprim). When someone explicitly asks for tests, run the full + # suite — they're investigating something specific. + if _has_test_labels(ci_inputs): + return "full", "test labels specified" + + # Priority 3: release builds run deeper test suites than regular CI. + # * 'nightly' gets comprehensive (deeper than standard, on a daily cadence) + # * 'prerelease' gets full (exhaustive pre-release validation) + # * 'dev' falls through to later priorities so changes can be tested quickly + if ci_inputs.release_type == "nightly": + return "comprehensive", "release build (nightly)" + if ci_inputs.release_type == "prerelease": + return "full", "release build (prerelease)" + + # Priority 4: schedule runs the full nightly suite — comprehensive + # coverage on a cadence, catching regressions that quick tests miss. + if ci_inputs.is_schedule: + return "comprehensive", "scheduled run" + + # Priority 5: a submodule change means actual library code changed + # (e.g. rocBLAS, MIOpen). These need full testing since the change + # could affect any downstream consumer. + if git_context.has_submodule_changes is True: + matching = set(git_context.submodule_paths) & set(git_context.changed_files) + return "standard", f"submodule(s) changed: {sorted(matching)}" + + # Default: quick tests for fast CI feedback. + return "quick", "default" + + +def decide_jobs( + ci_inputs: CIInputs, + git_context: GitContext, + targets: TargetSelection, +) -> JobDecisions: + """Determine which job groups to run, skip, or satisfy with prebuilt files. + ``targets`` (the per-platform family selection from ``select_targets()``) + scopes automatic stage reuse to the platforms and families actually being + built, so a stage is only reused when its artifacts exist for every one of + those platforms. + """ + + # Build ROCm. + # Parse explicit prebuilt stages from workflow_dispatch input. These are + # the MANUAL inputs and are always honored, unchanged, in every mode. + + stage_decisions: dict[str, JobAction] = {} + if ci_inputs.prebuilt_stages: + for stage in _parse_prebuilt_stages(ci_inputs.prebuilt_stages): + stage_decisions[stage] = JobAction.PREBUILT + + # Automatic stage reuse, behind STAGE_REUSE_MODE: in dry-run we only report + # which stages WOULD be reused and apply nothing; in "reuse-stage" the + # eligible stages are merged into stage_decisions so the orchestrator skips + # their builds and copies artifacts instead. The platforms and families to + # verify come straight from the resolved target selection. + + auto_stage_reuse = compute_auto_stage_reuse( + changed_files=git_context.changed_files, + mode=StageReuseMode.from_environ(), + linux_amdgpu_families=targets.linux_families, + windows_amdgpu_families=targets.windows_families, + ) + + # Only reuse-stage mode returns non-empty applied_reuse_stages. + for stage in auto_stage_reuse.applied_reuse_stages: + stage_decisions.setdefault(stage, JobAction.PREBUILT) + baseline_run_id = ci_inputs.baseline_run_id + if auto_stage_reuse.applied_reuse_stages and auto_stage_reuse.baseline_run_id: + baseline_run_id = auto_stage_reuse.baseline_run_id + + build_rocm = BuildRocmDecision( + action=JobAction.RUN, + stage_decisions=stage_decisions, + baseline_run_id=baseline_run_id, + ) + + # Test ROCm. + test_type, test_type_reason = _determine_test_type( + ci_inputs=ci_inputs, + git_context=git_context, + ) + test_rocm = TestRocmDecision( + action=JobAction.RUN, + test_type=test_type, + test_type_reason=test_type_reason, + ) + + # TODO(#3433): Plumb test_rocm.action through workflow outputs. Until then, + # the skip is enforced in _expand_build_config_for_platform() via test_runs_on. + if ci_inputs.build_variant == "asan": + # Only run ASAN tests on scheduled or workflow_dispatch runs, to avoid impact on submodule bumps + if not (ci_inputs.is_schedule or ci_inputs.is_workflow_dispatch): + test_rocm = TestRocmDecision( + action=JobAction.SKIP, + test_type=test_type, + test_type_reason="ASAN tests skipped due to non-nightly trigger", + ) + + build_pytorch_action = JobAction.RUN if ci_inputs.build_pytorch else JobAction.SKIP + build_jax_action = JobAction.RUN if ci_inputs.build_jax else JobAction.SKIP + + # Other jobs run unconditionally with no configuration. + # TODO: job pruning: skip pytorch if only JAX has been edited, etc. + + return JobDecisions( + build_rocm=build_rocm, + test_rocm=test_rocm, + build_rocm_python=JobGroupDecision(action=JobAction.RUN), + build_pytorch=JobGroupDecision(action=build_pytorch_action), + test_pytorch=JobGroupDecision(action=build_pytorch_action), + build_jax=JobGroupDecision(action=build_jax_action), + auto_stage_reuse=auto_stage_reuse, + ) + + # --------------------------------------------------------------------------- # Step 5: Build Configs # --------------------------------------------------------------------------- @@ -1272,6 +1309,11 @@ def write_outputs( ) ) + # Append the automatic stage-reuse analysis after the main summary so the + # two read top-to-bottom in the job step summary. + if outputs.jobs is not None and outputs.jobs.auto_stage_reuse is not None: + gha_append_step_summary(render_step_summary(outputs.jobs.auto_stage_reuse)) + # --------------------------------------------------------------------------- # Pipeline orchestration @@ -1294,14 +1336,16 @@ def configure(ci_inputs: CIInputs, git_context: GitContext) -> CIOutputs: return CIOutputs.skipped() print("Result: CI will run") - print("\n=== Deciding job configuration ===") - jobs = decide_jobs(ci_inputs=ci_inputs, git_context=git_context) - jobs.log() - print("\n=== Selecting GPU target families ===") targets = select_targets(ci_inputs) targets.log() + print("\n=== Deciding job configuration ===") + # Target selection runs first so automatic stage reuse can scope its + # prebuilt artifact checks to those targets. + jobs = decide_jobs(ci_inputs=ci_inputs, git_context=git_context, targets=targets) + jobs.log() + print("\n=== Building per-platform configs ===") builds = expand_build_configs( ci_inputs=ci_inputs, diff --git a/build_tools/github_actions/stage_reuse_decision.py b/build_tools/github_actions/stage_reuse_decision.py new file mode 100644 index 00000000000..6722adcac1e --- /dev/null +++ b/build_tools/github_actions/stage_reuse_decision.py @@ -0,0 +1,682 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Auto-compute per-stage rebuild/reuse decisions for multi-arch CI. + +This module decides, per build stage, whether the stage can be satisfied with +prebuilt artifacts (reuse) instead of being rebuilt. It reuses the existing +``stage_impact`` and ``baseline_runs`` tooling and adds two gates before a stage +is reported reusable: + +* Impact gate (``stage_impact.analyze_stage_impact``) - the change does not + affect the stage, so it is a *candidate* for reuse. +* Availability gate (``baseline_runs.select_baseline_run``) - a single healthy, + commit-compatible baseline run actually contains the artifacts that stage + would produce, verified independently for every platform being built. + +Mode switch +----------- +The module is wired into CI behind a two-way ``STAGE_REUSE_MODE`` switch so it +can be observed before it changes anything: + +* ``dry-run`` - DEFAULT. Compute the analysis and LOG, for each stage that + *would* be reused, a line to the console + step summary, but + return NO auto stages, so ``prebuilt_stages`` is unchanged + and every stage still builds exactly. +* ``reuse-stage`` - Compute the analysis and actually return the reuse stages so + the orchestrator copies their artifacts and skips the build. + +Note: this ``STAGE_REUSE_MODE`` switch drives the *automatic* detection layer. +It is orthogonal to the ``prebuilt_stages`` workflow input, which is the +explicit, manual list of stages to reuse and is always honored regardless of +mode. + +Environment Variables +-------------------- +``_default_baseline_selector`` reads the following environment variables. These +are set by ``setup_multi_arch.yml`` and form the stage-reuse interface: + +* ``STAGE_REUSE_MODE`` - ``dry-run`` (default) or ``reuse-stage``. +* ``GITHUB_REPOSITORY`` - ``owner/repo`` (default ``ROCm/TheRock``). +* ``STAGE_REUSE_BASELINE_BRANCH`` - baseline branch to search (default ``main``). +* ``STAGE_REUSE_BASELINE_WORKFLOW`` - baseline workflow file + (default ``multi_arch_ci.yml``). +* ``STAGE_REUSE_CURRENT_SHA`` - current commit SHA; enables the + commit-compatibility rule when set. +* ``STAGE_REUSE_MAX_AGE_HOURS`` - recency window in hours; disables the recency + rule when unset. +* ``STAGE_REUSE_COMMIT_HISTORY`` - number of branch commits to fetch for + ancestry (default ``50``). +""" + +import enum +import os +import logging +import functools +import sys +import baseline_runs +import github_actions_api +from pathlib import Path +from dataclasses import dataclass, field +from typing import Callable, Optional, Sequence + +# Add build_tools to the path so sibling CI modules and _therock_utils import +# cleanly regardless of the current working directory. +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from _therock_utils.artifact_backend import ARTIFACT_EXTENSIONS +from _therock_utils.build_topology import BuildTopology, get_topology +from artifact_manager import ARTIFACT_COMPONENTS +from baseline_runs import BaselineRun, RequiredArtifact +from github_actions_api import GitHubAPIError +from stage_impact import analyze_stage_impact + +logger = logging.getLogger(__name__) + +LOG_PREFIX = "[STAGE-REUSE]" +GENERIC_FAMILY = "generic" + + +class StageReuseMode(enum.Enum): + DRY_RUN = "dry-run" + REUSE_STAGE = "reuse-stage" + + @staticmethod + def from_environ(default: "StageReuseMode | None" = None) -> "StageReuseMode": + """Read STAGE_REUSE_MODE; default to dry-run when unset/invalid.""" + default = default or StageReuseMode.DRY_RUN + raw = (os.environ.get("STAGE_REUSE_MODE", "") or "").strip().lower() + for mode in StageReuseMode: + if raw == mode.value: + return mode + return default + + +BaselineSelector = Callable[[Sequence[RequiredArtifact]], BaselineRun | None] + + +@dataclass(frozen=True) +class StageReusePlan: + """Result of the *planning* step: which stages are reuse candidates. + This is a pure function of the changed files and topology -- no baseline + selection, artifact verification, or reporting. Keeping it separate lets + callers reuse the impact decision without triggering any network/API work. + """ + + candidate_stages: tuple[str, ...] + rebuild_stages: tuple[str, ...] + full_rebuild_required: bool + reasons: tuple[str, ...] + + +@dataclass(frozen=True) +class AutoStageReuse: + """Result of the auto stage-reuse analysis (planning + verification).""" + + mode: StageReuseMode + candidate_stages: tuple[str, ...] + rebuild_stages: tuple[str, ...] + full_rebuild_required: bool + baseline_run_id: str | None + baseline_html_url: str | None + available_stages: tuple[str, ...] + unavailable_stages: tuple[str, ...] + applied_reuse_stages: tuple[str, ...] + reasons: tuple[str, ...] + report_lines: tuple[str, ...] = field(default_factory=tuple) + platform_available: dict[str, tuple[str, ...]] = field(default_factory=dict) + + +def _target_families( + linux_amdgpu_families: Sequence[str], + windows_amdgpu_families: Sequence[str], +) -> tuple[str, ...]: + """Family list whose artifacts must be verified for stage reuse. + De-duplicates the requested Linux and Windows families and always appends + the generic pseudo-family, since every stage produces a generic archive. + """ + families = list(dict.fromkeys([*linux_amdgpu_families, *windows_amdgpu_families])) + if GENERIC_FAMILY not in families: + families.append(GENERIC_FAMILY) + return tuple(families) + + +def _required_artifacts_for_stages( + topology: BuildTopology, + stage_names: Sequence[str], + target_families: Sequence[str], +) -> list[RequiredArtifact]: + """Artifact/family pairs the given stages produce.""" + + artifacts_by_group = topology.get_artifact_group_to_artifacts() + required: list[RequiredArtifact] = [] + seen: set[RequiredArtifact] = set() + for stage_name in stage_names: + stage = topology.build_stages.get(stage_name) + if stage is None: + continue + for group_name in stage.artifact_groups: + for artifact_name in artifacts_by_group.get(group_name, []): + for family in target_families: + req = RequiredArtifact(name=artifact_name, target_family=family) + if req not in seen: + seen.add(req) + required.append(req) + return required + + +def _matched_filenames(baseline: BaselineRun | None) -> set[str]: + if baseline is None: + return set() + return set(baseline.artifact_availability.matched_filenames) + + +def _stage_artifacts_available( + topology: BuildTopology, + stage_name: str, + target_families: Sequence[str], + available_filenames: set[str], +) -> bool: + """True when every artifact this stage produces has an archive present.""" + + artifacts_by_group = topology.get_artifact_group_to_artifacts() + stage = topology.build_stages.get(stage_name) + if stage is None: + return False + for group_name in stage.artifact_groups: + for artifact_name in artifacts_by_group.get(group_name, []): + for family in target_families: + found = False + for component in ARTIFACT_COMPONENTS: + for extension in ARTIFACT_EXTENSIONS: + filename = f"{artifact_name}_{component}_{family}{extension}" + if filename in available_filenames: + found = True + break + if found: + break + if not found: + return False + return True + + +def plan_stage_reuse( + *, + changed_files: Sequence[str] | None, + platform: str | None = None, + topology: BuildTopology | None = None, +) -> StageReusePlan: + """Planning step: which stages are unaffected reuse candidates. + + Pure decision logic -- no baseline selection, artifact verification, or + reporting -- so it can be reused independently of the CI plumbing. + """ + + if changed_files is None: + return StageReusePlan( + candidate_stages=(), + rebuild_stages=(), + full_rebuild_required=True, + reasons=("no changed-file list available",), + ) + + if topology is None: + topology = get_topology() + + # TODO(#3399): thread build flags/variant through here for superrepos so a + # baseline built with different flags is not considered reusable. Not needed + # for the current single-variant multi-arch CI, but required before enabling + # reuse across superrepo builds that vary build configuration. + impact = analyze_stage_impact( + changed_inputs=list(changed_files), + topology=topology, + platform=platform, + ) + return StageReusePlan( + candidate_stages=tuple(impact.copy_stages), + rebuild_stages=tuple(impact.rebuild_stages), + full_rebuild_required=impact.full_rebuild_required, + reasons=tuple(impact.reasons), + ) + + +def compute_auto_stage_reuse( + *, + changed_files: Sequence[str] | None, + mode: StageReuseMode, + linux_amdgpu_families: Sequence[str] = (), + windows_amdgpu_families: Sequence[str] = (), + topology: BuildTopology | None = None, + baseline_selector: BaselineSelector | None = None, + baseline_selector_factory: Callable[[str], BaselineSelector] | None = None, +) -> AutoStageReuse: + """Compute auto stage-reuse decisions, verified against a baseline run. + A stage is only reusable when it is unaffected by the change AND its + artifacts are present in a healthy baseline run for *every* platform being + built. The platforms are derived from which family lists are non-empty: + ``linux_amdgpu_families`` implies the ``linux`` platform, and + ``windows_amdgpu_families`` implies ``windows``. This guards against the + case where a stage available only in the Linux baseline is skipped on + Windows. The report lines are logged before returning. + """ + platforms = _build_platforms(linux_amdgpu_families, windows_amdgpu_families) + if not platforms: + return _log_and_return( + _empty_result( + mode, + full_rebuild_required=True, + reasons=("no build platforms selected",), + report_lines=( + f"{LOG_PREFIX} no build platforms selected; " + f"automatic stage reuse disabled.", + ), + ) + ) + + if topology is None and changed_files is not None: + topology = get_topology() + + families = _target_families(linux_amdgpu_families, windows_amdgpu_families) + + plan = plan_stage_reuse( + changed_files=changed_files, + platform=platforms[0], + topology=topology, + ) + candidates = plan.candidate_stages + rebuild = plan.rebuild_stages + + if changed_files is None: + return _log_and_return( + _empty_result( + mode, + full_rebuild_required=True, + reasons=plan.reasons, + report_lines=( + f"{LOG_PREFIX} mode={mode.value}; no changed-file list. " + f"Conservatively rebuilding all stages.", + ), + ) + ) + + if plan.full_rebuild_required or not candidates: + lines = _format_report( + mode=mode, + candidates=candidates, + rebuild=rebuild, + full_rebuild_required=plan.full_rebuild_required, + reasons=plan.reasons, + baseline_run_id=None, + available=(), + unavailable=candidates, + ) + return _log_and_return( + AutoStageReuse( + mode=mode, + candidate_stages=candidates, + rebuild_stages=rebuild, + full_rebuild_required=plan.full_rebuild_required, + baseline_run_id=None, + baseline_html_url=None, + available_stages=(), + unavailable_stages=candidates, + applied_reuse_stages=(), + reasons=plan.reasons, + report_lines=lines, + ) + ) + + required = _required_artifacts_for_stages(topology, candidates, families) + # Verify artifact availability independently for each platform. A single + # ``baseline_selector`` (used by tests) applies to all platforms; otherwise + # a per-platform selector is built so each platform is checked against a + # baseline that actually produced that platform's artifacts. + platform_baseline_run_ids: dict[str, str | None] = {} + platform_baseline_urls: dict[str, str | None] = {} + per_platform_available: dict[str, tuple[str, ...]] = {} + baseline_error: str | None = None + + for platform in platforms: + if baseline_selector is not None: + selector = baseline_selector + elif baseline_selector_factory is not None: + selector = baseline_selector_factory(platform) + else: + selector = _default_baseline_selector(platform=platform) + + # Only transient GitHub API / network failures are tolerated here: a + # failed baseline lookup falls back to a full rebuild, which is safe. + # Configuration errors (e.g. a bad required-artifacts request) indicate + # a bug and must surface, so they are left to propagate. + try: + baseline = selector(required) + except GitHubAPIError as exc: + baseline_error = str(exc) + baseline = None + + platform_baseline_run_ids[platform] = ( + baseline.run_id if baseline is not None else None + ) + platform_baseline_urls[platform] = ( + baseline.html_url if baseline is not None else None + ) + + available_filenames = _matched_filenames(baseline) + available_here: list[str] = [] + if baseline is not None: + for stage_name in candidates: + if _stage_artifacts_available( + topology, stage_name, families, available_filenames + ): + available_here.append(stage_name) + per_platform_available[platform] = tuple(available_here) + + selected_run_ids = { + run_id for run_id in platform_baseline_run_ids.values() if run_id is not None + } + + # The workflow can only copy artifacts from a single baseline_run_id, so if + # platforms resolve to different baseline runs we cannot safely reuse. + + if len(selected_run_ids) > 1: + return _log_and_return( + _empty_result( + mode, + full_rebuild_required=True, + reasons=( + "automatic reuse resolved different baseline runs per platform", + ), + report_lines=( + f"{LOG_PREFIX} multiple baseline runs selected across " + f"platforms; disabling automatic reuse.", + ), + ) + ) + + reported_baseline_run_id = next(iter(selected_run_ids), None) + reported_baseline_url = next( + (url for url in platform_baseline_urls.values() if url), None + ) + + # A stage is available only when present on EVERY platform being built. + available: list[str] = [] + unavailable: list[str] = [] + + for stage_name in candidates: + if all( + stage_name in per_platform_available.get(platform, ()) + for platform in platforms + ): + available.append(stage_name) + else: + unavailable.append(stage_name) + + available_t = tuple(available) + unavailable_t = tuple(unavailable) + applied = available_t if mode is StageReuseMode.REUSE_STAGE else () + + lines = _format_report( + mode=mode, + candidates=candidates, + rebuild=rebuild, + full_rebuild_required=False, + reasons=plan.reasons, + baseline_run_id=reported_baseline_run_id, + available=available_t, + unavailable=unavailable_t, + baseline_error=baseline_error, + platforms=platforms, + platform_available=per_platform_available, + ) + return _log_and_return( + AutoStageReuse( + mode=mode, + candidate_stages=candidates, + rebuild_stages=rebuild, + full_rebuild_required=False, + baseline_run_id=reported_baseline_run_id, + baseline_html_url=reported_baseline_url, + available_stages=available_t, + unavailable_stages=unavailable_t, + applied_reuse_stages=applied, + reasons=plan.reasons, + report_lines=lines, + platform_available=per_platform_available, + ) + ) + + +def _build_platforms( + linux_amdgpu_families: Sequence[str], + windows_amdgpu_families: Sequence[str], +) -> tuple[str, ...]: + """Platforms whose baselines must contain a stage's artifacts to reuse it. + + Automatic reuse merges into the prebuilt_stages that flow to BOTH the Linux + and Windows build configs, so a stage may only be reused when its artifacts + are available for every platform actually being built. + """ + platforms: list[str] = [] + if linux_amdgpu_families: + platforms.append("linux") + if windows_amdgpu_families: + platforms.append("windows") + return tuple(platforms) + + +def _default_baseline_selector(*, platform: str) -> BaselineSelector: + """Build a selector bound to baseline_runs.select_baseline_run. + ``select_baseline_run`` already requires each candidate run to have healthy + build jobs (``required_successful_job_name_substrings=("Build",)``) AND to + contain all requested artifacts. A run with no artifacts (e.g. a docs-only + change) therefore fails the availability gate and is never selected, so no + extra "passing build" check is needed here. + """ + + github_repository = os.environ.get("GITHUB_REPOSITORY", "ROCm/TheRock") + branch = os.environ.get("STAGE_REUSE_BASELINE_BRANCH", "main") + workflow_name = os.environ.get("STAGE_REUSE_BASELINE_WORKFLOW", "multi_arch_ci.yml") + current_commit_sha = os.environ.get("STAGE_REUSE_CURRENT_SHA") or None + max_age_hours_raw = os.environ.get("STAGE_REUSE_MAX_AGE_HOURS") + max_age_hours = float(max_age_hours_raw) if max_age_hours_raw else None + history_count_raw = os.environ.get("STAGE_REUSE_COMMIT_HISTORY", "50") + try: + history_count = max(1, int(history_count_raw)) + except ValueError: + history_count = 50 + # The commit-compatibility rule needs the branch history (newest-first) to + # establish ancestry. select_baseline_run only accepts a candidate whose + # head_sha is `same` or `ancestor` of current_commit_sha; with an EMPTY + # window every candidate resolves to `unknown` and is rejected, so reuse + # never activates. Fetch the real history here. If the SHA is set but the + # history fetch fails (or returns empty), disable the commit rule (pass both + # as None) rather than enabling it with an empty window -- recency and + # artifact availability still gate the selection. + ordered_commit_shas = None + effective_commit_sha = current_commit_sha + if current_commit_sha is not None: + try: + ordered_commit_shas = github_actions_api.gha_query_recent_branch_commits( + github_repository_name=github_repository, + branch=branch, + max_count=history_count, + ) + except GitHubAPIError as exc: + logger.warning( + "%s could not fetch branch history (%s); " + "skipping commit-compatibility rule.", + LOG_PREFIX, + exc, + ) + ordered_commit_shas = None + if not ordered_commit_shas: + effective_commit_sha = None + ordered_commit_shas = None + + # A functools.partial binds the resolved configuration to + # select_baseline_run; the only free argument is required_artifacts, which + # matches the BaselineSelector signature. + return functools.partial( + _invoke_select_baseline_run, + github_repository=github_repository, + workflow_name=workflow_name, + branch=branch, + platform=platform, + current_commit_sha=effective_commit_sha, + ordered_commit_shas=ordered_commit_shas, + max_age_hours=max_age_hours, + ) + + +def _invoke_select_baseline_run( + required: Sequence[RequiredArtifact], **kwargs +) -> BaselineRun | None: + """Adapter so a partial can present the BaselineSelector(required) shape. + + Calls through the ``baseline_runs`` module attribute (rather than a bound + reference) so tests can monkeypatch ``select_baseline_run``. + """ + return baseline_runs.select_baseline_run(required_artifacts=required, **kwargs) + + +def _empty_result( + mode: StageReuseMode, + *, + full_rebuild_required: bool = False, + reasons: tuple[str, ...] = (), + report_lines: tuple[str, ...] = (), +) -> AutoStageReuse: + return AutoStageReuse( + mode=mode, + candidate_stages=(), + rebuild_stages=(), + full_rebuild_required=full_rebuild_required, + baseline_run_id=None, + baseline_html_url=None, + available_stages=(), + unavailable_stages=(), + applied_reuse_stages=(), + reasons=reasons, + report_lines=report_lines, + ) + + +def _format_report( + *, + mode: StageReuseMode, + candidates: Sequence[str], + rebuild: Sequence[str], + full_rebuild_required: bool, + reasons: Sequence[str], + baseline_run_id: str | None, + available: Sequence[str], + unavailable: Sequence[str], + baseline_error: str | None = None, + platforms: Sequence[str] = (), + platform_available: dict[str, tuple[str, ...]] | None = None, +) -> tuple[str, ...]: + platform_available = platform_available or {} + lines: list[str] = [f"{LOG_PREFIX} mode={mode.value}"] + if platforms: + lines.append(f"{LOG_PREFIX} platforms verified: {', '.join(platforms)}") + if full_rebuild_required: + lines.append( + f"{LOG_PREFIX} conservative full rebuild: no stages eligible for reuse." + ) + for reason in reasons: + lines.append(f"{LOG_PREFIX} reason: {reason}") + return tuple(lines) + if not candidates: + lines.append(f"{LOG_PREFIX} no unaffected stages; all stages rebuild.") + return tuple(lines) + if baseline_error: + lines.append( + f"{LOG_PREFIX} baseline lookup failed ({baseline_error}); " + f"cannot verify artifacts, rebuilding all candidates." + ) + elif baseline_run_id: + lines.append(f"{LOG_PREFIX} baseline run for artifact check: {baseline_run_id}") + else: + lines.append( + f"{LOG_PREFIX} no baseline run contains artifacts for all " + f"candidate stages; rebuilding all candidates." + ) + verb = ( + "WILL be skipped" if mode is StageReuseMode.REUSE_STAGE else "WOULD be skipped" + ) + for stage in available: + lines.append( + f"{LOG_PREFIX} stage '{stage}' unaffected AND available in " + f"baseline on all platforms -> {verb}" + ) + for stage in unavailable: + # Call out WHICH platforms are missing the artifacts so the mismatch + # (e.g. present on linux, absent on windows) is visible in the log. + if len(platforms) > 1: + missing = [ + platform + for platform in platforms + if stage not in platform_available.get(platform, ()) + ] + where = f" (missing on: {', '.join(missing)})" if missing else "" + else: + where = "" + lines.append( + f"{LOG_PREFIX} stage '{stage}' unaffected but artifacts " + f"NOT available -> rebuild{where}" + ) + if rebuild: + lines.append(f"{LOG_PREFIX} stages rebuilding (impacted): {', '.join(rebuild)}") + if mode is StageReuseMode.DRY_RUN and available: + lines.append( + f"{LOG_PREFIX} dry-run: prebuilt_stages NOT modified; all stages " + f"still build." + ) + return tuple(lines) + + +def _format_stage_list(stages: Sequence[str]) -> str: + """Render a tuple of stage names as backticked, comma-separated markdown.""" + if not stages: + return "_none_" + return ", ".join(f"`{stage}`" for stage in stages) + + +def render_step_summary(result: AutoStageReuse) -> str: + """Render a GitHub step-summary markdown block for the analysis.""" + baseline = f"`{result.baseline_run_id}`" if result.baseline_run_id else "_none_" + candidates = _format_stage_list(result.candidate_stages) + available = _format_stage_list(result.available_stages) + applied = _format_stage_list(result.applied_reuse_stages) + + out = ["### Stage reuse analysis", ""] + out.append(f"- mode: `{result.mode.value}`") + out.append(f"- full rebuild required: `{result.full_rebuild_required}`") + out.append(f"- baseline run checked: {baseline}") + out.append(f"- unaffected candidates: {candidates}") + out.append(f"- available in baseline: {available}") + out.append(f"- applied: {applied}") + if result.platform_available: + out.append("- available per platform:") + for platform, stages in result.platform_available.items(): + out.append(f" - {platform}: {_format_stage_list(stages)}") + if result.reasons: + out.append("- reasons:") + for reason in result.reasons: + out.append(f" - {reason}") + if result.mode is StageReuseMode.DRY_RUN and result.available_stages: + out.append("") + out.append( + "> Dry-run only: no build steps were skipped. Artifacts were " + "verified against the baseline run above. Set " + "`STAGE_REUSE_MODE=reuse-stage` after review to enable skipping." + ) + return "\n".join(out) + + +def _log_and_return(result: AutoStageReuse) -> AutoStageReuse: + """Emit the analysis report lines to the module logger and return result.""" + for line in result.report_lines: + logger.info(line) + return result diff --git a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py index 2943dce3213..624c3f4c611 100644 --- a/build_tools/github_actions/tests/configure_multi_arch_ci_test.py +++ b/build_tools/github_actions/tests/configure_multi_arch_ci_test.py @@ -343,7 +343,9 @@ def _inputs(self, **kwargs): def test_all_job_groups_run(self): """All job groups are set to run (subgraph selection is Phase 4).""" - result = cm.decide_jobs(self._inputs(), git_context=cm.GitContext()) + result = cm.decide_jobs( + self._inputs(), git_context=cm.GitContext(), targets=cm.TargetSelection() + ) self.assertIsInstance(result, cm.JobDecisions) self.assertEqual(result.build_rocm.action, cm.JobAction.RUN) self.assertEqual(result.test_rocm.action, cm.JobAction.RUN) @@ -356,6 +358,7 @@ def test_build_pytorch_input_skips_pytorch_jobs(self): result = cm.decide_jobs( self._inputs(build_pytorch=False), git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual(result.build_pytorch.action, cm.JobAction.SKIP) @@ -365,6 +368,7 @@ def test_build_jax_input_runs_jax_job(self): result = cm.decide_jobs( self._inputs(build_jax=True), git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual(result.build_jax.action, cm.JobAction.RUN) @@ -372,13 +376,17 @@ def test_build_jax_input_runs_jax_job(self): def test_default_test_type_is_quick(self): """Default test_type for PR/push with no special conditions.""" git = cm.GitContext(changed_files=["CMakeLists.txt"]) - result = cm.decide_jobs(self._inputs(), git_context=git) + result = cm.decide_jobs( + self._inputs(), git_context=git, targets=cm.TargetSelection() + ) self.assertEqual(result.test_rocm.test_type, "quick") def test_schedule_is_comprehensive(self): """Schedule trigger → comprehensive tests.""" result = cm.decide_jobs( - self._inputs(event_name="schedule"), git_context=cm.GitContext() + self._inputs(event_name="schedule"), + git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual(result.test_rocm.test_type, "comprehensive") @@ -388,7 +396,9 @@ def test_submodule_change_is_full(self): changed_files=["rocm-libraries", "CMakeLists.txt"], submodule_paths=["rocm-libraries", "rocm-systems"], ) - result = cm.decide_jobs(self._inputs(), git_context=git) + result = cm.decide_jobs( + self._inputs(), git_context=git, targets=cm.TargetSelection() + ) self.assertEqual(result.test_rocm.test_type, "standard") self.assertIn("submodule", result.test_rocm.test_type_reason) @@ -398,14 +408,18 @@ def test_no_submodule_change_stays_quick(self): changed_files=["CMakeLists.txt"], submodule_paths=["rocm-libraries", "rocm-systems"], ) - result = cm.decide_jobs(self._inputs(), git_context=git) + result = cm.decide_jobs( + self._inputs(), git_context=git, targets=cm.TargetSelection() + ) self.assertEqual(result.test_rocm.test_type, "quick") def test_pr_test_label_is_full(self): """PR with test:* label → full tests.""" git = cm.GitContext(changed_files=["CMakeLists.txt"]) result = cm.decide_jobs( - self._inputs(pr_labels=["test:rocprim"]), git_context=git + self._inputs(pr_labels=["test:rocprim"]), + git_context=git, + targets=cm.TargetSelection(), ) self.assertEqual(result.test_rocm.test_type, "full") @@ -417,13 +431,16 @@ def test_workflow_dispatch_test_labels_is_full(self): linux_test_labels=["test:rocprim"], ), git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual(result.test_rocm.test_type, "full") def test_nightly_release_is_comprehensive(self): """Nightly release → comprehensive tests.""" result = cm.decide_jobs( - self._inputs(release_type="nightly"), git_context=cm.GitContext() + self._inputs(release_type="nightly"), + git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual(result.test_rocm.test_type, "comprehensive") self.assertIn("release", result.test_rocm.test_type_reason) @@ -431,7 +448,9 @@ def test_nightly_release_is_comprehensive(self): def test_prerelease_is_full(self): """Prerelease → full tests.""" result = cm.decide_jobs( - self._inputs(release_type="prerelease"), git_context=cm.GitContext() + self._inputs(release_type="prerelease"), + git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual(result.test_rocm.test_type, "full") self.assertIn("release", result.test_rocm.test_type_reason) @@ -439,7 +458,11 @@ def test_prerelease_is_full(self): def test_dev_release_falls_through_to_default(self): """Dev release without other signals → quick (falls through).""" git = cm.GitContext(changed_files=["CMakeLists.txt"]) - result = cm.decide_jobs(self._inputs(release_type="dev"), git_context=git) + result = cm.decide_jobs( + self._inputs(release_type="dev"), + git_context=git, + targets=cm.TargetSelection(), + ) self.assertEqual(result.test_rocm.test_type, "quick") def test_test_filter_label_overrides(self): @@ -451,6 +474,7 @@ def test_test_filter_label_overrides(self): pr_labels=["test_filter:standard"], ), git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual(result.test_rocm.test_type, "standard") @@ -459,7 +483,9 @@ def test_test_filter_invalid_raises(self): git = cm.GitContext(changed_files=["CMakeLists.txt"]) with self.assertRaises(ValueError, msg="Unrecognized test_filter"): cm.decide_jobs( - self._inputs(pr_labels=["test_filter:bogus"]), git_context=git + self._inputs(pr_labels=["test_filter:bogus"]), + git_context=git, + targets=cm.TargetSelection(), ) def test_workflow_dispatch_test_filter_label_overrides(self): @@ -472,6 +498,7 @@ def test_workflow_dispatch_test_filter_label_overrides(self): linux_test_labels=["test_filter:comprehensive"], ), git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual(result.test_rocm.test_type, "comprehensive") self.assertIn("test_filter", result.test_rocm.test_type_reason) @@ -484,6 +511,7 @@ def test_explicit_prebuilt_stages(self): prebuilt_stages="compiler-runtime,runtime-tests", ), git_context=cm.GitContext(), + targets=cm.TargetSelection(), ) self.assertEqual( sorted(result.build_rocm.prebuilt_stages), @@ -491,9 +519,28 @@ def test_explicit_prebuilt_stages(self): ) self.assertEqual(result.build_rocm.rebuild_stages, []) + def test_reuse_scoped_to_selected_targets(self): + """decide_jobs threads the resolved targets into automatic reuse. + With no families selected there are no build platforms, so automatic + reuse is disabled and reports a full rebuild -- confirming the decision + is scoped to the passed-in target selection. + """ + result = cm.decide_jobs( + self._inputs(), + git_context=cm.GitContext( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"] + ), + targets=cm.TargetSelection(), + ) + self.assertIsNotNone(result.auto_stage_reuse) + self.assertTrue(result.auto_stage_reuse.full_rebuild_required) + self.assertEqual(result.auto_stage_reuse.applied_reuse_stages, ()) + def test_no_prebuilt_stages_by_default(self): """Without explicit prebuilt_stages, no stage decisions are set.""" - result = cm.decide_jobs(self._inputs(), git_context=cm.GitContext()) + result = cm.decide_jobs( + self._inputs(), git_context=cm.GitContext(), targets=cm.TargetSelection() + ) self.assertEqual(result.build_rocm.prebuilt_stages, []) self.assertEqual(result.build_rocm.stage_decisions, {}) @@ -523,6 +570,7 @@ def test_asan_tests_only_run_on_nightly_triggers(self): result = cm.decide_jobs( self._inputs(event_name=event, build_variant="asan"), git_context=git_context, + targets=cm.TargetSelection(), ) self.assertEqual( result.test_rocm.action, @@ -535,6 +583,7 @@ def test_asan_tests_only_run_on_nightly_triggers(self): result = cm.decide_jobs( self._inputs(event_name=event, build_variant="asan"), git_context=git_context, + targets=cm.TargetSelection(), ) self.assertEqual( result.test_rocm.action, @@ -546,6 +595,7 @@ def test_asan_tests_only_run_on_nightly_triggers(self): result = cm.decide_jobs( self._inputs(event_name="pull_request", build_variant="release"), git_context=git_context, + targets=cm.TargetSelection(), ) self.assertEqual(result.test_rocm.action, cm.JobAction.RUN) @@ -1444,6 +1494,50 @@ def test_no_test_labels_has_empty_outputs(self): self.assertEqual(outputs.linux_test_labels, []) self.assertEqual(outputs.windows_test_labels, []) + @patch("configure_multi_arch_ci.decide_jobs") + def test_configure_propagates_auto_reuse_baseline_run_id(self, mock_decide_jobs): + """When auto reuse applies, the selected baseline run id reaches BuildConfig.""" + mock_decide_jobs.return_value = cm.JobDecisions( + build_rocm=cm.BuildRocmDecision( + action=cm.JobAction.RUN, + stage_decisions={ + "compiler-runtime": cm.JobAction.PREBUILT, + "math-libs": cm.JobAction.RUN, + }, + baseline_run_id="123", + ), + test_rocm=cm.TestRocmDecision( + action=cm.JobAction.RUN, + test_type="quick", + ), + build_rocm_python=cm.JobGroupDecision(action=cm.JobAction.RUN), + build_pytorch=cm.JobGroupDecision(action=cm.JobAction.RUN), + test_pytorch=cm.JobGroupDecision(action=cm.JobAction.RUN), + build_jax=cm.JobGroupDecision(action=cm.JobAction.SKIP), + ) + + inputs = cm.CIInputs( + run_id="12345", + event_name="push", + commit_ref="main", + base_ref="HEAD^1", + build_variant="release", + ) + + outputs = cm.configure(inputs, cm.GitContext.empty()) + + self.assertIsNotNone(outputs.builds.linux) + self.assertIsNotNone(outputs.builds.windows) + + self.assertEqual(outputs.builds.linux.baseline_run_id, "123") + self.assertEqual(outputs.builds.windows.baseline_run_id, "123") + self.assertIn("compiler-runtime", outputs.builds.linux.prebuilt_stages) + self.assertIn("compiler-runtime", outputs.builds.windows.prebuilt_stages) + + linux_payload = outputs.builds.linux.to_dict() + self.assertEqual(linux_payload["baseline_run_id"], "123") + self.assertEqual(linux_payload["prebuilt_stages"], "compiler-runtime") + # --------------------------------------------------------------------------- # Contract: BuildConfig fields match workflow YAML references diff --git a/build_tools/github_actions/tests/stage_reuse_decision_test.py b/build_tools/github_actions/tests/stage_reuse_decision_test.py new file mode 100644 index 00000000000..3e6fedc88ae --- /dev/null +++ b/build_tools/github_actions/tests/stage_reuse_decision_test.py @@ -0,0 +1,508 @@ +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Tests for stage_reuse_decision: impact + baseline-availability gates.""" + +from pathlib import Path +import sys +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import stage_reuse_decision as srd +from stage_reuse_decision import StageReuseMode, compute_auto_stage_reuse +from baseline_runs import ( + BaselineRun, + WorkflowRunSummary, + ArtifactAvailability, + WorkflowJobHealth, +) +from github_actions_api import GitHubAPIError + + +class _FakeStage: + def __init__(self, groups): + self.artifact_groups = groups + + +class FakeTopology: + """Minimal BuildTopology stand-in for stage_impact + artifact derivation. + + compiler-runtime produces artifact 'base'; math-libs produces 'blas'. + """ + + def __init__(self): + self.build_stages = { + "compiler-runtime": _FakeStage(["base-group"]), + "math-libs": _FakeStage(["blas-group"]), + } + self.artifact_groups = { + "base-group": type("G", (), {"source_sets": ["core"]})(), + "blas-group": type("G", (), {"source_sets": ["libs"]})(), + } + + def get_source_set_to_artifact_groups(self): + return {"core": ["base-group"], "libs": ["blas-group"]} + + def get_artifact_group_to_build_stages(self): + return {"base-group": ["compiler-runtime"], "blas-group": ["math-libs"]} + + def get_artifact_group_to_artifacts(self): + return {"base-group": ["base"], "blas-group": ["blas"]} + + def get_produced_artifacts(self, stage_name): + return set() + + def get_artifacts_in_group(self, group_name): + return {"base-group": [], "blas-group": []}.get(group_name, []) + + def get_source_set_for_submodule(self, name, platform=None): + mapping = {"rocm-libraries": "libs"} + ss = mapping.get(name) + return type("S", (), {"name": ss})() if ss else None + + def get_source_set_for_path(self, path, platform=None): + return None + + +def _baseline(run_id, matched_filenames): + summary = WorkflowRunSummary( + repository="ROCm/TheRock", + branch="main", + commit=f"sha-{run_id}", + workflow="multi_arch_ci.yml", + run_id=run_id, + status="completed", + conclusion="success", + timestamp="2026-06-17T20:00:00Z", + html_url=f"https://github.com/ROCm/TheRock/actions/runs/{run_id}", + ) + return BaselineRun( + source_ref=summary, + platform="linux", + job_health=WorkflowJobHealth( + required_name_substrings=("Build",), + matched_job_names=("Build",), + failed_job_names=(), + missing_name_substrings=(), + ), + artifact_availability=ArtifactAvailability( + required_artifacts=(), + matched_filenames=tuple(matched_filenames), + missing_artifacts=(), + ), + ) + + +def _selector(baseline): + """Return an injected baseline_selector that always yields `baseline`.""" + return lambda required: baseline + + +class ModeParsingTest(unittest.TestCase): + def test_default_is_dry_run(self): + import os + + os.environ.pop("STAGE_REUSE_MODE", None) + self.assertEqual(StageReuseMode.from_environ(), StageReuseMode.DRY_RUN) + + def test_explicit_modes(self): + import os + + for value, expected in [ + ("dry-run", StageReuseMode.DRY_RUN), + ("reuse-stage", StageReuseMode.REUSE_STAGE), + ("garbage", StageReuseMode.DRY_RUN), + ]: + os.environ["STAGE_REUSE_MODE"] = value + self.assertEqual(StageReuseMode.from_environ(), expected) + os.environ.pop("STAGE_REUSE_MODE", None) + + +class AvailabilityGateTest(unittest.TestCase): + def test_dry_run_verifies_artifacts_present(self): + # math-libs changed -> compiler-runtime unaffected (candidate). + # Baseline HAS base artifact -> reported as WOULD skip, but applied=(). + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.DRY_RUN, + linux_amdgpu_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + self.assertIn("compiler-runtime", result.candidate_stages) + self.assertIn("compiler-runtime", result.available_stages) + self.assertEqual(result.applied_reuse_stages, ()) # dry-run applies nothing + self.assertEqual(result.baseline_run_id, "123") + joined = "\n".join(result.report_lines) + self.assertIn("available in baseline", joined) + self.assertIn("WOULD be skipped", joined) + + def test_dry_run_unaffected_but_artifacts_missing_rebuilds(self): + # compiler-runtime unaffected, but baseline only has blas (not base). + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.DRY_RUN, + linux_amdgpu_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["blas_lib_generic.tar.zst"])), + ) + self.assertIn("compiler-runtime", result.candidate_stages) + self.assertIn("compiler-runtime", result.unavailable_stages) + self.assertEqual(result.available_stages, ()) + joined = "\n".join(result.report_lines) + self.assertIn("artifacts NOT available", joined) + + def test_no_baseline_found_rebuilds_candidates(self): + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.DRY_RUN, + linux_amdgpu_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(None), + ) + self.assertIn("compiler-runtime", result.candidate_stages) + self.assertEqual(result.available_stages, ()) + self.assertIsNone(result.baseline_run_id) + joined = "\n".join(result.report_lines) + self.assertIn("no baseline run contains artifacts", joined) + + def test_partial_family_availability_rebuilds(self): + # Needs base for a real family + generic; baseline only has the generic + # archive, so the real family's artifact is missing -> rebuild. + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.DRY_RUN, + linux_amdgpu_families=["gfx94X-dcgpu"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + self.assertIn("compiler-runtime", result.unavailable_stages) + self.assertEqual(result.available_stages, ()) + + def test_reuse_stage_applies_only_available_stages(self): + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.REUSE_STAGE, + linux_amdgpu_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + self.assertEqual(result.applied_reuse_stages, ("compiler-runtime",)) + joined = "\n".join(result.report_lines) + self.assertIn("WILL be skipped", joined) + + def test_baseline_lookup_error_is_safe(self): + def boom(required): + raise GitHubAPIError("network down") + + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.REUSE_STAGE, + linux_amdgpu_families=["generic"], + topology=FakeTopology(), + baseline_selector=boom, + ) + self.assertEqual(result.applied_reuse_stages, ()) + joined = "\n".join(result.report_lines) + self.assertIn("baseline lookup failed", joined) + + def test_non_api_selector_error_propagates(self): + def boom(required): + raise ValueError("bad required-artifacts request") + + with self.assertRaises(ValueError): + compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.REUSE_STAGE, + linux_amdgpu_families=["generic"], + topology=FakeTopology(), + baseline_selector=boom, + ) + + +class GuardrailTest(unittest.TestCase): + def test_full_ci_trigger_skips_baseline_query(self): + called = {"n": 0} + + def selector(required): + called["n"] += 1 + return None + + result = compute_auto_stage_reuse( + changed_files=["build_tools/foo.py"], + mode=StageReuseMode.REUSE_STAGE, + topology=FakeTopology(), + baseline_selector=selector, + ) + self.assertTrue(result.full_rebuild_required) + self.assertEqual(result.applied_reuse_stages, ()) + self.assertEqual(called["n"], 0) + + def test_no_diff_is_conservative(self): + result = compute_auto_stage_reuse( + changed_files=None, + mode=StageReuseMode.REUSE_STAGE, + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + self.assertTrue(result.full_rebuild_required) + self.assertEqual(result.applied_reuse_stages, ()) + + def test_step_summary_shows_baseline_and_availability(self): + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.DRY_RUN, + linux_amdgpu_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + summary = srd.render_step_summary(result) + self.assertIn("baseline run checked: `123`", summary) + self.assertIn("available in baseline", summary) + self.assertIn("no build steps were skipped", summary) + + +class DefaultBaselineSelectorTest(unittest.TestCase): + """_default_baseline_selector must fetch real branch history and never pass + an empty ordered_commit_shas window while current_commit_sha is set.""" + + def _run_with_env(self, env, fake_history, fake_select): + import os + + old_env = {k: os.environ.get(k) for k in env} + os.environ.update({k: v for k, v in env.items() if v is not None}) + for k, v in env.items(): + if v is None: + os.environ.pop(k, None) + + import baseline_runs + import github_actions_api + + orig_select = baseline_runs.select_baseline_run + orig_hist = getattr(github_actions_api, "gha_query_recent_branch_commits", None) + captured = {} + + def _capturing_select(**kwargs): + captured.update(kwargs) + return fake_select + + baseline_runs.select_baseline_run = _capturing_select + github_actions_api.gha_query_recent_branch_commits = fake_history + try: + selector = srd._default_baseline_selector(platform="linux") + result = selector([("base", "generic")]) + finally: + baseline_runs.select_baseline_run = orig_select + if orig_hist is not None: + github_actions_api.gha_query_recent_branch_commits = orig_hist + for k, v in old_env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + return captured, result + + def test_history_is_fetched_and_threaded(self): + def fake_history(**kwargs): + return ["sha-current", "sha-old", "sha-older"] + + captured, _ = self._run_with_env( + {"STAGE_REUSE_CURRENT_SHA": "sha-current"}, + fake_history, + fake_select="baseline", + ) + # Real history passed through (NOT an empty list). + self.assertEqual( + captured["ordered_commit_shas"], ["sha-current", "sha-old", "sha-older"] + ) + self.assertEqual(captured["current_commit_sha"], "sha-current") + + def test_empty_history_disables_commit_rule(self): + def fake_history(**kwargs): + return [] + + captured, _ = self._run_with_env( + {"STAGE_REUSE_CURRENT_SHA": "sha-current"}, + fake_history, + fake_select="baseline", + ) + # Disabled rather than enabled-with-empty-window: both None. + self.assertIsNone(captured["current_commit_sha"]) + self.assertIsNone(captured["ordered_commit_shas"]) + + def test_history_fetch_error_disables_commit_rule(self): + def fake_history(**kwargs): + raise GitHubAPIError("api down") + + captured, _ = self._run_with_env( + {"STAGE_REUSE_CURRENT_SHA": "sha-current"}, + fake_history, + fake_select="baseline", + ) + self.assertIsNone(captured["current_commit_sha"]) + self.assertIsNone(captured["ordered_commit_shas"]) + + def test_no_sha_means_no_history_fetch(self): + calls = {"n": 0} + + def fake_history(**kwargs): + calls["n"] = 1 + return ["x"] + + captured, _ = self._run_with_env( + {"STAGE_REUSE_CURRENT_SHA": None}, + fake_history, + fake_select="baseline", + ) + self.assertEqual(calls["n"], 0) + self.assertIsNone(captured["current_commit_sha"]) + self.assertIsNone(captured["ordered_commit_shas"]) + + +class PlatformAwareAvailabilityTest(unittest.TestCase): + """A stage is only reusable when its artifacts exist for EVERY platform. + + Guards the review concern: prebuilt_stages flow to both the Linux and + Windows build configs, so a stage available only in the Linux baseline must + not be reused when Windows is also being built. + """ + + def _selector_factory(self, per_platform): + """Return a factory yielding a per-platform baseline selector.""" + + def factory(platform): + baseline = per_platform.get(platform) + return lambda required: baseline + + return factory + + def test_stage_reused_only_when_available_on_all_platforms(self): + # compiler-runtime unaffected. Linux baseline HAS base; Windows does NOT. + per_platform = { + "linux": _baseline("L1", ["base_lib_generic.tar.zst"]), + "windows": _baseline("L1", ["blas_lib_generic.tar.zst"]), + } + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.REUSE_STAGE, + windows_amdgpu_families=["generic"], + linux_amdgpu_families=["generic"], + topology=FakeTopology(), + baseline_selector_factory=self._selector_factory(per_platform), + ) + # Available on linux but missing on windows -> NOT applied. + self.assertIn("compiler-runtime", result.candidate_stages) + self.assertIn("compiler-runtime", result.unavailable_stages) + self.assertEqual(result.applied_reuse_stages, ()) + self.assertEqual(result.platform_available["linux"], ("compiler-runtime",)) + self.assertEqual(result.platform_available["windows"], ()) + joined = "\n".join(result.report_lines) + self.assertIn("missing on: windows", joined) + + def test_stage_reused_when_present_on_both_platforms(self): + per_platform = { + "linux": _baseline("L1", ["base_lib_generic.tar.zst"]), + "windows": _baseline("L1", ["base_lib_generic.tar.zst"]), + } + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.REUSE_STAGE, + windows_amdgpu_families=["generic"], + linux_amdgpu_families=["generic"], + topology=FakeTopology(), + baseline_selector_factory=self._selector_factory(per_platform), + ) + self.assertEqual(result.applied_reuse_stages, ("compiler-runtime",)) + self.assertIn("compiler-runtime", result.available_stages) + + def test_single_platform_default_is_linux(self): + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.REUSE_STAGE, + linux_amdgpu_families=["generic"], + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + self.assertEqual(result.applied_reuse_stages, ("compiler-runtime",)) + self.assertIn("linux", result.platform_available) + + def test_no_platforms_selected_disables_auto_reuse(self): + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.REUSE_STAGE, + topology=FakeTopology(), + baseline_selector=_selector(_baseline("123", ["base_lib_generic.tar.zst"])), + ) + self.assertTrue(result.full_rebuild_required) + self.assertEqual(result.applied_reuse_stages, ()) + self.assertIn("no build platforms selected", "\n".join(result.report_lines)) + + def test_different_platform_baselines_disable_reuse(self): + """Automatic reuse is disabled when platforms resolve to different baseline runs.""" + per_platform = { + "linux": _baseline("L1", ["base_lib_generic.tar.zst"]), + "windows": _baseline("W1", ["base_lib_generic.tar.zst"]), + } + + result = compute_auto_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + mode=StageReuseMode.REUSE_STAGE, + windows_amdgpu_families=["generic"], + linux_amdgpu_families=["generic"], + topology=FakeTopology(), + baseline_selector_factory=self._selector_factory(per_platform), + ) + + self.assertTrue(result.full_rebuild_required) + self.assertEqual(result.applied_reuse_stages, ()) + self.assertIn( + "automatic reuse resolved different baseline runs per platform", + result.reasons, + ) + + +class PlanStageReuseTest(unittest.TestCase): + """The pure planning step is independent of baseline/reporting.""" + + def test_plan_returns_candidates_without_baseline(self): + plan = srd.plan_stage_reuse( + changed_files=["rocm-libraries/projects/rocBLAS/x.cpp"], + topology=FakeTopology(), + ) + self.assertIn("compiler-runtime", plan.candidate_stages) + self.assertFalse(plan.full_rebuild_required) + + def test_plan_none_changed_files_is_full_rebuild(self): + plan = srd.plan_stage_reuse(changed_files=None, topology=FakeTopology()) + self.assertTrue(plan.full_rebuild_required) + self.assertEqual(plan.candidate_stages, ()) + + +class TargetFamiliesTest(unittest.TestCase): + def test_always_includes_generic(self): + self.assertEqual(srd._target_families((), ()), ("generic",)) + + def test_dedupes_and_appends_generic(self): + fams = srd._target_families(["gfx94x"], ["gfx94x", "gfx120x"]) + self.assertEqual(fams, ("gfx94x", "gfx120x", "generic")) + + +class BuildPlatformsTest(unittest.TestCase): + def test_no_families_is_empty(self): + self.assertEqual(srd._build_platforms((), ()), ()) + + def test_linux_only(self): + self.assertEqual(srd._build_platforms(["gfx94x"], ()), ("linux",)) + + def test_windows_only(self): + self.assertEqual(srd._build_platforms((), ["gfx110x"]), ("windows",)) + + def test_both_platforms(self): + self.assertEqual( + srd._build_platforms(["gfx94x"], ["gfx110x"]), ("linux", "windows") + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/development/README.md b/docs/development/README.md index 22bfab7aa4d..49c5b68388c 100644 --- a/docs/development/README.md +++ b/docs/development/README.md @@ -30,6 +30,7 @@ - [S3 Buckets](s3_buckets.md) - [GitHub Actions Debugging](github_actions_debugging.md) - [CI Behavior Manipulation](ci_behavior_manipulation.md) +- [Stage Reuse](stage_reuse.md) - [CCache Troubleshooting](ccache_troubleshooting.md) - [WSL ROCDXG CI Stage](wsl_rocdxg.md) diff --git a/docs/development/ci_behavior_manipulation.md b/docs/development/ci_behavior_manipulation.md index 1d700d84e0e..da7aa6212c0 100644 --- a/docs/development/ci_behavior_manipulation.md +++ b/docs/development/ci_behavior_manipulation.md @@ -54,7 +54,9 @@ stage configuration. > This feature is under active development and will evolve as > automatic stage selection and baseline run lookup are added. > -> See https://github.com/ROCm/TheRock/issues/3399 for details. +> See https://github.com/ROCm/TheRock/issues/3399 for details and +> [`stage_reuse.md`](stage_reuse.md) for the automatic stage-reuse layer +> that builds on the manual inputs described below. The [Multi-Arch CI](https://github.com/ROCm/TheRock/actions/workflows/multi_arch_ci.yml) workflow supports skipping individual build stages by copying their artifacts diff --git a/docs/development/ci_overview.md b/docs/development/ci_overview.md index 5408a1deae3..8d6e1d06b7b 100644 --- a/docs/development/ci_overview.md +++ b/docs/development/ci_overview.md @@ -167,6 +167,7 @@ See [workflow_outputs.md](workflow_outputs.md) for the S3 layout structure and [ - [workflow_outputs.md](workflow_outputs.md) - CI output directory structure - [github_actions_debugging.md](github_actions_debugging.md) - Debugging GitHub Actions - [ci_behavior_manipulation.md](ci_behavior_manipulation.md) - Controlling CI behavior with labels and inputs +- [stage_reuse.md](stage_reuse.md) - Reusing unaffected build stages from a baseline run - [manifest_diff.md](manifest_diff.md) - Manifest diff report (submodule SHA changes between two commits) ## Getting Help diff --git a/docs/development/stage_reuse.md b/docs/development/stage_reuse.md new file mode 100644 index 00000000000..ac0855224e3 --- /dev/null +++ b/docs/development/stage_reuse.md @@ -0,0 +1,121 @@ +# Stage Reuse + +Stage reuse lets a multi-arch CI run **skip build stages that a change does not +affect** and instead copy those stages' artifacts from a previous, compatible +workflow run. On a change that only touches one component, the unaffected +lower/parallel stages (for example `compiler-runtime`) can be satisfied from a +baseline run instead of rebuilt, cutting build time. + +There are two independent mechanisms: + +- **Manual reuse** (`prebuilt_stages` + `baseline_run_id`) — you explicitly name + the stages to reuse and the run to copy them from. +- **Automatic reuse** (`stage_reuse_mode`) — the setup job computes which stages + are unaffected, verifies their artifacts exist in a healthy baseline run, and + reuses them for you. + +Both feed the same underlying `prebuilt_stages` decision consumed by the build +jobs; automatic reuse simply *adds* to whatever you set manually. + +## How it works + +[`setup_multi_arch.yml`](https://github.com/ROCm/TheRock/.github/workflows/setup_multi_arch.yml) runs +[`configure_multi_arch_ci.py`](https://github.com/ROCm/TheRock/build_tools/github_actions/configure_multi_arch_ci.py), +which calls into +[`stage_reuse_decision.py`](https://github.com/ROCm/TheRock/build_tools/github_actions/stage_reuse_decision.py). +For every stage the change leaves unaffected, two gates must both pass before the +stage is reused: + +1. **Impact gate** (`stage_impact.analyze_stage_impact`) — the changed files do + not affect the stage, so it is a *candidate* for reuse. +1. **Availability gate** (`baseline_runs.select_baseline_run`) — a single + healthy, commit-compatible baseline run actually contains the artifacts that + stage would produce. Availability is verified **independently for every + platform being built** (Linux and/or Windows): a stage is only reused when + its artifacts are present for *all* of those platforms. A stage available in + the Linux baseline but missing from Windows is rebuilt. + +The availability gate reuses the existing baseline-selection logic, so a +candidate run must have healthy `Build` jobs *and* contain all required +artifacts. A run with no artifacts is never +selected. + +### Modes + +`stage_reuse_mode` controls the automatic layer: + +- `dry-run` (default) — compute the analysis and **report** which stages would + be reused (console log + job step summary), but change nothing. Every stage + still builds. Use this to observe decisions. +- `reuse-stage` — actually reuse the eligible stages: they are merged into + `prebuilt_stages`, so the orchestrator copies their artifacts and skips the + build. + +Automatic reuse is **disabled** when no build platform is selected. + +### Inputs + +| Input | Default | Purpose | +| ---------------------------- | --------- | ------------------------------------------------------------------------------------------------------ | +| `stage_reuse_mode` | `dry-run` | `dry-run` (report only) or `reuse-stage` (auto-reuse unaffected stages). | +| `stage_reuse_max_age_hours` | `72` | Reject baseline runs older than this many hours (recency window). | +| `stage_reuse_commit_history` | `50` | Number of recent branch commits to fetch when establishing ancestry for the commit-compatibility rule. | +| `prebuilt_stages` | `""` | Manual, comma-separated stages to reuse (or `all`). Always honored, independent of `stage_reuse_mode`. | +| `baseline_run_id` | `""` | Run ID to copy manually-listed `prebuilt_stages` artifacts from. | + +Example `workflow_call` from a component CI: + +```yaml +jobs: + therock: + uses: ROCm/TheRock/.github/workflows/setup_multi_arch.yml@main + with: + linux_amdgpu_families: "gfx94X" + stage_reuse_mode: "dry-run" + # stage_reuse_mode: "reuse-stage" + # Optional tuning: + # stage_reuse_max_age_hours: "72" + # stage_reuse_commit_history: "50" +``` + +### What "compatible baseline" means + +A baseline run is only used when it is: + +- on the configured baseline branch (default `main`) of the configured workflow + (default `multi_arch_ci.yml`), +- recent enough (`stage_reuse_max_age_hours`), +- commit-compatible — its commit is the same as, or an ancestor of, the current + commit (established from the last `stage_reuse_commit_history` commits), and +- healthy — its `Build` jobs succeeded and it contains every required artifact + for every platform being built. + +If no run satisfies all of these, nothing is reused and every candidate stage is +rebuilt — reuse fails safe toward a full build. + +## Reading the report + +Both modes emit a `Stage reuse analysis` section to the job step summary and +`[STAGE-REUSE]` lines to the console, including: + +- the platforms verified, +- unaffected candidate stages, +- which stages are available in the baseline (per platform), and for + unavailable ones, which platform is missing the artifacts, +- the baseline run used, and +- in `dry-run`, an explicit note that no build steps were skipped. + +## Further features + +- Reuse is scoped to a single build configuration/variant. Threading build + flags through for superrepo builds that vary configuration is tracked in + [#3399](https://github.com/ROCm/TheRock/issues/3399); until then, do not rely + on reuse across builds with differing build flags. + +## Related + +- [ci_overview.md](ci_overview.md) — overall CI pipeline and stages. +- [ci_behavior_manipulation.md](ci_behavior_manipulation.md) — other CI inputs + and labels. +- `build_tools/github_actions/stage_reuse_decision.py` — implementation and the + full `STAGE_REUSE_*` environment contract.