feat: Adding Artifact reuse and stage skip functionality to multi-arch CI

.github/workflows/multi_arch_ci.yml

@@ -42,6 +42,10 @@ on:
         type: string
         default: ""
         description: "Workflow run ID to copy prebuilt stage artifacts from; required when prebuilt_stages is set"
+      stage_reuse_mode:
+        type: string
+        default: "dry-run"
+        description: "dry-run (report only) | skip-stage (actually skip unaffected stages)"
       changed_projects:
         type: string
         default: ""
@@ -82,6 +86,7 @@ jobs:
       windows_test_labels: ${{ inputs.windows_test_labels || '' }}
       prebuilt_stages: ${{ inputs.prebuilt_stages || '' }}
       baseline_run_id: ${{ inputs.baseline_run_id || '' }}
+      stage_reuse_mode: ${{ inputs.stage_reuse_mode || 'dry-run' }}
       build_pytorch: ${{ github.event_name != 'workflow_dispatch' || inputs.build_pytorch }}
       build_jax: false
 

.github/workflows/setup_multi_arch.yml

@@ -43,6 +43,10 @@ on:
         description: "Workflow run ID to copy prebuilt stage artifacts from."
         type: string
         default: ""
+      stage_reuse_mode:
+        description: "dry-run (default) only reports which stages would be skipped; skip-stage actually skips unaffected stages."
+        type: string
+        default: "dry-run"
       build_pytorch:
         description: "Build PyTorch wheels."
         type: boolean
@@ -159,6 +163,10 @@ jobs:
           WINDOWS_TEST_LABELS: ${{ inputs.windows_test_labels }}
           PREBUILT_STAGES: ${{ inputs.prebuilt_stages }}
           BASELINE_RUN_ID: ${{ inputs.baseline_run_id }}
+          STAGE_REUSE_MODE: ${{ inputs.stage_reuse_mode }}
+          STAGE_REUSE_CURRENT_SHA: ${{ steps.checkout.outputs.commit }}
+          STAGE_REUSE_MAX_AGE_HOURS: "72"
+          STAGE_REUSE_COMMIT_HISTORY: "50"
           # Skip path filtering for external repos (git sha won't exist in TheRock)
           SKIP_PATH_FILTERS: ${{ inputs.external_repo != '' && 'true' || '' }}
           CI_CONFIG_PATH: ci-config

build_tools/github_actions/configure_multi_arch_ci.py

@@ -685,14 +685,38 @@ def decide_jobs(
     """Determine which job groups to run, skip, or satisfy with prebuilt files."""
 
     # Build ROCm.
-    # TODO(#3399): Use changed files and build_topology.py to:
-    #   1. set per-stage prebuilt decisions
-    #   2. skip job groups that aren't reachable from the changed files
-    # Parse explicit prebuilt stages from workflow_dispatch input.
+    # Parse explicit prebuilt stages from workflow_dispatch input. These are
+    # the MANUAL inputs and are always honored, unchanged, in every mode.
+
     stage_decisions: dict[str, JobAction] = {}
     if ci_inputs.prebuilt_stages:
         for stage in _parse_prebuilt_stages(ci_inputs.prebuilt_stages):
             stage_decisions[stage] = JobAction.PREBUILT
+
+    # Behind STAGE_REUSE_MODE: in dry-run we only PRINT
+    # which stages WOULD be skipped and apply nothing; in "skip-stage" the
+    # eligible stages are merged into stage_decisions so the orchestrator
+    # skips their builds and copies artifacts instead.
+    from stage_reuse_decision import (
+        StageReuseMode,
+        compute_auto_stage_reuse,
+        render_step_summary,
+    )
+    target_families = _stage_reuse_target_families(ci_inputs)
+    auto = compute_auto_stage_reuse(
+        changed_files=git_context.changed_files,
+        mode=StageReuseMode.from_environ(),
+        target_families=target_families,
+    )
+    for line in auto.report_lines:
+        print(line)
+    try:
+        gha_append_step_summary(render_step_summary(auto))
+    except Exception as exc:  # never fail CI config on a reporting
+        print(f"[STAGE-REUSE] could not write step summary: {exc}")
+    # Only skip-stage mode returns non-empty applied_reuse_stages.
+    for stage in auto.applied_reuse_stages:
+        stage_decisions.setdefault(stage, JobAction.PREBUILT)
     build_rocm = BuildRocmDecision(
         action=JobAction.RUN,
         stage_decisions=stage_decisions,
@@ -741,6 +765,12 @@ def decide_jobs(
 # Step 4: Select Targets
 # ---------------------------------------------------------------------------
 
+def _stage_reuse_target_families(ci_inputs: "CIInputs") -> list[str]:
+    families = list(dict.fromkeys(
+        [*ci_inputs.linux_amdgpu_families, *ci_inputs.windows_amdgpu_families]))
+    if "generic" not in families:
+        families.append("generic")
+    return families
 
 def _validate_family_names(
     names: list[str],