Fork configuration scripts and workflows for multi-arch CI

.github/workflows/multi_arch_ci.yml

@@ -60,72 +60,44 @@ concurrency:
 
 jobs:
   setup:
-    uses: ./.github/workflows/setup.yml
+    uses: ./.github/workflows/setup_multi_arch.yml
     with:
       build_variant: "release"
-      multi_arch: true
-      prebuilt_stages: ${{ inputs.prebuilt_stages }}
 
   linux_build_and_test:
-    name: Linux::${{ matrix.variant.build_variant_label }}
+    name: Linux
     needs: setup
     if: >-
       ${{
-        needs.setup.outputs.linux_variants != '[]' &&
+        needs.setup.outputs.linux_build_config != '' &&
         needs.setup.outputs.enable_build_jobs == 'true'
       }}
-    strategy:
-      fail-fast: false
-      matrix:
-        variant: ${{ fromJSON(needs.setup.outputs.linux_variants) }}
     uses: ./.github/workflows/multi_arch_ci_linux.yml
     secrets: inherit
     with:
-      matrix_per_family_json: ${{ matrix.variant.matrix_per_family_json }}
-      dist_amdgpu_families: ${{ matrix.variant.dist_amdgpu_families }}
-      artifact_group: ${{ matrix.variant.artifact_group }}
-      build_variant_label: ${{ matrix.variant.build_variant_label }}
-      build_variant_cmake_preset: ${{ matrix.variant.build_variant_cmake_preset }}
-      build_variant_suffix: ${{ matrix.variant.build_variant_suffix }}
+      build_config: ${{ needs.setup.outputs.linux_build_config }}
       test_labels: ${{ needs.setup.outputs.linux_test_labels }}
-      expect_failure: ${{ matrix.variant.expect_failure == true }}
-      prebuilt_stages: ${{ inputs.prebuilt_stages }}
-      baseline_run_id: ${{ inputs.baseline_run_id }}
       rocm_package_version: ${{ needs.setup.outputs.rocm_package_version }}
       test_type: ${{ needs.setup.outputs.test_type }}
-      build_pytorch: ${{ matrix.variant.build_pytorch == true }}
     permissions:
       contents: read
       id-token: write
 
   windows_build_and_test:
-    name: Windows::${{ matrix.variant.build_variant_label }}
+    name: Windows
     needs: setup
     if: >-
       ${{
-        needs.setup.outputs.windows_variants != '[]' &&
+        needs.setup.outputs.windows_build_config != '' &&
         needs.setup.outputs.enable_build_jobs == 'true'
       }}
-    strategy:
-      fail-fast: false
-      matrix:
-        variant: ${{ fromJSON(needs.setup.outputs.windows_variants) }}
     uses: ./.github/workflows/multi_arch_ci_windows.yml
     secrets: inherit
     with:
-      matrix_per_family_json: ${{ matrix.variant.matrix_per_family_json }}
-      dist_amdgpu_families: ${{ matrix.variant.dist_amdgpu_families }}
-      artifact_group: ${{ matrix.variant.artifact_group }}
-      build_variant_label: ${{ matrix.variant.build_variant_label }}
-      build_variant_cmake_preset: ${{ matrix.variant.build_variant_cmake_preset }}
-      build_variant_suffix: ${{ matrix.variant.build_variant_suffix }}
+      build_config: ${{ needs.setup.outputs.windows_build_config }}
       test_labels: ${{ needs.setup.outputs.windows_test_labels }}
-      expect_failure: ${{ matrix.variant.expect_failure == true }}
-      prebuilt_stages: ${{ inputs.prebuilt_stages }}
-      baseline_run_id: ${{ inputs.baseline_run_id }}
       rocm_package_version: ${{ needs.setup.outputs.rocm_package_version }}
       test_type: ${{ needs.setup.outputs.test_type }}
-      build_pytorch: ${{ matrix.variant.build_pytorch == true }}
     permissions:
       contents: read
       id-token: write

.github/workflows/multi_arch_ci_linux.yml

@@ -6,47 +6,28 @@ name: Multi-Arch CI - Linux
 on:
   workflow_call:
     inputs:
-      artifact_group:
-        type: string
-      matrix_per_family_json:
-        type: string
-        description: "JSON array of {amdgpu_family, test-runs-on} objects for per-arch stages"
-      dist_amdgpu_families:
-        type: string
-        description: "Semicolon-separated list of all GPU families for dist targets"
-      build_variant_label:
-        type: string
-      build_variant_cmake_preset:
-        type: string
-      build_variant_suffix:
+      build_config:
         type: string
+        description: >-
+          JSON object with build configuration for this platform. Fields:
+          artifact_group, per_family_info, dist_amdgpu_families,
+          build_variant_label, build_variant_cmake_preset,
+          build_variant_suffix, expect_failure, build_pytorch,
+          prebuilt_stages, baseline_run_id.
       test_labels:
         type: string
-      expect_failure:
-        type: boolean
-      prebuilt_stages:
-        type: string
-        default: ""
-        description: "Comma-separated build stages to skip; artifacts are copied from baseline_run_id instead"
-      baseline_run_id:
-        type: string
-        default: ""
-        description: "Workflow run ID to copy prebuilt stage artifacts from"
       rocm_package_version:
         type: string
       test_type:
         type: string
-      build_pytorch:
-        type: boolean
-        default: false
 
 permissions:
   contents: read
 
 jobs:
   copy_prebuilt_stages:
     name: Copy Prebuilt Stages
-    if: ${{ inputs.prebuilt_stages != '' && inputs.baseline_run_id != '' }}
+    if: ${{ fromJSON(inputs.build_config).prebuilt_stages != '' && fromJSON(inputs.build_config).baseline_run_id != '' }}
     runs-on: azure-linux-scale-rocm
     permissions:
       contents: read
@@ -80,9 +61,9 @@ jobs:
           GITHUB_TOKEN: ${{ github.token }}
         run: |
           python build_tools/artifact_manager.py copy \
-            --source-run-id=${{ inputs.baseline_run_id }} \
-            --stage="${{ inputs.prebuilt_stages }}" \
-            --amdgpu-families="${{ inputs.dist_amdgpu_families }}"
+            --source-run-id=${{ fromJSON(inputs.build_config).baseline_run_id }} \
+            --stage="${{ fromJSON(inputs.build_config).prebuilt_stages }}" \
+            --amdgpu-families="${{ fromJSON(inputs.build_config).dist_amdgpu_families }}"
 
   build_multi_arch_stages:
     name: Build Multi-Arch Stages
@@ -91,14 +72,14 @@ jobs:
     uses: ./.github/workflows/multi_arch_build_portable_linux.yml
     secrets: inherit
     with:
-      matrix_per_family_json: ${{ inputs.matrix_per_family_json }}
-      dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }}
-      artifact_group: ${{ inputs.artifact_group }}
-      build_variant_label: ${{ inputs.build_variant_label }}
-      build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }}
-      build_variant_suffix: ${{ inputs.build_variant_suffix }}
-      expect_failure: ${{ inputs.expect_failure }}
-      prebuilt_stages: ${{ inputs.prebuilt_stages }}
+      matrix_per_family_json: ${{ toJSON(fromJSON(inputs.build_config).per_family_info) }}
+      dist_amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }}
+      artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }}
+      build_variant_label: ${{ fromJSON(inputs.build_config).build_variant_label }}
+      build_variant_cmake_preset: ${{ fromJSON(inputs.build_config).build_variant_cmake_preset }}
+      build_variant_suffix: ${{ fromJSON(inputs.build_config).build_variant_suffix }}
+      expect_failure: ${{ fromJSON(inputs.build_config).expect_failure }}
+      prebuilt_stages: ${{ fromJSON(inputs.build_config).prebuilt_stages }}
       rocm_package_version: ${{ inputs.rocm_package_version }}
       test_type: ${{ inputs.test_type }}
     permissions:
@@ -109,21 +90,21 @@ jobs:
     needs: [build_multi_arch_stages]
     name: Validate Artifact Structure
     # If we are expecting a build failure, do not run tests to save machine capacity.
-    if: ${{ !failure() && !cancelled() && inputs.expect_failure == false }}
+    if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).expect_failure == false }}
     uses: ./.github/workflows/test_artifacts_structure.yml
     with:
-      artifact_group: ${{ inputs.artifact_group }}
+      artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }}
       platform: linux
 
   test_artifacts_per_family:
     needs: [copy_prebuilt_stages, build_multi_arch_stages]
     name: Test ${{ matrix.family_info.amdgpu_family }}
     # If we are expecting a build failure, do not run tests to save machine capacity.
-    if: ${{ !failure() && !cancelled() && inputs.expect_failure == false }}
+    if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).expect_failure == false }}
     strategy:
       fail-fast: false
       matrix:
-        family_info: ${{ fromJSON(inputs.matrix_per_family_json) }}
+        family_info: ${{ fromJSON(inputs.build_config).per_family_info }}
         # TODO(TheRock#3288): Re-enable gfx950-dcgpu runners once we get more capacity
         exclude:
           - family_info:
@@ -142,11 +123,11 @@ jobs:
   build_python_packages:
     needs: [build_multi_arch_stages]
     name: Build Python Packages
-    if: ${{ !failure() && !cancelled() && inputs.expect_failure == false }}
+    if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).expect_failure == false }}
     uses: ./.github/workflows/build_portable_linux_python_packages.yml
     with:
-      artifact_group: ${{ inputs.artifact_group }}
-      amdgpu_families: ${{ inputs.dist_amdgpu_families }}
+      artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }}
+      amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }}
       multiarch_index: true
       package_version: ${{ inputs.rocm_package_version }}
     permissions:
@@ -156,11 +137,11 @@ jobs:
   test_python_packages_per_family:
     needs: [build_python_packages]
     name: Test Python ${{ matrix.family_info.amdgpu_family }}
-    if: ${{ !failure() && !cancelled() && inputs.expect_failure == false }}
+    if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).expect_failure == false }}
     strategy:
       fail-fast: false
       matrix:
-        family_info: ${{ fromJSON(inputs.matrix_per_family_json) }}
+        family_info: ${{ fromJSON(inputs.build_config).per_family_info }}
         # TODO(TheRock#3288): Re-enable gfx950-dcgpu runners once we get more capacity
         exclude:
           - family_info:
@@ -180,11 +161,11 @@ jobs:
   build_pytorch_wheels_per_family:
     needs: [build_python_packages]
     name: Build PyTorch | ${{ matrix.family_info.amdgpu_family }}
-    if: ${{ !failure() && !cancelled() && inputs.build_pytorch == true }}
+    if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).build_pytorch == true }}
     strategy:
       fail-fast: false
       matrix:
-        family_info: ${{ fromJSON(inputs.matrix_per_family_json) }}
+        family_info: ${{ fromJSON(inputs.build_config).per_family_info }}
         # TODO(TheRock#3288): Re-enable gfx950-dcgpu runners once we get more capacity
         exclude:
           - family_info:

.github/workflows/multi_arch_ci_windows.yml

@@ -6,47 +6,28 @@ name: Multi-Arch CI - Windows
 on:
   workflow_call:
     inputs:
-      artifact_group:
-        type: string
-      matrix_per_family_json:
-        type: string
-        description: "JSON array of {amdgpu_family, test-runs-on} objects for per-arch stages"
-      dist_amdgpu_families:
-        type: string
-        description: "Semicolon-separated list of all GPU families for dist targets"
-      build_variant_label:
-        type: string
-      build_variant_cmake_preset:
-        type: string
-      build_variant_suffix:
+      build_config:
         type: string
+        description: >-
+          JSON object with build configuration for this platform. Fields:
+          artifact_group, per_family_info, dist_amdgpu_families,
+          build_variant_label, build_variant_cmake_preset,
+          build_variant_suffix, expect_failure, build_pytorch,
+          prebuilt_stages, baseline_run_id.
       test_labels:
         type: string
-      expect_failure:
-        type: boolean
-      prebuilt_stages:
-        type: string
-        default: ""
-        description: "Comma-separated build stages to skip; artifacts are copied from baseline_run_id instead"
-      baseline_run_id:
-        type: string
-        default: ""
-        description: "Workflow run ID to copy prebuilt stage artifacts from"
       rocm_package_version:
         type: string
       test_type:
         type: string
-      build_pytorch:
-        type: boolean
-        default: false
 
 permissions:
   contents: read
 
 jobs:
   copy_prebuilt_stages:
     name: Copy Prebuilt Stages
-    if: ${{ inputs.prebuilt_stages != '' && inputs.baseline_run_id != '' }}
+    if: ${{ fromJSON(inputs.build_config).prebuilt_stages != '' && fromJSON(inputs.build_config).baseline_run_id != '' }}
     # TODO: Consider running on a Linux runner with --platform=windows to
     #   avoid Windows runner setup overhead (setup-python ~51s).
     runs-on: azure-windows-scale-rocm
@@ -85,9 +66,9 @@ jobs:
           GITHUB_TOKEN: ${{ github.token }}
         run: |
           python build_tools/artifact_manager.py copy \
-            --source-run-id=${{ inputs.baseline_run_id }} \
-            --stage="${{ inputs.prebuilt_stages }}" \
-            --amdgpu-families="${{ inputs.dist_amdgpu_families }}"
+            --source-run-id=${{ fromJSON(inputs.build_config).baseline_run_id }} \
+            --stage="${{ fromJSON(inputs.build_config).prebuilt_stages }}" \
+            --amdgpu-families="${{ fromJSON(inputs.build_config).dist_amdgpu_families }}"
 
   build_multi_arch_stages:
     name: Build Multi-Arch Stages
@@ -96,14 +77,14 @@ jobs:
     uses: ./.github/workflows/multi_arch_build_windows.yml
     secrets: inherit
     with:
-      matrix_per_family_json: ${{ inputs.matrix_per_family_json }}
-      dist_amdgpu_families: ${{ inputs.dist_amdgpu_families }}
-      artifact_group: ${{ inputs.artifact_group }}
-      build_variant_label: ${{ inputs.build_variant_label }}
-      build_variant_cmake_preset: ${{ inputs.build_variant_cmake_preset }}
-      build_variant_suffix: ${{ inputs.build_variant_suffix }}
-      expect_failure: ${{ inputs.expect_failure }}
-      prebuilt_stages: ${{ inputs.prebuilt_stages }}
+      matrix_per_family_json: ${{ toJSON(fromJSON(inputs.build_config).per_family_info) }}
+      dist_amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }}
+      artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }}
+      build_variant_label: ${{ fromJSON(inputs.build_config).build_variant_label }}
+      build_variant_cmake_preset: ${{ fromJSON(inputs.build_config).build_variant_cmake_preset }}
+      build_variant_suffix: ${{ fromJSON(inputs.build_config).build_variant_suffix }}
+      expect_failure: ${{ fromJSON(inputs.build_config).expect_failure }}
+      prebuilt_stages: ${{ fromJSON(inputs.build_config).prebuilt_stages }}
       rocm_package_version: ${{ inputs.rocm_package_version }}
       test_type: ${{ inputs.test_type }}
     permissions:
@@ -114,21 +95,21 @@ jobs:
     needs: [build_multi_arch_stages]
     name: Validate Artifact Structure
     # If we are expecting a build failure, do not run tests to save machine capacity.
-    if: ${{ !failure() && !cancelled() && inputs.expect_failure == false }}
+    if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).expect_failure == false }}
     uses: ./.github/workflows/test_artifacts_structure.yml
     with:
-      artifact_group: ${{ inputs.artifact_group }}
+      artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }}
       platform: windows
 
   test_artifacts_per_family:
     needs: [copy_prebuilt_stages, build_multi_arch_stages]
     name: Test ${{ matrix.family_info.amdgpu_family }}
     # If we are expecting a build failure, do not run tests to save machine capacity.
-    if: ${{ !failure() && !cancelled() && inputs.expect_failure == false }}
+    if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).expect_failure == false }}
     strategy:
       fail-fast: false
       matrix:
-        family_info: ${{ fromJSON(inputs.matrix_per_family_json) }}
+        family_info: ${{ fromJSON(inputs.build_config).per_family_info }}
     uses: ./.github/workflows/test_artifacts.yml
     with:
       # Use architecture-specific artifact group for fetching per-arch artifacts
@@ -143,11 +124,11 @@ jobs:
   build_python_packages:
     needs: [build_multi_arch_stages]
     name: Build Python Packages
-    if: ${{ !failure() && !cancelled() && inputs.expect_failure == false }}
+    if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).expect_failure == false }}
     uses: ./.github/workflows/build_windows_python_packages.yml
     with:
-      artifact_group: ${{ inputs.artifact_group }}
-      amdgpu_families: ${{ inputs.dist_amdgpu_families }}
+      artifact_group: ${{ fromJSON(inputs.build_config).artifact_group }}
+      amdgpu_families: ${{ fromJSON(inputs.build_config).dist_amdgpu_families }}
       multiarch_index: true
       package_version: ${{ inputs.rocm_package_version }}
     permissions:
@@ -161,11 +142,11 @@ jobs:
   build_pytorch_wheels_per_family:
     needs: [build_python_packages]
     name: Build PyTorch | ${{ matrix.family_info.amdgpu_family }}
-    if: ${{ !failure() && !cancelled() && inputs.build_pytorch == true }}
+    if: ${{ !failure() && !cancelled() && fromJSON(inputs.build_config).build_pytorch == true }}
     strategy:
       fail-fast: false
       matrix:
-        family_info: ${{ fromJSON(inputs.matrix_per_family_json) }}
+        family_info: ${{ fromJSON(inputs.build_config).per_family_info }}
     uses: ./.github/workflows/build_windows_pytorch_wheels_ci.yml
     with:
       artifact_group: ${{ matrix.family_info.amdgpu_family }}