From fce72303a90e4e1b355a328ef27c552533a62f72 Mon Sep 17 00:00:00 2001 From: geomin12 Date: Wed, 4 Jun 2025 14:38:18 -0700 Subject: [PATCH 1/4] Adding dynamic selecting --- .github/scripts/therock_configure_ci.py | 39 ++++++++++++ .github/scripts/therock_matrix.py | 9 +++ .github/workflows/therock-ci-linux.yml | 34 ++++++++++ .github/workflows/therock-ci.yml | 54 ++++++++-------- .github/workflows/therock-test-packages.yml | 69 +++++++++++++++++++++ 5 files changed, 176 insertions(+), 29 deletions(-) create mode 100644 .github/scripts/therock_configure_ci.py create mode 100644 .github/scripts/therock_matrix.py create mode 100644 .github/workflows/therock-ci-linux.yml create mode 100644 .github/workflows/therock-test-packages.yml diff --git a/.github/scripts/therock_configure_ci.py b/.github/scripts/therock_configure_ci.py new file mode 100644 index 00000000000..fa42158952b --- /dev/null +++ b/.github/scripts/therock_configure_ci.py @@ -0,0 +1,39 @@ +""" +This script determines which build flag and tests to run based on SUBTREES + +Required environment variables: + - SUBTREES +""" + +import json +from therock_matrix import monorepo_map +from typing import Mapping +import os + +SUBTREES = os.getenv("SUBTREES", "") + + +def set_github_output(d: Mapping[str, str]): + """Sets GITHUB_OUTPUT values. + See https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/passing-information-between-jobs + """ + print(f"Setting github output:\n{d}") + step_output_file = os.environ.get("GITHUB_OUTPUT", "") + if not step_output_file: + print("Warning: GITHUB_OUTPUT env var not set, can't set github outputs") + return + with open(step_output_file, "a") as f: + f.writelines(f"{k}={v}" + "\n" for k, v in d.items()) + + +def run(): + subtrees = SUBTREES.split("\n") + projects = [] + for subtree in subtrees: + if subtree in monorepo_map: + projects.append(monorepo_map.get(subtree)) + set_github_output({"projects": json.dumps(projects)}) + + +if __name__ == "__main__": + run() diff --git a/.github/scripts/therock_matrix.py b/.github/scripts/therock_matrix.py new file mode 100644 index 00000000000..37dfb17f5bc --- /dev/null +++ b/.github/scripts/therock_matrix.py @@ -0,0 +1,9 @@ +""" +This dictionary is used to map specific file directory changes to the corresponding build flag and tests +""" +monorepo_map = { + "projects/rocprim": { + "flag": "-DTHEROCK_ENABLE_PRIM=ON -DTHEROCK_ENABLE_ALL=OFF", + "test": "test_rocprim", + } +} diff --git a/.github/workflows/therock-ci-linux.yml b/.github/workflows/therock-ci-linux.yml new file mode 100644 index 00000000000..aa91b84f961 --- /dev/null +++ b/.github/workflows/therock-ci-linux.yml @@ -0,0 +1,34 @@ +name: TheRock CI + +on: + workflow_call: + inputs: + flag: + type: string + project_to_test: + type: string + +permissions: + contents: read + +jobs: + therock-build-linux: + name: Build Linux Packages + permissions: + contents: read + id-token: write + uses: ROCm/TheRock/.github/workflows/build_linux_packages.yml@0730cccf8b98e012d64750f26032bd760360fa45 + with: + amdgpu_families: "gfx94X-dcgpu" + expect_failure: false + extra_cmake_options: "-DTHEROCK_USE_EXTERNAL_ROCM_LIBRARIES=ON -DTHEROCK_ROCM_LIBRARIES_SOURCE_DIR=rocm-libraries ${{ inputs.flag }}" + + therock-test-linux: + name: "Test" + needs: [therock-build-linux] + uses: ./.github/workflows/therock-test-packages.yml + with: + project_to_test: ${{ inputs.project_to_test }} + amdgpu_families: "gfx94X-dcgpu" + test_runs_on: "linux-mi300-1gpu-ossci-rocm" + platform: "linux" diff --git a/.github/workflows/therock-ci.yml b/.github/workflows/therock-ci.yml index ed03f77dcd4..e6da6dbe69a 100644 --- a/.github/workflows/therock-ci.yml +++ b/.github/workflows/therock-ci.yml @@ -27,7 +27,7 @@ jobs: name: "Setup" runs-on: ubuntu-24.04 outputs: - jobs: ${{ steps.jobs.outputs.jobs }} + projects: ${{ steps.projects.outputs.projects }} steps: # Checkout the mono-repo which includes the PR. - name: Checkout rocm-libraries Repository @@ -35,12 +35,13 @@ jobs: with: path: rocm-libraries - - name: Checkout TheRock Repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: "ROCm/TheRock" - path: "TheRock" - ref: a742718b33993332131863885a621fceef6f9b48 + # # will be needed for `patch_monorepo.py` but necessary now + # - name: Checkout TheRock Repository + # uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + # with: + # repository: "ROCm/TheRock" + # path: "TheRock" + # ref: 0730cccf8b98e012d64750f26032bd760360fa45 - name: Generate a token id: generate-token @@ -77,32 +78,27 @@ jobs: --pr "${{ github.event.pull_request.number }}" \ --config "rocm-libraries/.github/repos-config.json" - - name: Determine jobs to run - id: jobs + - name: Determine projects to run + id: projects env: - SUBTREES: ${{ steps.detect.outputs.subtrees }} + # temporary for testing + SUBTREES: "projects/rocprim\nprojects/hipcub" + # SUBTREES: ${{ steps.detect.outputs.subtrees }} run: | - python ./TheRock/build_tools/github_actions/monorepo_configure_ci.py + python rocm-libraries/.github/scripts/therock_configure_ci.py - therock-build-linux: - name: Build Linux Packages - needs: setup - if: ${{ needs.setup.outputs.jobs != '[]' }} + therock-ci-linux: + name: TheRock CI Linux permissions: contents: read id-token: write - uses: ROCm/TheRock/.github/workflows/build_linux_packages.yml@a742718b33993332131863885a621fceef6f9b48 - with: - amdgpu_families: "gfx94X-dcgpu" - expect_failure: false - extra_cmake_options: "-DTHEROCK_USE_EXTERNAL_ROCM_LIBRARIES=ON -DTHEROCK_ROCM_LIBRARIES_SOURCE_DIR=rocm-libraries -DTHEROCK_ENABLE_PRIM=ON -DTHEROCK_ENABLE_ALL=OFF" - - therock-test-linux: - name: "rocPRIM math-lib test" - needs: therock-build-linux - uses: ROCm/TheRock/.github/workflows/test_rocprim.yml@a742718b33993332131863885a621fceef6f9b48 + needs: setup + if: ${{ needs.setup.outputs.projects != '[]' }} + strategy: + fail-fast: false + matrix: + projects: ${{ fromJSON(needs.setup.outputs.projects) }} + uses: ./.github/workflows/therock-ci-linux.yml with: - artifact_run_id: "${{ github.run_id }}" - amdgpu_families: "gfx94X-dcgpu" - test_runs_on: "linux-mi300-1gpu-ossci-rocm" - platform: "linux" + flag: ${{ matrix.projects.flag }} + project_to_test: ${{ matrix.projects.test }} diff --git a/.github/workflows/therock-test-packages.yml b/.github/workflows/therock-test-packages.yml new file mode 100644 index 00000000000..563f8448280 --- /dev/null +++ b/.github/workflows/therock-test-packages.yml @@ -0,0 +1,69 @@ +name: TheRock Test Packages + +on: + workflow_call: + inputs: + project_to_test: + type: string + amdgpu_families: + type: string + test_runs_on: + type: string + platform: + type: string + +permissions: + contents: read + +jobs: + # BLAS tests + test_hipblaslt: + name: "hipBLASLt math-lib" + if: ${{ inputs.project_to_test == 'test_hipblaslt' }} + uses: ROCm/TheRock/.github/workflows/test_hipblaslt.yml@fb0863bc0a191051df965251b3d08875f07e607b + with: + artifact_run_id: "${{ github.run_id }}" + amdgpu_families: ${{ inputs.amdgpu_families }} + test_runs_on: ${{ inputs.test_runs_on }} + platform: ${{ inputs.platform }} + + test_rocblas: + name: "rocBLAS math-lib" + if: ${{ inputs.project_to_test == 'test_rocblas' }} + uses: ROCm/TheRock/.github/workflows/test_rocblas.yml@fb0863bc0a191051df965251b3d08875f07e607b + with: + artifact_run_id: "${{ github.run_id }}" + amdgpu_families: ${{ inputs.amdgpu_families }} + test_runs_on: ${{ inputs.test_runs_on }} + platform: ${{ inputs.platform }} + + # PRIM tests + test_rocprim: + name: "rocPRIM math-lib" + if: ${{ inputs.project_to_test == 'test_rocprim' }} + uses: ROCm/TheRock/.github/workflows/test_rocprim.yml@fb0863bc0a191051df965251b3d08875f07e607b + with: + artifact_run_id: "${{ github.run_id }}" + amdgpu_families: ${{ inputs.amdgpu_families }} + test_runs_on: ${{ inputs.test_runs_on }} + platform: ${{ inputs.platform }} + + test_rocthrust: + name: "rocTHRUST math-lib" + if: ${{ inputs.project_to_test == 'test_rocthrust' }} + uses: ROCm/TheRock/.github/workflows/test_rocthrust.yml@fb0863bc0a191051df965251b3d08875f07e607b + with: + artifact_run_id: "${{ github.run_id }}" + amdgpu_families: ${{ inputs.amdgpu_families }} + test_runs_on: ${{ inputs.test_runs_on }} + platform: ${{ inputs.platform }} + + test_hipcub: + name: "hipCUB math-lib" + if: ${{ inputs.project_to_test == 'test_hipcub' }} + uses: ROCm/TheRock/.github/workflows/test_hipcub.yml@fb0863bc0a191051df965251b3d08875f07e607b + with: + artifact_run_id: "${{ github.run_id }}" + amdgpu_families: ${{ inputs.amdgpu_families }} + test_runs_on: ${{ inputs.test_runs_on }} + platform: ${{ inputs.platform }} From 5ee19171cd9b2e085ca7434ddfe61cf9ae7e83d4 Mon Sep 17 00:00:00 2001 From: geomin12 Date: Wed, 4 Jun 2025 14:43:00 -0700 Subject: [PATCH 2/4] Updating SHAs --- .github/workflows/therock-ci-linux.yml | 2 +- .github/workflows/therock-test-packages.yml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/therock-ci-linux.yml b/.github/workflows/therock-ci-linux.yml index aa91b84f961..8f6117c92fa 100644 --- a/.github/workflows/therock-ci-linux.yml +++ b/.github/workflows/therock-ci-linux.yml @@ -17,7 +17,7 @@ jobs: permissions: contents: read id-token: write - uses: ROCm/TheRock/.github/workflows/build_linux_packages.yml@0730cccf8b98e012d64750f26032bd760360fa45 + uses: ROCm/TheRock/.github/workflows/build_linux_packages.yml@866f163742246445138d7addff8a4010e86bddee with: amdgpu_families: "gfx94X-dcgpu" expect_failure: false diff --git a/.github/workflows/therock-test-packages.yml b/.github/workflows/therock-test-packages.yml index 563f8448280..4ffa2fd273c 100644 --- a/.github/workflows/therock-test-packages.yml +++ b/.github/workflows/therock-test-packages.yml @@ -20,7 +20,7 @@ jobs: test_hipblaslt: name: "hipBLASLt math-lib" if: ${{ inputs.project_to_test == 'test_hipblaslt' }} - uses: ROCm/TheRock/.github/workflows/test_hipblaslt.yml@fb0863bc0a191051df965251b3d08875f07e607b + uses: ROCm/TheRock/.github/workflows/test_hipblaslt.yml@866f163742246445138d7addff8a4010e86bddee with: artifact_run_id: "${{ github.run_id }}" amdgpu_families: ${{ inputs.amdgpu_families }} @@ -30,7 +30,7 @@ jobs: test_rocblas: name: "rocBLAS math-lib" if: ${{ inputs.project_to_test == 'test_rocblas' }} - uses: ROCm/TheRock/.github/workflows/test_rocblas.yml@fb0863bc0a191051df965251b3d08875f07e607b + uses: ROCm/TheRock/.github/workflows/test_rocblas.yml@866f163742246445138d7addff8a4010e86bddee with: artifact_run_id: "${{ github.run_id }}" amdgpu_families: ${{ inputs.amdgpu_families }} @@ -41,7 +41,7 @@ jobs: test_rocprim: name: "rocPRIM math-lib" if: ${{ inputs.project_to_test == 'test_rocprim' }} - uses: ROCm/TheRock/.github/workflows/test_rocprim.yml@fb0863bc0a191051df965251b3d08875f07e607b + uses: ROCm/TheRock/.github/workflows/test_rocprim.yml@866f163742246445138d7addff8a4010e86bddee with: artifact_run_id: "${{ github.run_id }}" amdgpu_families: ${{ inputs.amdgpu_families }} @@ -51,7 +51,7 @@ jobs: test_rocthrust: name: "rocTHRUST math-lib" if: ${{ inputs.project_to_test == 'test_rocthrust' }} - uses: ROCm/TheRock/.github/workflows/test_rocthrust.yml@fb0863bc0a191051df965251b3d08875f07e607b + uses: ROCm/TheRock/.github/workflows/test_rocthrust.yml@866f163742246445138d7addff8a4010e86bddee with: artifact_run_id: "${{ github.run_id }}" amdgpu_families: ${{ inputs.amdgpu_families }} @@ -61,7 +61,7 @@ jobs: test_hipcub: name: "hipCUB math-lib" if: ${{ inputs.project_to_test == 'test_hipcub' }} - uses: ROCm/TheRock/.github/workflows/test_hipcub.yml@fb0863bc0a191051df965251b3d08875f07e607b + uses: ROCm/TheRock/.github/workflows/test_hipcub.yml@866f163742246445138d7addff8a4010e86bddee with: artifact_run_id: "${{ github.run_id }}" amdgpu_families: ${{ inputs.amdgpu_families }} From 1b3d4097336fcb4c2c68937ab076a6416f44946d Mon Sep 17 00:00:00 2001 From: geomin12 Date: Wed, 4 Jun 2025 16:12:09 -0700 Subject: [PATCH 3/4] Removing temporary test --- .github/workflows/therock-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/therock-ci.yml b/.github/workflows/therock-ci.yml index e6da6dbe69a..c07348ccdb0 100644 --- a/.github/workflows/therock-ci.yml +++ b/.github/workflows/therock-ci.yml @@ -81,9 +81,7 @@ jobs: - name: Determine projects to run id: projects env: - # temporary for testing - SUBTREES: "projects/rocprim\nprojects/hipcub" - # SUBTREES: ${{ steps.detect.outputs.subtrees }} + SUBTREES: ${{ steps.detect.outputs.subtrees }} run: | python rocm-libraries/.github/scripts/therock_configure_ci.py From ae4430c030bbb1a43039b1bb1091b08793c2a529 Mon Sep 17 00:00:00 2001 From: geomin12 Date: Thu, 5 Jun 2025 11:50:03 -0700 Subject: [PATCH 4/4] PR comments --- .github/scripts/therock_matrix.py | 4 ++-- .github/workflows/therock-ci-linux.yml | 4 ++-- .github/workflows/therock-ci.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/scripts/therock_matrix.py b/.github/scripts/therock_matrix.py index 37dfb17f5bc..0786d3f71dc 100644 --- a/.github/scripts/therock_matrix.py +++ b/.github/scripts/therock_matrix.py @@ -3,7 +3,7 @@ """ monorepo_map = { "projects/rocprim": { - "flag": "-DTHEROCK_ENABLE_PRIM=ON -DTHEROCK_ENABLE_ALL=OFF", - "test": "test_rocprim", + "cmake_options": "-DTHEROCK_ENABLE_PRIM=ON -DTHEROCK_ENABLE_ALL=OFF", + "project_to_test": "test_rocprim", } } diff --git a/.github/workflows/therock-ci-linux.yml b/.github/workflows/therock-ci-linux.yml index 8f6117c92fa..4db6135f2ba 100644 --- a/.github/workflows/therock-ci-linux.yml +++ b/.github/workflows/therock-ci-linux.yml @@ -3,7 +3,7 @@ name: TheRock CI on: workflow_call: inputs: - flag: + cmake_options: type: string project_to_test: type: string @@ -21,7 +21,7 @@ jobs: with: amdgpu_families: "gfx94X-dcgpu" expect_failure: false - extra_cmake_options: "-DTHEROCK_USE_EXTERNAL_ROCM_LIBRARIES=ON -DTHEROCK_ROCM_LIBRARIES_SOURCE_DIR=rocm-libraries ${{ inputs.flag }}" + extra_cmake_options: "-DTHEROCK_USE_EXTERNAL_ROCM_LIBRARIES=ON -DTHEROCK_ROCM_LIBRARIES_SOURCE_DIR=rocm-libraries ${{ inputs.cmake_options }}" therock-test-linux: name: "Test" diff --git a/.github/workflows/therock-ci.yml b/.github/workflows/therock-ci.yml index c07348ccdb0..29a3c5df447 100644 --- a/.github/workflows/therock-ci.yml +++ b/.github/workflows/therock-ci.yml @@ -98,5 +98,5 @@ jobs: projects: ${{ fromJSON(needs.setup.outputs.projects) }} uses: ./.github/workflows/therock-ci-linux.yml with: - flag: ${{ matrix.projects.flag }} - project_to_test: ${{ matrix.projects.test }} + cmake_options: ${{ matrix.projects.cmake_options }} + project_to_test: ${{ matrix.projects.project_to_test }}