From e77ba12ec1796a640ea2903f6c77f253a11b8b66 Mon Sep 17 00:00:00 2001 From: geomin12 Date: Thu, 5 Jun 2025 14:44:18 -0700 Subject: [PATCH 01/16] Adding fork fix --- .github/workflows/build_linux_packages.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml index fd7c35cdb5c..bc4cd514aa7 100644 --- a/.github/workflows/build_linux_packages.yml +++ b/.github/workflows/build_linux_packages.yml @@ -25,6 +25,12 @@ on: type: boolean extra_cmake_options: type: string + subtree: + type: string + secrets: + monorepo_token: + required: false + # See the details regarding permissions from the link: # https://github.com/aws-actions/configure-aws-credentials?tab=readme-ov-file#oidc @@ -48,19 +54,25 @@ jobs: CCACHE_MAXSIZE: "700M" AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} TEATIME_FORCE_INTERACTIVE: 0 - BUCKET: ${{ github.event.repository.name == 'TheRock' && 'therock-artifacts' || 'therock-artifacts-external' }} steps: - name: "Checking out repository" uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "ROCm/TheRock" + # temporary + ref: "users/geomin12/fix-monorepo-fork" - name: "Checking out repository for rocm-libraries" - if: ${{ github.repository == 'ROCm/rocm-libraries' }} + if: ${{ github.event.repository.name == 'rocm-libraries' }} uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "ROCm/rocm-libraries" - ref: ${{ github.head_ref || github.ref_name }} + ref: refs/pull/${{ github.event.pull_request.number }}/merge + sparse-checkout: | + .github + ${{ inputs.subtree }} + fetch-depth: 0 # Needed for subtree splitting + token: ${{ secrets.monorepo_token }} path: "rocm-libraries" - name: Runner Health Settings From 5f4b85d93b9e6e02c2d0d83476471b8194d14be0 Mon Sep 17 00:00:00 2001 From: geomin12 Date: Thu, 5 Jun 2025 15:37:01 -0700 Subject: [PATCH 02/16] Adding generate token --- .github/workflows/build_linux_packages.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml index bc4cd514aa7..655d5863591 100644 --- a/.github/workflows/build_linux_packages.yml +++ b/.github/workflows/build_linux_packages.yml @@ -27,9 +27,6 @@ on: type: string subtree: type: string - secrets: - monorepo_token: - required: false # See the details regarding permissions from the link: @@ -62,6 +59,15 @@ jobs: # temporary ref: "users/geomin12/fix-monorepo-fork" + - name: Generate a token + if: ${{ github.event.repository.name == 'rocm-libraries' }} + id: generate-token + uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + - name: "Checking out repository for rocm-libraries" if: ${{ github.event.repository.name == 'rocm-libraries' }} uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -72,7 +78,7 @@ jobs: .github ${{ inputs.subtree }} fetch-depth: 0 # Needed for subtree splitting - token: ${{ secrets.monorepo_token }} + token: ${{ steps.generate-token.outputs.token }} path: "rocm-libraries" - name: Runner Health Settings From 36dec848bd209aed7a810b63d7973b9763559643 Mon Sep 17 00:00:00 2001 From: geomin12 Date: Thu, 5 Jun 2025 15:52:36 -0700 Subject: [PATCH 03/16] token input --- .github/workflows/build_linux_packages.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml index 655d5863591..4ef05f499c9 100644 --- a/.github/workflows/build_linux_packages.yml +++ b/.github/workflows/build_linux_packages.yml @@ -27,6 +27,12 @@ on: type: string subtree: type: string + secrets: + rocm_libraries_app_id: + required: false + rocm_libraries_app_private_key: + required: false + # See the details regarding permissions from the link: @@ -59,13 +65,13 @@ jobs: # temporary ref: "users/geomin12/fix-monorepo-fork" - - name: Generate a token + - name: Generate a token for rocm-libraries if: ${{ github.event.repository.name == 'rocm-libraries' }} id: generate-token uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 with: - app-id: ${{ secrets.APP_ID }} - private-key: ${{ secrets.APP_PRIVATE_KEY }} + app-id: ${{ secrets.rocm_libraries_app_id }} + private-key: ${{ secrets.rocm_libraries_app_private_key }} owner: ${{ github.repository_owner }} - name: "Checking out repository for rocm-libraries" From dd3ad6893fc72c50e0bf6c65226d1e202c6c0453 Mon Sep 17 00:00:00 2001 From: geomin12 Date: Thu, 5 Jun 2025 16:05:15 -0700 Subject: [PATCH 04/16] Adding proper github repo --- .github/workflows/build_linux_packages.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml index 4ef05f499c9..7b116f94ea4 100644 --- a/.github/workflows/build_linux_packages.yml +++ b/.github/workflows/build_linux_packages.yml @@ -34,7 +34,6 @@ on: required: false - # See the details regarding permissions from the link: # https://github.com/aws-actions/configure-aws-credentials?tab=readme-ov-file#oidc permissions: @@ -78,7 +77,7 @@ jobs: if: ${{ github.event.repository.name == 'rocm-libraries' }} uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - repository: "ROCm/rocm-libraries" + repository: ${{ github.repository }} ref: refs/pull/${{ github.event.pull_request.number }}/merge sparse-checkout: | .github From 1ce5b42a8fac7afe04e1c3325f8a2f86956828df Mon Sep 17 00:00:00 2001 From: geomin12 Date: Tue, 10 Jun 2025 11:04:21 -0700 Subject: [PATCH 05/16] Adding composite action --- .../actions/checkout_source_linux/action.yml | 37 +++++++++++++++++++ .github/workflows/build_linux_packages.yml | 30 +-------------- 2 files changed, 39 insertions(+), 28 deletions(-) create mode 100644 .github/actions/checkout_source_linux/action.yml diff --git a/.github/actions/checkout_source_linux/action.yml b/.github/actions/checkout_source_linux/action.yml new file mode 100644 index 00000000000..423c53351db --- /dev/null +++ b/.github/actions/checkout_source_linux/action.yml @@ -0,0 +1,37 @@ +name: "Checkout sources for Linux" + +inputs: + SUBTREE: + type: string + +runs: + using: "composite" + steps: + - name: "Checking out repository" + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: "ROCm/TheRock" + # temporary + ref: "users/geomin12/fix-monorepo-fork" + + - name: Generate a token for rocm-libraries + if: ${{ github.event.repository.name == 'rocm-libraries' }} + id: generate-token + uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 + with: + app-id: ${{ secrets.rocm_libraries_app_id }} + private-key: ${{ secrets.rocm_libraries_app_private_key }} + owner: ${{ github.repository_owner }} + + - name: "Checking out repository for rocm-libraries" + if: ${{ github.event.repository.name == 'rocm-libraries' }} + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: ${{ github.repository }} + ref: refs/pull/${{ github.event.pull_request.number }}/merge + sparse-checkout: | + .github + ${{ inputs.subtree }} + fetch-depth: 0 # Needed for subtree splitting + token: ${{ steps.generate-token.outputs.token }} + path: "rocm-libraries" diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml index 7b116f94ea4..e53f38f7c60 100644 --- a/.github/workflows/build_linux_packages.yml +++ b/.github/workflows/build_linux_packages.yml @@ -57,34 +57,8 @@ jobs: AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} TEATIME_FORCE_INTERACTIVE: 0 steps: - - name: "Checking out repository" - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: "ROCm/TheRock" - # temporary - ref: "users/geomin12/fix-monorepo-fork" - - - name: Generate a token for rocm-libraries - if: ${{ github.event.repository.name == 'rocm-libraries' }} - id: generate-token - uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 - with: - app-id: ${{ secrets.rocm_libraries_app_id }} - private-key: ${{ secrets.rocm_libraries_app_private_key }} - owner: ${{ github.repository_owner }} - - - name: "Checking out repository for rocm-libraries" - if: ${{ github.event.repository.name == 'rocm-libraries' }} - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: ${{ github.repository }} - ref: refs/pull/${{ github.event.pull_request.number }}/merge - sparse-checkout: | - .github - ${{ inputs.subtree }} - fetch-depth: 0 # Needed for subtree splitting - token: ${{ steps.generate-token.outputs.token }} - path: "rocm-libraries" + - name: Checking out repository + run: './.github/actions/checkout_source_linux' - name: Runner Health Settings run: | From 6412545287f194b6877114f1279c2d489cd38b2c Mon Sep 17 00:00:00 2001 From: geomin12 Date: Tue, 10 Jun 2025 11:06:12 -0700 Subject: [PATCH 06/16] Clean up --- .github/actions/checkout_source_linux/action.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/actions/checkout_source_linux/action.yml b/.github/actions/checkout_source_linux/action.yml index 423c53351db..f8324a4c5b1 100644 --- a/.github/actions/checkout_source_linux/action.yml +++ b/.github/actions/checkout_source_linux/action.yml @@ -11,8 +11,6 @@ runs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "ROCm/TheRock" - # temporary - ref: "users/geomin12/fix-monorepo-fork" - name: Generate a token for rocm-libraries if: ${{ github.event.repository.name == 'rocm-libraries' }} From 53b66f6c198261789b63747ea2e38cb37daa051f Mon Sep 17 00:00:00 2001 From: geomin12 Date: Tue, 10 Jun 2025 13:16:43 -0700 Subject: [PATCH 07/16] Fixing uses --- .github/workflows/build_linux_packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml index e53f38f7c60..ad53d8200e9 100644 --- a/.github/workflows/build_linux_packages.yml +++ b/.github/workflows/build_linux_packages.yml @@ -58,7 +58,7 @@ jobs: TEATIME_FORCE_INTERACTIVE: 0 steps: - name: Checking out repository - run: './.github/actions/checkout_source_linux' + uses: './.github/actions/checkout_source_linux' - name: Runner Health Settings run: | From 6fbbc6d2517d8b39161c0548f36707de407620b8 Mon Sep 17 00:00:00 2001 From: geomin12 Date: Wed, 11 Jun 2025 16:22:15 -0700 Subject: [PATCH 08/16] Adding composite actions --- .../actions/artifact_upload_action/action.yml | 48 ++++++++ .github/actions/build_linux_action/action.yml | 65 +++++++++++ .../actions/checkout_source_linux/action.yml | 35 ------ .github/workflows/build_linux_packages.yml | 103 +++--------------- .github/workflows/build_windows_packages.yml | 40 +------ 5 files changed, 131 insertions(+), 160 deletions(-) create mode 100644 .github/actions/artifact_upload_action/action.yml create mode 100644 .github/actions/build_linux_action/action.yml delete mode 100644 .github/actions/checkout_source_linux/action.yml diff --git a/.github/actions/artifact_upload_action/action.yml b/.github/actions/artifact_upload_action/action.yml new file mode 100644 index 00000000000..a6dd2fd1dfb --- /dev/null +++ b/.github/actions/artifact_upload_action/action.yml @@ -0,0 +1,48 @@ +name: "Artifact Upload" + +inputs: + build_directory: + type: string + amdgpu_families: + type: string + +runs: + using: "composite" + steps: + - name: Configure AWS Credentials + if: always() + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 + with: + aws-region: us-east-2 + role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts + + - name: Create Logs index Files + if: always() + run: | + python3 build_tools/github_actions/create_log_index.py \ + --build-dir=${{ inputs.build_directory }} \ + --amdgpu-family=${{ inputs.amdgpu_families }} + + - name: Upload artifacts + if: always() + run: | + python build_tools/github_actions/upload_build_artifacts.py \ + --run-id ${{ github.run_id }} \ + --amdgpu-family ${{ inputs.amdgpu_families }} \ + --build-dir ${{ inputs.build_directory }} + + - name: Upload Logs + if: always() + run: | + python3 build_tools/github_actions/upload_build_logs_to_s3.py \ + --build-dir=${{ inputs.build_directory }} \ + --run-id ${{ github.run_id }} \ + --amdgpu-family ${{ inputs.amdgpu_families }} + + - name: Add Links to Job Summary + if: always() + run: | + python build_tools/github_actions/upload_build_summary.py \ + --run-id ${{ github.run_id }} \ + --amdgpu-family ${{ inputs.amdgpu_families }} \ + --build-dir ${{ inputs.build_directory }} diff --git a/.github/actions/build_linux_action/action.yml b/.github/actions/build_linux_action/action.yml new file mode 100644 index 00000000000..e09cfdaf51b --- /dev/null +++ b/.github/actions/build_linux_action/action.yml @@ -0,0 +1,65 @@ +name: "Build Linux Package Action" + +inputs: + package_version: + type: string + amdgpu_families: + type: string + extra_cmake_options: + type: string + +runs: + using: "composite" + steps: + - name: Fetch sources + run: | + ./build_tools/fetch_sources.py --jobs 12 + + - name: Install python deps + run: | + pip install -r requirements.txt + + - name: Configure Projects + run: | + # Generate a new build id. + package_version="${{ inputs.package_version }}" + extra_cmake_options="${{ inputs.extra_cmake_options }}" + echo "Building package ${package_version}" + + # Build. + cmake -B build -GNinja . \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DTHEROCK_AMDGPU_FAMILIES=${{env.AMDGPU_FAMILIES}} \ + -DTHEROCK_PACKAGE_VERSION="${package_version}" \ + -DTHEROCK_VERBOSE=ON \ + -DBUILD_TESTING=ON \ + ${extra_cmake_options} + + - name: Build therock-dist + run: cmake --build build --target therock-dist + + - name: Build therock-archives + run: cmake --build build --target therock-archives + + - name: Test Packaging + if: ${{ github.event.repository.name == 'TheRock' }} + run: | + ctest --test-dir build --output-on-failure + + - name: Report + if: ${{ !cancelled() }} + run: | + echo "Full SDK du:" + echo "------------" + du -h -d 1 build/dist/rocm + echo "Artifact Archives:" + echo "------------------" + ls -lh build/artifacts/*.tar.xz + echo "Artifacts:" + echo "----------" + du -h -d 1 build/artifacts + echo "CCache Stats:" + echo "-------------" + ccache -s + + diff --git a/.github/actions/checkout_source_linux/action.yml b/.github/actions/checkout_source_linux/action.yml deleted file mode 100644 index f8324a4c5b1..00000000000 --- a/.github/actions/checkout_source_linux/action.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: "Checkout sources for Linux" - -inputs: - SUBTREE: - type: string - -runs: - using: "composite" - steps: - - name: "Checking out repository" - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: "ROCm/TheRock" - - - name: Generate a token for rocm-libraries - if: ${{ github.event.repository.name == 'rocm-libraries' }} - id: generate-token - uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6 - with: - app-id: ${{ secrets.rocm_libraries_app_id }} - private-key: ${{ secrets.rocm_libraries_app_private_key }} - owner: ${{ github.repository_owner }} - - - name: "Checking out repository for rocm-libraries" - if: ${{ github.event.repository.name == 'rocm-libraries' }} - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: ${{ github.repository }} - ref: refs/pull/${{ github.event.pull_request.number }}/merge - sparse-checkout: | - .github - ${{ inputs.subtree }} - fetch-depth: 0 # Needed for subtree splitting - token: ${{ steps.generate-token.outputs.token }} - path: "rocm-libraries" diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml index ad53d8200e9..aa2782be12c 100644 --- a/.github/workflows/build_linux_packages.yml +++ b/.github/workflows/build_linux_packages.yml @@ -57,8 +57,8 @@ jobs: AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} TEATIME_FORCE_INTERACTIVE: 0 steps: - - name: Checking out repository - uses: './.github/actions/checkout_source_linux' + - name: Checkout Repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Runner Health Settings run: | @@ -84,98 +84,23 @@ jobs: restore-keys: | linux-build-packages-manylinux-v2- - - name: Fetch sources - run: | - ./build_tools/fetch_sources.py --jobs 12 - - - name: Install python deps - run: | - pip install -r requirements.txt - - - name: Configure Projects - run: | - # Generate a new build id. - package_version="${{ inputs.package_version }}" - extra_cmake_options="${{ inputs.extra_cmake_options }}" - echo "Building package ${package_version}" - - # Build. - cmake -B build -GNinja . \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DTHEROCK_AMDGPU_FAMILIES=${{env.AMDGPU_FAMILIES}} \ - -DTHEROCK_PACKAGE_VERSION="${package_version}" \ - -DTHEROCK_VERBOSE=ON \ - -DBUILD_TESTING=ON \ - ${extra_cmake_options} - - - name: Build therock-dist - run: cmake --build build --target therock-dist - - - name: Build therock-archives - run: cmake --build build --target therock-archives - - - name: Test Packaging - if: ${{ github.event.repository.name == 'TheRock' }} - run: | - ctest --test-dir build --output-on-failure - - - name: Report - if: ${{ !cancelled() }} - run: | - echo "Full SDK du:" - echo "------------" - du -h -d 1 build/dist/rocm - echo "Artifact Archives:" - echo "------------------" - ls -lh build/artifacts/*.tar.xz - echo "Artifacts:" - echo "----------" - du -h -d 1 build/artifacts - echo "CCache Stats:" - echo "-------------" - ccache -s - - - name: Configure AWS Credentials - if: always() - uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 + - name: Build Linux Packages + uses: './.github/actions/build_linux_action' with: - aws-region: us-east-2 - role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts - - - name: Create Logs index Files - if: always() - run: | - python3 build_tools/github_actions/create_log_index.py \ - --build-dir=build \ - --amdgpu-family=${{ env.AMDGPU_FAMILIES }} - - - name: Upload artifacts - if: always() - run: | - python build_tools/github_actions/upload_build_artifacts.py \ - --run-id ${{ github.run_id }} \ - --amdgpu-family ${{ env.AMDGPU_FAMILIES }} \ - --build-dir build/ - - - name: Upload Logs - if: always() - run: | - python3 build_tools/github_actions/upload_build_logs_to_s3.py \ - --build-dir=build \ - --run-id ${{ github.run_id }} \ - --amdgpu-family ${{ env.AMDGPU_FAMILIES }} - - - name: Add Links to Job Summary - if: always() - run: | - python build_tools/github_actions/upload_build_summary.py \ - --run-id ${{ github.run_id }} \ - --amdgpu-family ${{ env.AMDGPU_FAMILIES }} \ - --build-dir build/ + package_version: ${{ inputs.package_version }} + amdgpu_families: ${{ inputs.amdgpu_families }} + extra_cmake_options: ${{ inputs.extra_cmake_options }} + - name: Artifact Upload + uses: './.github/actions/artifact_upload_action' + with: + build_directory: ${{ inputs.package_version }} + amdgpu_families: ${{ inputs.amdgpu_families }} + - name: Save cache uses: actions/cache/save@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 if: always() with: path: ${{ env.CACHE_DIR }} key: linux-build-packages-manylinux-v2-${{ github.sha }} + diff --git a/.github/workflows/build_windows_packages.yml b/.github/workflows/build_windows_packages.yml index d1edbc3c1ad..0b92a7c0d6b 100644 --- a/.github/workflows/build_windows_packages.yml +++ b/.github/workflows/build_windows_packages.yml @@ -189,43 +189,11 @@ jobs: $fsout | % {$_.Used/=1GB;$_.Free/=1GB;$_} | Write-Host get-disk | Select-object @{Name="Size(GB)";Expression={$_.Size/1GB}} | Write-Host - - name: Configure AWS Credentials - if: always() - uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 + - name: Artifact Upload + uses: './.github/actions/artifact_upload_action' with: - aws-region: us-east-2 - role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts - - - name: Create Logs index Files - if: always() - run: | - python3 build_tools/github_actions/create_log_index.py \ - --build-dir=${{ env.BUILD_DIR_BASH }} \ - --amdgpu-family=${{ env.AMDGPU_FAMILIES }} - - - name: Upload artifacts - if: always() - run: | - python build_tools/github_actions/upload_build_artifacts.py \ - --run-id ${{ github.run_id }} \ - --amdgpu-family ${{ env.AMDGPU_FAMILIES }} \ - --build-dir ${{ env.BUILD_DIR_BASH }} - - - name: Upload Logs - if: always() - run: | - python3 build_tools/github_actions/upload_build_logs_to_s3.py \ - --build-dir=${{ env.BUILD_DIR_BASH }} \ - --run-id ${{ github.run_id }} \ - --amdgpu-family ${{ env.AMDGPU_FAMILIES }} - - - name: Add Links to Job Summary - if: always() - run: | - python build_tools/github_actions/upload_build_summary.py \ - --run-id ${{ github.run_id }} \ - --amdgpu-family ${{ env.AMDGPU_FAMILIES }} \ - --build-dir ${{ env.BUILD_DIR_BASH }} + build_directory: ${{ env.BUILD_DIR_BASH }} + amdgpu_families: ${{ inputs.amdgpu_families }} - name: Save cache uses: actions/cache/save@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 From 17455430c75371a4b25b774990d04b0a34e66f03 Mon Sep 17 00:00:00 2001 From: geomin12 Date: Wed, 11 Jun 2025 16:35:43 -0700 Subject: [PATCH 09/16] Adding correct build dir --- .github/actions/build_linux_action/action.yml | 2 -- .github/workflows/build_linux_packages.yml | 5 ++--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/actions/build_linux_action/action.yml b/.github/actions/build_linux_action/action.yml index e09cfdaf51b..52254fcd3f1 100644 --- a/.github/actions/build_linux_action/action.yml +++ b/.github/actions/build_linux_action/action.yml @@ -61,5 +61,3 @@ runs: echo "CCache Stats:" echo "-------------" ccache -s - - diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml index aa2782be12c..0489778154e 100644 --- a/.github/workflows/build_linux_packages.yml +++ b/.github/workflows/build_linux_packages.yml @@ -94,13 +94,12 @@ jobs: - name: Artifact Upload uses: './.github/actions/artifact_upload_action' with: - build_directory: ${{ inputs.package_version }} + build_directory: "build" amdgpu_families: ${{ inputs.amdgpu_families }} - + - name: Save cache uses: actions/cache/save@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 if: always() with: path: ${{ env.CACHE_DIR }} key: linux-build-packages-manylinux-v2-${{ github.sha }} - From af1129769ad08451cbda1a005e29ff2d7394d2ee Mon Sep 17 00:00:00 2001 From: geomin12 Date: Wed, 11 Jun 2025 17:40:50 -0700 Subject: [PATCH 10/16] Adding shell --- .github/actions/artifact_upload_action/action.yml | 5 +++++ .github/actions/build_linux_action/action.yml | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/.github/actions/artifact_upload_action/action.yml b/.github/actions/artifact_upload_action/action.yml index a6dd2fd1dfb..f993696027a 100644 --- a/.github/actions/artifact_upload_action/action.yml +++ b/.github/actions/artifact_upload_action/action.yml @@ -10,6 +10,7 @@ runs: using: "composite" steps: - name: Configure AWS Credentials + shell: bash if: always() uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 with: @@ -17,6 +18,7 @@ runs: role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts - name: Create Logs index Files + shell: bash if: always() run: | python3 build_tools/github_actions/create_log_index.py \ @@ -24,6 +26,7 @@ runs: --amdgpu-family=${{ inputs.amdgpu_families }} - name: Upload artifacts + shell: bash if: always() run: | python build_tools/github_actions/upload_build_artifacts.py \ @@ -32,6 +35,7 @@ runs: --build-dir ${{ inputs.build_directory }} - name: Upload Logs + shell: bash if: always() run: | python3 build_tools/github_actions/upload_build_logs_to_s3.py \ @@ -40,6 +44,7 @@ runs: --amdgpu-family ${{ inputs.amdgpu_families }} - name: Add Links to Job Summary + shell: bash if: always() run: | python build_tools/github_actions/upload_build_summary.py \ diff --git a/.github/actions/build_linux_action/action.yml b/.github/actions/build_linux_action/action.yml index 52254fcd3f1..821a50b967e 100644 --- a/.github/actions/build_linux_action/action.yml +++ b/.github/actions/build_linux_action/action.yml @@ -12,14 +12,17 @@ runs: using: "composite" steps: - name: Fetch sources + shell: bash run: | ./build_tools/fetch_sources.py --jobs 12 - name: Install python deps + shell: bash run: | pip install -r requirements.txt - name: Configure Projects + shell: bash run: | # Generate a new build id. package_version="${{ inputs.package_version }}" @@ -36,17 +39,21 @@ runs: ${extra_cmake_options} - name: Build therock-dist + shell: bash run: cmake --build build --target therock-dist - name: Build therock-archives + shell: bash run: cmake --build build --target therock-archives - name: Test Packaging + shell: bash if: ${{ github.event.repository.name == 'TheRock' }} run: | ctest --test-dir build --output-on-failure - name: Report + shell: bash if: ${{ !cancelled() }} run: | echo "Full SDK du:" From 868ac3d55baf5a1f0bb8fa76c0e4918faea541be Mon Sep 17 00:00:00 2001 From: geomin12 Date: Thu, 12 Jun 2025 08:13:50 -0700 Subject: [PATCH 11/16] fixing action --- .github/actions/artifact_upload_action/action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/actions/artifact_upload_action/action.yml b/.github/actions/artifact_upload_action/action.yml index f993696027a..f8f04292b56 100644 --- a/.github/actions/artifact_upload_action/action.yml +++ b/.github/actions/artifact_upload_action/action.yml @@ -10,7 +10,6 @@ runs: using: "composite" steps: - name: Configure AWS Credentials - shell: bash if: always() uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 with: From 19f168ba05ff381a24cce1f3cafed7f377fa1190 Mon Sep 17 00:00:00 2001 From: geomin12 Date: Fri, 13 Jun 2025 11:03:28 -0700 Subject: [PATCH 12/16] Adding scripts --- .../actions/artifact_upload_action/action.yml | 52 ------------ .github/actions/build_linux_action/action.yml | 70 ---------------- .github/workflows/build_linux_packages.yml | 83 +++++++++++++++---- .../github_actions/build_linux_configure.py | 39 +++++++++ .../github_actions/build_linux_packages.py | 39 +++++++++ 5 files changed, 147 insertions(+), 136 deletions(-) delete mode 100644 .github/actions/artifact_upload_action/action.yml delete mode 100644 .github/actions/build_linux_action/action.yml create mode 100644 build_tools/github_actions/build_linux_configure.py create mode 100644 build_tools/github_actions/build_linux_packages.py diff --git a/.github/actions/artifact_upload_action/action.yml b/.github/actions/artifact_upload_action/action.yml deleted file mode 100644 index f8f04292b56..00000000000 --- a/.github/actions/artifact_upload_action/action.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: "Artifact Upload" - -inputs: - build_directory: - type: string - amdgpu_families: - type: string - -runs: - using: "composite" - steps: - - name: Configure AWS Credentials - if: always() - uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 - with: - aws-region: us-east-2 - role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts - - - name: Create Logs index Files - shell: bash - if: always() - run: | - python3 build_tools/github_actions/create_log_index.py \ - --build-dir=${{ inputs.build_directory }} \ - --amdgpu-family=${{ inputs.amdgpu_families }} - - - name: Upload artifacts - shell: bash - if: always() - run: | - python build_tools/github_actions/upload_build_artifacts.py \ - --run-id ${{ github.run_id }} \ - --amdgpu-family ${{ inputs.amdgpu_families }} \ - --build-dir ${{ inputs.build_directory }} - - - name: Upload Logs - shell: bash - if: always() - run: | - python3 build_tools/github_actions/upload_build_logs_to_s3.py \ - --build-dir=${{ inputs.build_directory }} \ - --run-id ${{ github.run_id }} \ - --amdgpu-family ${{ inputs.amdgpu_families }} - - - name: Add Links to Job Summary - shell: bash - if: always() - run: | - python build_tools/github_actions/upload_build_summary.py \ - --run-id ${{ github.run_id }} \ - --amdgpu-family ${{ inputs.amdgpu_families }} \ - --build-dir ${{ inputs.build_directory }} diff --git a/.github/actions/build_linux_action/action.yml b/.github/actions/build_linux_action/action.yml deleted file mode 100644 index 821a50b967e..00000000000 --- a/.github/actions/build_linux_action/action.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: "Build Linux Package Action" - -inputs: - package_version: - type: string - amdgpu_families: - type: string - extra_cmake_options: - type: string - -runs: - using: "composite" - steps: - - name: Fetch sources - shell: bash - run: | - ./build_tools/fetch_sources.py --jobs 12 - - - name: Install python deps - shell: bash - run: | - pip install -r requirements.txt - - - name: Configure Projects - shell: bash - run: | - # Generate a new build id. - package_version="${{ inputs.package_version }}" - extra_cmake_options="${{ inputs.extra_cmake_options }}" - echo "Building package ${package_version}" - - # Build. - cmake -B build -GNinja . \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DTHEROCK_AMDGPU_FAMILIES=${{env.AMDGPU_FAMILIES}} \ - -DTHEROCK_PACKAGE_VERSION="${package_version}" \ - -DTHEROCK_VERBOSE=ON \ - -DBUILD_TESTING=ON \ - ${extra_cmake_options} - - - name: Build therock-dist - shell: bash - run: cmake --build build --target therock-dist - - - name: Build therock-archives - shell: bash - run: cmake --build build --target therock-archives - - - name: Test Packaging - shell: bash - if: ${{ github.event.repository.name == 'TheRock' }} - run: | - ctest --test-dir build --output-on-failure - - - name: Report - shell: bash - if: ${{ !cancelled() }} - run: | - echo "Full SDK du:" - echo "------------" - du -h -d 1 build/dist/rocm - echo "Artifact Archives:" - echo "------------------" - ls -lh build/artifacts/*.tar.xz - echo "Artifacts:" - echo "----------" - du -h -d 1 build/artifacts - echo "CCache Stats:" - echo "-------------" - ccache -s diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml index 2a6fb9e01e3..6f1c94b634f 100644 --- a/.github/workflows/build_linux_packages.yml +++ b/.github/workflows/build_linux_packages.yml @@ -27,12 +27,6 @@ on: type: string subtree: type: string - secrets: - rocm_libraries_app_id: - required: false - rocm_libraries_app_private_key: - required: false - # See the details regarding permissions from the link: # https://github.com/aws-actions/configure-aws-credentials?tab=readme-ov-file#oidc @@ -84,18 +78,79 @@ jobs: restore-keys: | linux-build-packages-manylinux-v3-${{ inputs.amdgpu_families }}- - - name: Build Linux Packages - uses: './.github/actions/build_linux_action' - with: - package_version: ${{ inputs.package_version }} + - name: Fetch sources + run: | + ./build_tools/fetch_sources.py --jobs 12 + + - name: Install python deps + run: | + pip install -r requirements.txt + + - name: Configure Projects + env: amdgpu_families: ${{ inputs.amdgpu_families }} + package_version: ${{ inputs.package_version }} extra_cmake_options: ${{ inputs.extra_cmake_options }} + run: | + python3 build_tools/github_actions/build_linux_configure.py + + - name: Build Projects + run: | + python3 build_tools/github_actions/build_linux_packages.py - - name: Artifact Upload - uses: './.github/actions/artifact_upload_action' + - name: Report + if: ${{ !cancelled() }} + run: | + echo "Full SDK du:" + echo "------------" + du -h -d 1 build/dist/rocm + echo "Artifact Archives:" + echo "------------------" + ls -lh build/artifacts/*.tar.xz + echo "Artifacts:" + echo "----------" + du -h -d 1 build/artifacts + echo "CCache Stats:" + echo "-------------" + ccache -s + + - name: Configure AWS Credentials + if: always() + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 with: - build_directory: "build" - amdgpu_families: ${{ inputs.amdgpu_families }} + aws-region: us-east-2 + role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts + + - name: Create Logs index Files + if: always() + run: | + python3 build_tools/github_actions/create_log_index.py \ + --build-dir=build \ + --amdgpu-family=${{ inputs.amdgpu_families }} + + - name: Upload artifacts + if: always() + run: | + python build_tools/github_actions/upload_build_artifacts.py \ + --run-id ${{ github.run_id }} \ + --amdgpu-family ${{ inputs.amdgpu_families }} \ + --build-dir build + + - name: Upload Logs + if: always() + run: | + python3 build_tools/github_actions/upload_build_logs_to_s3.py \ + --build-dir=build \ + --run-id ${{ github.run_id }} \ + --amdgpu-family ${{ inputs.amdgpu_families }} + + - name: Add Links to Job Summary + if: always() + run: | + python build_tools/github_actions/upload_build_summary.py \ + --run-id ${{ github.run_id }} \ + --amdgpu-family ${{ inputs.amdgpu_families }} \ + --build-dir build - name: Save cache uses: actions/cache/save@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 diff --git a/build_tools/github_actions/build_linux_configure.py b/build_tools/github_actions/build_linux_configure.py new file mode 100644 index 00000000000..a73aa36d5b8 --- /dev/null +++ b/build_tools/github_actions/build_linux_configure.py @@ -0,0 +1,39 @@ +""" +This script runs the Linux build configuration + +Required environment variables: + - amdgpu_families + - package_version + - extra_cmake_options +""" + +import logging +import os +from pathlib import Path +import subprocess + +logging.basicConfig(level=logging.INFO) +THIS_SCRIPT_DIR = Path(__file__).resolve().parent +THEROCK_DIR = THIS_SCRIPT_DIR.parent.parent + +amdgpu_families = os.getenv("amdgpu_families") +package_version = os.getenv("package_version") +extra_cmake_options = os.getenv("extra_cmake_options") + + +def build_linux_configure(): + logging.info(f"Building package {package_version}") + cmd = f""" + cmake -B build -GNinja . + -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + -DTHEROCK_AMDGPU_FAMILIES={amdgpu_families} + -DTHEROCK_PACKAGE_VERSION="{package_version}" + -DTHEROCK_VERBOSE=ON + -DBUILD_TESTING=ON + {extra_cmake_options} + """ + subprocess.run(cmd, cwd=THEROCK_DIR, check=True) + + +if __name__ == "__main__": + build_linux_configure() diff --git a/build_tools/github_actions/build_linux_packages.py b/build_tools/github_actions/build_linux_packages.py new file mode 100644 index 00000000000..74c6caa89c8 --- /dev/null +++ b/build_tools/github_actions/build_linux_packages.py @@ -0,0 +1,39 @@ +""" +This script runs the Linux build steps +""" + +import logging +import os +from pathlib import Path +import subprocess + +logging.basicConfig(level=logging.INFO) +THIS_SCRIPT_DIR = Path(__file__).resolve().parent +THEROCK_DIR = THIS_SCRIPT_DIR.parent.parent + + +def build_therock_dist(): + logging.info(f"Building therock-dist") + cmd = "cmake --build build --target therock-dist" + subprocess.run(cmd, cwd=THEROCK_DIR, check=True) + + +def build_therock_archives(): + logging.info(f"Building therock-archives") + cmd = "cmake --build build --target therock-archives" + subprocess.run(cmd, cwd=THEROCK_DIR, check=True) + + +def test_therock_packaging(): + github_repository = os.getenv("GITHUB_REPOSITORY", "ROCm/TheRock") + _, repo_name = github_repository.split("/") + if repo_name == "TheRock": + logging.info(f"Running TheRock test packaging") + cmd = "ctest --test-dir build --output-on-failure" + subprocess.run(cmd, cwd=THEROCK_DIR, check=True) + + +if __name__ == "__main__": + build_therock_dist() + build_therock_archives() + test_therock_packaging() From bdeed2e73e17871313bda8127c7a0e46219c6985 Mon Sep 17 00:00:00 2001 From: geomin12 Date: Fri, 13 Jun 2025 11:14:11 -0700 Subject: [PATCH 13/16] fixing --- .../github_actions/build_linux_configure.py | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/build_tools/github_actions/build_linux_configure.py b/build_tools/github_actions/build_linux_configure.py index a73aa36d5b8..0082040bad0 100644 --- a/build_tools/github_actions/build_linux_configure.py +++ b/build_tools/github_actions/build_linux_configure.py @@ -23,15 +23,20 @@ def build_linux_configure(): logging.info(f"Building package {package_version}") - cmd = f""" - cmake -B build -GNinja . - -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - -DTHEROCK_AMDGPU_FAMILIES={amdgpu_families} - -DTHEROCK_PACKAGE_VERSION="{package_version}" - -DTHEROCK_VERBOSE=ON - -DBUILD_TESTING=ON - {extra_cmake_options} - """ + cmd = [ + "cmake", + "-B", + "build", + "-GNinja", + ".", + "-DCMAKE_C_COMPILER_LAUNCHER=ccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache", + f"-DTHEROCK_AMDGPU_FAMILIES={amdgpu_families}", + f"-DTHEROCK_PACKAGE_VERSION='{package_version}'", + "-DTHEROCK_VERBOSE=ON", + "-DBUILD_TESTING=ON", + extra_cmake_options, + ] subprocess.run(cmd, cwd=THEROCK_DIR, check=True) From 612c14a96439269449752178c96552f8cc702485 Mon Sep 17 00:00:00 2001 From: geomin12 Date: Fri, 13 Jun 2025 11:26:23 -0700 Subject: [PATCH 14/16] Updating to remove python boilerplate --- .github/workflows/build_linux_packages.yml | 11 +++++- .../github_actions/build_linux_packages.py | 39 ------------------- 2 files changed, 9 insertions(+), 41 deletions(-) delete mode 100644 build_tools/github_actions/build_linux_packages.py diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml index 6f1c94b634f..f7a8e881b4d 100644 --- a/.github/workflows/build_linux_packages.yml +++ b/.github/workflows/build_linux_packages.yml @@ -94,9 +94,16 @@ jobs: run: | python3 build_tools/github_actions/build_linux_configure.py - - name: Build Projects + - name: Build therock-dist + run: cmake --build build --target therock-dist + + - name: Build therock-archives + run: cmake --build build --target therock-archives + + - name: Test Packaging + if: ${{ github.event.repository.name == 'TheRock' }} run: | - python3 build_tools/github_actions/build_linux_packages.py + ctest --test-dir build --output-on-failure - name: Report if: ${{ !cancelled() }} diff --git a/build_tools/github_actions/build_linux_packages.py b/build_tools/github_actions/build_linux_packages.py deleted file mode 100644 index 74c6caa89c8..00000000000 --- a/build_tools/github_actions/build_linux_packages.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -This script runs the Linux build steps -""" - -import logging -import os -from pathlib import Path -import subprocess - -logging.basicConfig(level=logging.INFO) -THIS_SCRIPT_DIR = Path(__file__).resolve().parent -THEROCK_DIR = THIS_SCRIPT_DIR.parent.parent - - -def build_therock_dist(): - logging.info(f"Building therock-dist") - cmd = "cmake --build build --target therock-dist" - subprocess.run(cmd, cwd=THEROCK_DIR, check=True) - - -def build_therock_archives(): - logging.info(f"Building therock-archives") - cmd = "cmake --build build --target therock-archives" - subprocess.run(cmd, cwd=THEROCK_DIR, check=True) - - -def test_therock_packaging(): - github_repository = os.getenv("GITHUB_REPOSITORY", "ROCm/TheRock") - _, repo_name = github_repository.split("/") - if repo_name == "TheRock": - logging.info(f"Running TheRock test packaging") - cmd = "ctest --test-dir build --output-on-failure" - subprocess.run(cmd, cwd=THEROCK_DIR, check=True) - - -if __name__ == "__main__": - build_therock_dist() - build_therock_archives() - test_therock_packaging() From 394824d1fc6adae47cb4b4163b93b923405753fc Mon Sep 17 00:00:00 2001 From: geomin12 Date: Fri, 13 Jun 2025 12:06:32 -0700 Subject: [PATCH 15/16] Reverting back --- .github/workflows/build_windows_packages.yml | 40 ++++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_windows_packages.yml b/.github/workflows/build_windows_packages.yml index 34c7f1d8376..1d92adda82e 100644 --- a/.github/workflows/build_windows_packages.yml +++ b/.github/workflows/build_windows_packages.yml @@ -189,11 +189,43 @@ jobs: $fsout | % {$_.Used/=1GB;$_.Free/=1GB;$_} | Write-Host get-disk | Select-object @{Name="Size(GB)";Expression={$_.Size/1GB}} | Write-Host - - name: Artifact Upload - uses: './.github/actions/artifact_upload_action' + - name: Configure AWS Credentials + if: always() + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 with: - build_directory: ${{ env.BUILD_DIR_BASH }} - amdgpu_families: ${{ inputs.amdgpu_families }} + aws-region: us-east-2 + role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts + + - name: Create Logs index Files + if: always() + run: | + python3 build_tools/github_actions/create_log_index.py \ + --build-dir=${{ env.BUILD_DIR_BASH }} \ + --amdgpu-family=${{ env.AMDGPU_FAMILIES }} + + - name: Upload artifacts + if: always() + run: | + python build_tools/github_actions/upload_build_artifacts.py \ + --run-id ${{ github.run_id }} \ + --amdgpu-family ${{ env.AMDGPU_FAMILIES }} \ + --build-dir ${{ env.BUILD_DIR_BASH }} + + - name: Upload Logs + if: always() + run: | + python3 build_tools/github_actions/upload_build_logs_to_s3.py \ + --build-dir=${{ env.BUILD_DIR_BASH }} \ + --run-id ${{ github.run_id }} \ + --amdgpu-family ${{ env.AMDGPU_FAMILIES }} + + - name: Add Links to Job Summary + if: always() + run: | + python build_tools/github_actions/upload_build_summary.py \ + --run-id ${{ github.run_id }} \ + --amdgpu-family ${{ env.AMDGPU_FAMILIES }} \ + --build-dir ${{ env.BUILD_DIR_BASH }} - name: Save cache uses: actions/cache/save@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 From acad16d6806d72e70ce75d4811796f1e4f882765 Mon Sep 17 00:00:00 2001 From: geomin12 Date: Fri, 13 Jun 2025 14:29:35 -0700 Subject: [PATCH 16/16] removing unused var --- .github/workflows/build_linux_packages.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml index f7a8e881b4d..a9ce1fa9525 100644 --- a/.github/workflows/build_linux_packages.yml +++ b/.github/workflows/build_linux_packages.yml @@ -25,8 +25,6 @@ on: type: boolean extra_cmake_options: type: string - subtree: - type: string # See the details regarding permissions from the link: # https://github.com/aws-actions/configure-aws-credentials?tab=readme-ov-file#oidc