From 90043d8cb9417c13c34713ad3df1f1b934692219 Mon Sep 17 00:00:00 2001 From: arravikum Date: Mon, 21 Jul 2025 15:25:46 +0000 Subject: [PATCH 01/57] Made changes to split upload S3 with conditions --- .../build_portable_linux_pytorch_wheels.yml | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 8a5d4dec0ce..7d44eb6ee84 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -167,18 +167,6 @@ jobs: run: | python external-builds/pytorch/sanity_check_wheel.py ${{ env.PACKAGE_DIST_DIR }}/ - - name: Upload wheels to S3 - if: ${{ github.repository_owner == 'ROCm' }} - run: | - aws s3 cp ${{ env.PACKAGE_DIST_DIR }}/ s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/ \ - --recursive --exclude "*" --include "*.whl" - - - name: (Re-)Generate Python package release index - if: ${{ github.repository_owner == 'ROCm' }} - run: | - pip install boto3 packaging - python ./build_tools/third_party/s3_management/manage.py ${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }} - generate_target_to_run: name: Generate target_to_run runs-on: ubuntu-24.04 @@ -207,3 +195,23 @@ jobs: python_version: ${{ inputs.python_version }} torch_version: ${{ needs.build_pytorch_wheels.outputs.torch_version }} pytorch_version: ${{ inputs.pytorch_version }} + + upload_pytorch_wheels: + name: Upload PyTorch Wheels to S3 + needs: [build_pytorch_wheels, generate_target_to_run, test_pytorch_wheels] + if: success() && github.repository_owner == 'ROCm' + runs-on: ubuntu-24.04 + env: + PACKAGE_DIST_DIR: ${{ github.workspace }}/output/packages/dist + S3_BUCKET_PY: "therock-${{ inputs.release_type }}-python" + steps: + - name: Upload wheels to S3 + run: | + aws s3 cp ${{ env.PACKAGE_DIST_DIR }}/ s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/ \ + --recursive --exclude "*" --include "*.whl" + + - name: (Re-)Generate Python package release index + run: | + pip install boto3 packaging + python ./build_tools/third_party/s3_management/manage.py ${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }} + From 2830983152d7e4774dec30b70281051638caf38a Mon Sep 17 00:00:00 2001 From: arravikum Date: Mon, 21 Jul 2025 15:28:34 +0000 Subject: [PATCH 02/57] refactor changes --- .../workflows/build_portable_linux_pytorch_wheels.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 7d44eb6ee84..d9c9e778dc2 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -205,6 +205,17 @@ jobs: PACKAGE_DIST_DIR: ${{ github.workspace }}/output/packages/dist S3_BUCKET_PY: "therock-${{ inputs.release_type }}-python" steps: + - name: Configure AWS Credentials + if: always() + uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 + with: + aws-region: us-east-2 + role-to-assume: arn:aws:iam::692859939525:role/therock-${{ inputs.release_type }}-releases + + - name: Sanity Check Wheel + run: | + python external-builds/pytorch/sanity_check_wheel.py ${{ env.PACKAGE_DIST_DIR }}/ + - name: Upload wheels to S3 run: | aws s3 cp ${{ env.PACKAGE_DIST_DIR }}/ s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/ \ From 2db249560db6b18ece0297e2a6c8b1d7c9098ed2 Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 22 Jul 2025 13:23:49 +0000 Subject: [PATCH 03/57] Added copy to staging sub-dir and copy from staging to release steps --- .../build_portable_linux_pytorch_wheels.yml | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index d9c9e778dc2..676f80e6c91 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -167,6 +167,16 @@ jobs: run: | python external-builds/pytorch/sanity_check_wheel.py ${{ env.PACKAGE_DIST_DIR }}/ + - name: Upload wheels to Staging S3 + run: | + aws s3 cp s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/staging/ \ + --recursive --exclude "*" --include "*.whl" + + - name: (Re-)Generate Python package release index + run: | + pip install boto3 packaging + python ./build_tools/third_party/s3_management/manage.py ${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/staging + generate_target_to_run: name: Generate target_to_run runs-on: ubuntu-24.04 @@ -183,6 +193,9 @@ jobs: PLATFORM: "linux" run: python ./build_tools/github_actions/configure_target_run.py + - name: Debug test-runs-on + run: echo "Test runs on: ${{ steps.configure.outputs.test-runs-on }}" + test_pytorch_wheels: name: Test | ${{ inputs.amdgpu_family }} | ${{ needs.generate_target_to_run.outputs.test_runs_on }} if: ${{ needs.generate_target_to_run.outputs.test_runs_on != '' }} @@ -215,10 +228,11 @@ jobs: - name: Sanity Check Wheel run: | python external-builds/pytorch/sanity_check_wheel.py ${{ env.PACKAGE_DIST_DIR }}/ - - - name: Upload wheels to S3 + + - name: Copy wheels from staging sub-dir to release dir S3 run: | - aws s3 cp ${{ env.PACKAGE_DIST_DIR }}/ s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/ \ + aws s3 cp s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/staging/ \ + s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/ \ --recursive --exclude "*" --include "*.whl" - name: (Re-)Generate Python package release index From c7ed8460c11eb6738dd6f5c7bdf4cd59277b6d13 Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 22 Jul 2025 13:27:45 +0000 Subject: [PATCH 04/57] Added copy to staging sub-dir and copy from staging to release steps --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 676f80e6c91..3f67b05ab27 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -192,10 +192,7 @@ jobs: TARGET: ${{ inputs.amdgpu_family }} PLATFORM: "linux" run: python ./build_tools/github_actions/configure_target_run.py - - - name: Debug test-runs-on - run: echo "Test runs on: ${{ steps.configure.outputs.test-runs-on }}" - + test_pytorch_wheels: name: Test | ${{ inputs.amdgpu_family }} | ${{ needs.generate_target_to_run.outputs.test_runs_on }} if: ${{ needs.generate_target_to_run.outputs.test_runs_on != '' }} From 25e4ffe4da04299a0d33d6eda9f53ad5374b351e Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 22 Jul 2025 13:50:21 +0000 Subject: [PATCH 05/57] Updated directories for test --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 3f67b05ab27..dea90b03d65 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -169,13 +169,13 @@ jobs: - name: Upload wheels to Staging S3 run: | - aws s3 cp s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/staging/ \ + aws s3 cp ${{ env.PACKAGE_DIST_DIR }}/ s3://${{ env.S3_BUCKET_PY }}/staging/${{ inputs.amdgpu_family }}/ \ --recursive --exclude "*" --include "*.whl" - name: (Re-)Generate Python package release index run: | pip install boto3 packaging - python ./build_tools/third_party/s3_management/manage.py ${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/staging + python ./build_tools/third_party/s3_management/manage.py staging/${{ inputs.amdgpu_family }} generate_target_to_run: name: Generate target_to_run @@ -192,7 +192,7 @@ jobs: TARGET: ${{ inputs.amdgpu_family }} PLATFORM: "linux" run: python ./build_tools/github_actions/configure_target_run.py - + test_pytorch_wheels: name: Test | ${{ inputs.amdgpu_family }} | ${{ needs.generate_target_to_run.outputs.test_runs_on }} if: ${{ needs.generate_target_to_run.outputs.test_runs_on != '' }} @@ -228,7 +228,7 @@ jobs: - name: Copy wheels from staging sub-dir to release dir S3 run: | - aws s3 cp s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/staging/ \ + aws s3 cp s3://${{ env.S3_BUCKET_PY }}/staging/${{ inputs.amdgpu_family }}/ \ s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/ \ --recursive --exclude "*" --include "*.whl" From 3c794069401409d1b8f632b76e6d6c9ec216871c Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 22 Jul 2025 14:26:58 +0000 Subject: [PATCH 06/57] Adding Staging sub-dir --- .../build_portable_linux_pytorch_wheels.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index dea90b03d65..47684e9cce9 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -17,6 +17,10 @@ on: description: S3 subdirectory, not including the GPU-family required: true type: string + s3_staging_subdir: + description: S3 staging subdirectory, not including the GPU-family + required: true + type: string cloudfront_url: description: CloudFront URL pointing to Python index required: true @@ -55,6 +59,10 @@ on: description: S3 subdirectory, not including the GPU-family type: string default: "v2" + s3_staging_subdir: + description: S3 staging subdirectory, not including the GPU-family + type: string + default: "v2-staging" cloudfront_url: description: CloudFront base URL pointing to Python index type: string @@ -168,14 +176,16 @@ jobs: python external-builds/pytorch/sanity_check_wheel.py ${{ env.PACKAGE_DIST_DIR }}/ - name: Upload wheels to Staging S3 + if: ${{ github.repository_owner == 'ROCm' }} run: | - aws s3 cp ${{ env.PACKAGE_DIST_DIR }}/ s3://${{ env.S3_BUCKET_PY }}/staging/${{ inputs.amdgpu_family }}/ \ + aws s3 cp ${{ env.PACKAGE_DIST_DIR }}/ s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_staging_subdir }}/${{ inputs.amdgpu_family }}/ \ --recursive --exclude "*" --include "*.whl" - name: (Re-)Generate Python package release index + if: ${{ github.repository_owner == 'ROCm' }} run: | pip install boto3 packaging - python ./build_tools/third_party/s3_management/manage.py staging/${{ inputs.amdgpu_family }} + python ./build_tools/third_party/s3_management/manage.py ${{ inputs.s3_staging_subdir }}/${{ inputs.amdgpu_family }} generate_target_to_run: name: Generate target_to_run @@ -208,7 +218,7 @@ jobs: upload_pytorch_wheels: name: Upload PyTorch Wheels to S3 - needs: [build_pytorch_wheels, generate_target_to_run, test_pytorch_wheels] + needs: [build_pytorch_wheels, generate_target_to_run] if: success() && github.repository_owner == 'ROCm' runs-on: ubuntu-24.04 env: @@ -228,7 +238,7 @@ jobs: - name: Copy wheels from staging sub-dir to release dir S3 run: | - aws s3 cp s3://${{ env.S3_BUCKET_PY }}/staging/${{ inputs.amdgpu_family }}/ \ + aws s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_staging_subdir }}/${{ inputs.amdgpu_family }}/ \ s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/ \ --recursive --exclude "*" --include "*.whl" From a22814aaa59e9a38d1157e369c45439af6f5de9b Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 22 Jul 2025 14:30:37 +0000 Subject: [PATCH 07/57] Adding staging prefixes to s3_management script --- build_tools/third_party/s3_management/manage.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build_tools/third_party/s3_management/manage.py b/build_tools/third_party/s3_management/manage.py index 24bb3a6d5cb..8522d4b82c4 100644 --- a/build_tools/third_party/s3_management/manage.py +++ b/build_tools/third_party/s3_management/manage.py @@ -36,10 +36,15 @@ ACCEPTED_FILE_EXTENSIONS = ("whl", "zip", "tar.gz") PREFIXES = [ "v2/gfx110X-dgpu", + "v2-staging/gfx110X-dgpu", "v2/gfx1151", + "v2-staging/gfx1151", "v2/gfx120X-all", + "v2-staging/gfx120X-all", "v2/gfx94X-dcgpu", + "v2-staging/gfx94X-dcgpu" "v2/gfx950-dcgpu", + "v2-staging/gfx950-dcgpu" ] CUSTOM_PREFIX = getenv('CUSTOM_PREFIX') From 4f2445212376ff9a94deb415d3b1d0fccca176cc Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 22 Jul 2025 15:02:49 +0000 Subject: [PATCH 08/57] fixed_typo --- build_tools/third_party/s3_management/manage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/third_party/s3_management/manage.py b/build_tools/third_party/s3_management/manage.py index 8522d4b82c4..6489208a88e 100644 --- a/build_tools/third_party/s3_management/manage.py +++ b/build_tools/third_party/s3_management/manage.py @@ -42,7 +42,7 @@ "v2/gfx120X-all", "v2-staging/gfx120X-all", "v2/gfx94X-dcgpu", - "v2-staging/gfx94X-dcgpu" + "v2-staging/gfx94X-dcgpu", "v2/gfx950-dcgpu", "v2-staging/gfx950-dcgpu" ] From c7bc598a30494d7dbcafded7df6a7209d90af328 Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 22 Jul 2025 16:02:35 +0000 Subject: [PATCH 09/57] Made test execute from staging bucket --- .../workflows/build_portable_linux_pytorch_wheels.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 47684e9cce9..169b62a0833 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -25,6 +25,10 @@ on: description: CloudFront URL pointing to Python index required: true type: string + cloudfront_staging_url: + description: CloudFront URL pointing to Python index + required: true + type: string rocm_version: description: ROCm version to pip install type: string @@ -67,6 +71,10 @@ on: description: CloudFront base URL pointing to Python index type: string default: "https://d25kgig7rdsyks.cloudfront.net/v2" + cloudfront_staging_url: + description: CloudFront base URL pointing to Python index + type: string + default: "https://d25kgig7rdsyks.cloudfront.net/v2-staging" rocm_version: description: ROCm version to pip install type: string @@ -211,7 +219,7 @@ jobs: with: amdgpu_family: ${{ inputs.amdgpu_family }} test_runs_on: ${{ needs.generate_target_to_run.outputs.test_runs_on }} - cloudfront_url: ${{ inputs.cloudfront_url }} + cloudfront_url: ${{ inputs.cloudfront_staging_url }} python_version: ${{ inputs.python_version }} torch_version: ${{ needs.build_pytorch_wheels.outputs.torch_version }} pytorch_version: ${{ inputs.pytorch_version }} From c40e61ed35642ea51e129d8a1ca11434acfbb5c8 Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 22 Jul 2025 16:16:05 +0000 Subject: [PATCH 10/57] Made test execute from staging bucket --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 169b62a0833..91c6af17662 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -240,10 +240,6 @@ jobs: aws-region: us-east-2 role-to-assume: arn:aws:iam::692859939525:role/therock-${{ inputs.release_type }}-releases - - name: Sanity Check Wheel - run: | - python external-builds/pytorch/sanity_check_wheel.py ${{ env.PACKAGE_DIST_DIR }}/ - - name: Copy wheels from staging sub-dir to release dir S3 run: | aws s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_staging_subdir }}/${{ inputs.amdgpu_family }}/ \ From 76a4f93927262b1f9eeee3dd8f5dd374eec95b52 Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 22 Jul 2025 19:04:34 +0000 Subject: [PATCH 11/57] Changed the build index to staging bucket --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 91c6af17662..8fdf62c7e02 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -167,7 +167,7 @@ jobs: build \ --install-rocm \ --pip-cache-dir /tmp/pipcache \ - --index-url "${{ inputs.cloudfront_url }}/${{ inputs.amdgpu_family }}/" \ + --index-url "${{ inputs.cloudfront_staging_url }}/${{ inputs.amdgpu_family }}/" \ --clean \ --output-dir ${{ env.PACKAGE_DIST_DIR }} ${{ env.optional_build_prod_arguments }} python ./build_tools/github_actions/write_torch_versions.py --dist-dir ${{ env.PACKAGE_DIST_DIR }} @@ -242,8 +242,9 @@ jobs: - name: Copy wheels from staging sub-dir to release dir S3 run: | - aws s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_staging_subdir }}/${{ inputs.amdgpu_family }}/ \ - s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/ \ + aws s3 cp \ + s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_staging_subdir }}/${{ inputs.amdgpu_family }}/ \ + s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/ \ --recursive --exclude "*" --include "*.whl" - name: (Re-)Generate Python package release index From c47dad235bdc2d009922c23f06444ed17521c1f2 Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 22 Jul 2025 20:05:33 +0000 Subject: [PATCH 12/57] Changed the build index to dev bucket --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 8fdf62c7e02..036488f7352 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -167,7 +167,7 @@ jobs: build \ --install-rocm \ --pip-cache-dir /tmp/pipcache \ - --index-url "${{ inputs.cloudfront_staging_url }}/${{ inputs.amdgpu_family }}/" \ + --index-url "${{ inputs.cloudfront_url }}/${{ inputs.amdgpu_family }}/" \ --clean \ --output-dir ${{ env.PACKAGE_DIST_DIR }} ${{ env.optional_build_prod_arguments }} python ./build_tools/github_actions/write_torch_versions.py --dist-dir ${{ env.PACKAGE_DIST_DIR }} From 26acce75c2fa14cf3b49da02c7ecbffcad42b33e Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 22 Jul 2025 20:06:40 +0000 Subject: [PATCH 13/57] Added test gating --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 036488f7352..e45b1075bbc 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -226,7 +226,7 @@ jobs: upload_pytorch_wheels: name: Upload PyTorch Wheels to S3 - needs: [build_pytorch_wheels, generate_target_to_run] + needs: [build_pytorch_wheels, generate_target_to_run, test_pytorch_wheels] if: success() && github.repository_owner == 'ROCm' runs-on: ubuntu-24.04 env: From d28412897b8fea7b64013b51a75e978a3789228d Mon Sep 17 00:00:00 2001 From: arravikum Date: Wed, 23 Jul 2025 21:50:54 +0000 Subject: [PATCH 14/57] Added repo checkout in upload_pytorch_wheels step --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index e45b1075bbc..5b538c2cd96 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -233,6 +233,9 @@ jobs: PACKAGE_DIST_DIR: ${{ github.workspace }}/output/packages/dist S3_BUCKET_PY: "therock-${{ inputs.release_type }}-python" steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Configure AWS Credentials if: always() uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 From 55e3938304c49518027b9f367aa0098ee04208be Mon Sep 17 00:00:00 2001 From: arravikum Date: Thu, 24 Jul 2025 00:40:37 +0000 Subject: [PATCH 15/57] Added staging sub-dirs and upload of python packages to staging --- .../release_portable_linux_packages.yml | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_portable_linux_packages.yml b/.github/workflows/release_portable_linux_packages.yml index 78c1e0d8853..9d7aa48e5c5 100644 --- a/.github/workflows/release_portable_linux_packages.yml +++ b/.github/workflows/release_portable_linux_packages.yml @@ -14,6 +14,10 @@ on: description: "Subdirectory to push the Python packages" type: string default: "v2" + s3_staging_subdir: + description: "Staging Subdirectory to push the Python packages" + type: string + default: "v2-staging" # Trigger manually (typically to test the workflow or manually build a release [candidate]) workflow_dispatch: inputs: @@ -27,6 +31,10 @@ on: description: "Subdirectory to push the Python packages" type: string default: "v2" + s3_staging_subdir: + description: "Staging Subdirectory to push the Python packages" + type: string + default: "v2-staging" families: description: "Comma separated list of AMD GPU families, e.g. `gfx94X,gfx103x`" type: string @@ -44,6 +52,7 @@ jobs: runs-on: ubuntu-24.04 env: S3_SUBDIR: ${{ inputs.s3_subdir || 'v2' }} + S3_STAGING_SUBDIR: ${{ inputs.s3_subdir || 'v2-staging' }} release_type: ${{ inputs.release_type || 'nightly' }} outputs: version: ${{ steps.release_information.outputs.version }} @@ -170,7 +179,8 @@ jobs: with: aws-region: us-east-2 role-to-assume: arn:aws:iam::692859939525:role/therock-${{ env.RELEASE_TYPE }}-releases - + + ## TODO: Restrict uploading to the non-staging S3 directory until ROCm sanity checks and all validation tests have successfully passed. - name: Upload Releases to S3 if: ${{ github.repository_owner == 'ROCm' }} run: | @@ -181,6 +191,16 @@ jobs: --include "*.whl" \ --include "*.tar.gz" + - name: Upload Releases to Staging S3 + if: ${{ github.repository_owner == 'ROCm' }} + run: | + aws s3 cp ${{ env.DIST_ARCHIVE }} s3://${{ env.S3_BUCKET_TAR }} + aws s3 cp ${{ env.OUTPUT_DIR }}/packages/dist/ s3://${{ env.S3_BUCKET_PY }}/${{ env.S3_STAGING_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }}/ \ + --recursive --no-follow-symlinks \ + --exclude "*" \ + --include "*.whl" \ + --include "*.tar.gz" + - name: Setup Python if: ${{ github.repository_owner == 'ROCm' }} uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 @@ -193,6 +213,12 @@ jobs: pip install boto3 packaging python ./build_tools/third_party/s3_management/manage.py ${{ env.S3_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }} + - name: (Re-)Generate Python package release index for Staging + if: ${{ github.repository_owner == 'ROCm' }} + run: | + pip install boto3 packaging + python ./build_tools/third_party/s3_management/manage.py ${{ env.S3_STAGING_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }} + - name: Trigger testing nightly tarball if: ${{ env.RELEASE_TYPE == 'nightly' }} uses: benc-uk/workflow-dispatch@e2e5e9a103e331dad343f381a29e654aea3cf8fc # v1.2.4 From 0f31e5caf7dffec5b0acb69fe40b6b8e87cd03be Mon Sep 17 00:00:00 2001 From: arravikum Date: Thu, 24 Jul 2025 01:08:11 +0000 Subject: [PATCH 16/57] Updated description for added staging attributes --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 5b538c2cd96..7ad79cf7752 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -26,7 +26,7 @@ on: required: true type: string cloudfront_staging_url: - description: CloudFront URL pointing to Python index + description: CloudFront Staging URL pointing to Python index required: true type: string rocm_version: @@ -72,7 +72,7 @@ on: type: string default: "https://d25kgig7rdsyks.cloudfront.net/v2" cloudfront_staging_url: - description: CloudFront base URL pointing to Python index + description: CloudFront Staging base URL pointing to Python index type: string default: "https://d25kgig7rdsyks.cloudfront.net/v2-staging" rocm_version: From 9495af62c9ffd2bdbda8e07631e1f95145ec8507 Mon Sep 17 00:00:00 2001 From: arravikum Date: Thu, 24 Jul 2025 01:15:33 +0000 Subject: [PATCH 17/57] Fixed precommit issues --- .github/workflows/release_portable_linux_packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_portable_linux_packages.yml b/.github/workflows/release_portable_linux_packages.yml index 9d7aa48e5c5..ca29081b3f9 100644 --- a/.github/workflows/release_portable_linux_packages.yml +++ b/.github/workflows/release_portable_linux_packages.yml @@ -179,7 +179,7 @@ jobs: with: aws-region: us-east-2 role-to-assume: arn:aws:iam::692859939525:role/therock-${{ env.RELEASE_TYPE }}-releases - + ## TODO: Restrict uploading to the non-staging S3 directory until ROCm sanity checks and all validation tests have successfully passed. - name: Upload Releases to S3 if: ${{ github.repository_owner == 'ROCm' }} From bfb8546387431e73e46944e5bc686f79500b355a Mon Sep 17 00:00:00 2001 From: arravikum Date: Thu, 24 Jul 2025 13:12:23 +0000 Subject: [PATCH 18/57] Added staging sub_dir variable in portable_linux_packages job --- .github/workflows/release_portable_linux_packages.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_portable_linux_packages.yml b/.github/workflows/release_portable_linux_packages.yml index ca29081b3f9..44452d404f7 100644 --- a/.github/workflows/release_portable_linux_packages.yml +++ b/.github/workflows/release_portable_linux_packages.yml @@ -52,7 +52,7 @@ jobs: runs-on: ubuntu-24.04 env: S3_SUBDIR: ${{ inputs.s3_subdir || 'v2' }} - S3_STAGING_SUBDIR: ${{ inputs.s3_subdir || 'v2-staging' }} + S3_STAGING_SUBDIR: ${{ inputs.s3_staging_subdir || 'v2-staging' }} release_type: ${{ inputs.release_type || 'nightly' }} outputs: version: ${{ steps.release_information.outputs.version }} @@ -118,6 +118,7 @@ jobs: S3_BUCKET_TAR: "therock-${{ needs.setup_metadata.outputs.release_type }}-tarball" S3_BUCKET_PY: "therock-${{ needs.setup_metadata.outputs.release_type }}-python" S3_SUBDIR: ${{ inputs.s3_subdir || 'v2' }} + S3_STAGING_SUBDIR: ${{ inputs.s3_staging_subdir || 'v2-staging' }} steps: - name: "Checking out repository" From fd5c912539facbd5ae78687d1dc43af10628ba98 Mon Sep 17 00:00:00 2001 From: arravikum Date: Thu, 24 Jul 2025 20:57:48 +0000 Subject: [PATCH 19/57] Addressed PR comments on #1110 --- .../build_portable_linux_pytorch_wheels.yml | 37 ++++++++++++++----- .../release_portable_linux_packages.yml | 17 ++++----- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 7ad79cf7752..46daf6338ec 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -26,7 +26,7 @@ on: required: true type: string cloudfront_staging_url: - description: CloudFront Staging URL pointing to Python index + description: CloudFront base URL pointing to staging Python index required: true type: string rocm_version: @@ -72,7 +72,7 @@ on: type: string default: "https://d25kgig7rdsyks.cloudfront.net/v2" cloudfront_staging_url: - description: CloudFront Staging base URL pointing to Python index + description: CloudFront base URL pointing to staging Python index type: string default: "https://d25kgig7rdsyks.cloudfront.net/v2-staging" rocm_version: @@ -106,6 +106,7 @@ jobs: optional_build_prod_arguments: "" outputs: torch_version: ${{ steps.build-pytorch-wheels.outputs.torch_version }} + cp_version: ${{ steps.select-python.outputs.cp_version }} steps: - name: Checkout uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -116,9 +117,11 @@ jobs: git config --global user.email "therockbot@amd.com" - name: Select Python version + id: select-python run: | - python build_tools/github_actions/python_to_cp_version.py \ - --python-version ${{ inputs.python_version }} + cp_version=$(python build_tools/github_actions/python_to_cp_version.py \ + --python-version ${{ inputs.python_version }}) + echo "cp_version=$cp_version" >> "$GITHUB_OUTPUT" - name: Add selected Python version to PATH run: | @@ -183,7 +186,7 @@ jobs: run: | python external-builds/pytorch/sanity_check_wheel.py ${{ env.PACKAGE_DIST_DIR }}/ - - name: Upload wheels to Staging S3 + - name: Upload wheels to staging S3 if: ${{ github.repository_owner == 'ROCm' }} run: | aws s3 cp ${{ env.PACKAGE_DIST_DIR }}/ s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_staging_subdir }}/${{ inputs.amdgpu_family }}/ \ @@ -232,6 +235,8 @@ jobs: env: PACKAGE_DIST_DIR: ${{ github.workspace }}/output/packages/dist S3_BUCKET_PY: "therock-${{ inputs.release_type }}-python" + CP_VERSION: ${{ needs.build_pytorch_wheels.outputs.cp_version }} + steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -243,12 +248,24 @@ jobs: aws-region: us-east-2 role-to-assume: arn:aws:iam::692859939525:role/therock-${{ inputs.release_type }}-releases - - name: Copy wheels from staging sub-dir to release dir S3 + - name: Copy only tested PyTorch wheels from staging to release S3 run: | - aws s3 cp \ - s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_staging_subdir }}/${{ inputs.amdgpu_family }}/ \ - s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/ \ - --recursive --exclude "*" --include "*.whl" + torch_version="${{ needs.build_pytorch_wheels.outputs.torch_version }}" + cp_version="${CP_VERSION}" + echo "Torch version: $torch_version" + # Extract the ROCm suffix from the torch version (everything after '+') + rocm_suffix=$(echo "$torch_version" | cut -d '+' -f 2) + echo "ROCm suffix: $rocm_suffix" + + for package in torch torchaudio torchvision; do + echo "Copying matching files for $package with +$rocm_suffix..." + aws s3 cp \ + s3://${S3_BUCKET_PY}/${{ inputs.s3_staging_subdir }}/${{ inputs.amdgpu_family }}/ \ + s3://${S3_BUCKET_PY}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/ \ + --recursive \ + --exclude "*" \ + --include "${package}-*+${rocm_suffix}-${cp_version}*.whl" + done - name: (Re-)Generate Python package release index run: | diff --git a/.github/workflows/release_portable_linux_packages.yml b/.github/workflows/release_portable_linux_packages.yml index 44452d404f7..a032459e879 100644 --- a/.github/workflows/release_portable_linux_packages.yml +++ b/.github/workflows/release_portable_linux_packages.yml @@ -15,7 +15,7 @@ on: type: string default: "v2" s3_staging_subdir: - description: "Staging Subdirectory to push the Python packages" + description: "Staging subdirectory to push the Python packages" type: string default: "v2-staging" # Trigger manually (typically to test the workflow or manually build a release [candidate]) @@ -32,7 +32,7 @@ on: type: string default: "v2" s3_staging_subdir: - description: "Staging Subdirectory to push the Python packages" + description: "Staging subdirectory to push the Python packages" type: string default: "v2-staging" families: @@ -181,22 +181,21 @@ jobs: aws-region: us-east-2 role-to-assume: arn:aws:iam::692859939525:role/therock-${{ env.RELEASE_TYPE }}-releases - ## TODO: Restrict uploading to the non-staging S3 directory until ROCm sanity checks and all validation tests have successfully passed. - - name: Upload Releases to S3 + - name: Upload Releases to staging S3 if: ${{ github.repository_owner == 'ROCm' }} run: | - aws s3 cp ${{ env.DIST_ARCHIVE }} s3://${{ env.S3_BUCKET_TAR }} - aws s3 cp ${{ env.OUTPUT_DIR }}/packages/dist/ s3://${{ env.S3_BUCKET_PY }}/${{ env.S3_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }}/ \ + aws s3 cp ${{ env.OUTPUT_DIR }}/packages/dist/ s3://${{ env.S3_BUCKET_PY }}/${{ env.S3_STAGING_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }}/ \ --recursive --no-follow-symlinks \ --exclude "*" \ --include "*.whl" \ --include "*.tar.gz" - - name: Upload Releases to Staging S3 + ## TODO: Restrict uploading to the non-staging S3 directory until ROCm sanity checks and all validation tests have successfully passed. + - name: Upload Releases to S3 if: ${{ github.repository_owner == 'ROCm' }} run: | aws s3 cp ${{ env.DIST_ARCHIVE }} s3://${{ env.S3_BUCKET_TAR }} - aws s3 cp ${{ env.OUTPUT_DIR }}/packages/dist/ s3://${{ env.S3_BUCKET_PY }}/${{ env.S3_STAGING_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }}/ \ + aws s3 cp ${{ env.OUTPUT_DIR }}/packages/dist/ s3://${{ env.S3_BUCKET_PY }}/${{ env.S3_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }}/ \ --recursive --no-follow-symlinks \ --exclude "*" \ --include "*.whl" \ @@ -214,7 +213,7 @@ jobs: pip install boto3 packaging python ./build_tools/third_party/s3_management/manage.py ${{ env.S3_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }} - - name: (Re-)Generate Python package release index for Staging + - name: (Re-)Generate Python package release index for staging if: ${{ github.repository_owner == 'ROCm' }} run: | pip install boto3 packaging From c9d86fa097c0cbde1616f417d7c80f90e47bbaf7 Mon Sep 17 00:00:00 2001 From: arravikum Date: Thu, 24 Jul 2025 21:07:54 +0000 Subject: [PATCH 20/57] Fixed ordering of Staging index creation based on PR comments --- .../workflows/release_portable_linux_packages.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release_portable_linux_packages.yml b/.github/workflows/release_portable_linux_packages.yml index a032459e879..afef95660d8 100644 --- a/.github/workflows/release_portable_linux_packages.yml +++ b/.github/workflows/release_portable_linux_packages.yml @@ -190,6 +190,12 @@ jobs: --include "*.whl" \ --include "*.tar.gz" + - name: (Re-)Generate Python package release index for staging + if: ${{ github.repository_owner == 'ROCm' }} + run: | + pip install boto3 packaging + python ./build_tools/third_party/s3_management/manage.py ${{ env.S3_STAGING_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }} + ## TODO: Restrict uploading to the non-staging S3 directory until ROCm sanity checks and all validation tests have successfully passed. - name: Upload Releases to S3 if: ${{ github.repository_owner == 'ROCm' }} @@ -206,19 +212,13 @@ jobs: uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: 3.12 - + - name: (Re-)Generate Python package release index if: ${{ github.repository_owner == 'ROCm' }} run: | pip install boto3 packaging python ./build_tools/third_party/s3_management/manage.py ${{ env.S3_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }} - - name: (Re-)Generate Python package release index for staging - if: ${{ github.repository_owner == 'ROCm' }} - run: | - pip install boto3 packaging - python ./build_tools/third_party/s3_management/manage.py ${{ env.S3_STAGING_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }} - - name: Trigger testing nightly tarball if: ${{ env.RELEASE_TYPE == 'nightly' }} uses: benc-uk/workflow-dispatch@e2e5e9a103e331dad343f381a29e654aea3cf8fc # v1.2.4 From 81654c3b4e796eccfe575641eea5a0cfacf23d4c Mon Sep 17 00:00:00 2001 From: arravikum Date: Thu, 24 Jul 2025 21:09:10 +0000 Subject: [PATCH 21/57] Fixed pre-commit failures --- .github/workflows/release_portable_linux_packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_portable_linux_packages.yml b/.github/workflows/release_portable_linux_packages.yml index afef95660d8..da194854fcc 100644 --- a/.github/workflows/release_portable_linux_packages.yml +++ b/.github/workflows/release_portable_linux_packages.yml @@ -212,7 +212,7 @@ jobs: uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: 3.12 - + - name: (Re-)Generate Python package release index if: ${{ github.repository_owner == 'ROCm' }} run: | From d700c996235b8afd3b93fb477f525e6bfc133dd0 Mon Sep 17 00:00:00 2001 From: arravikum Date: Thu, 24 Jul 2025 22:50:34 +0000 Subject: [PATCH 22/57] Added PR comments changes --- .../workflows/build_portable_linux_pytorch_wheels.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 46daf6338ec..75df39a19c2 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -233,9 +233,9 @@ jobs: if: success() && github.repository_owner == 'ROCm' runs-on: ubuntu-24.04 env: - PACKAGE_DIST_DIR: ${{ github.workspace }}/output/packages/dist S3_BUCKET_PY: "therock-${{ inputs.release_type }}-python" CP_VERSION: ${{ needs.build_pytorch_wheels.outputs.cp_version }} + TORCH_VERSION: "${{ needs.build_pytorch_wheels.outputs.torch_version }}" steps: - name: Checkout @@ -250,11 +250,9 @@ jobs: - name: Copy only tested PyTorch wheels from staging to release S3 run: | - torch_version="${{ needs.build_pytorch_wheels.outputs.torch_version }}" - cp_version="${CP_VERSION}" - echo "Torch version: $torch_version" + echo "Torch version: $TORCH_VERSION" # Extract the ROCm suffix from the torch version (everything after '+') - rocm_suffix=$(echo "$torch_version" | cut -d '+' -f 2) + rocm_suffix=$(echo "$TORCH_VERSION" | cut -d '+' -f 2) echo "ROCm suffix: $rocm_suffix" for package in torch torchaudio torchvision; do @@ -264,7 +262,7 @@ jobs: s3://${S3_BUCKET_PY}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/ \ --recursive \ --exclude "*" \ - --include "${package}-*+${rocm_suffix}-${cp_version}*.whl" + --include "${package}-*+${rocm_suffix}-${CP_VERSION}-linux_x86_64.whl" done - name: (Re-)Generate Python package release index From 731d1d1ce3cd878062b729196ae004623e7fbc76 Mon Sep 17 00:00:00 2001 From: arravikum Date: Fri, 25 Jul 2025 01:45:49 +0000 Subject: [PATCH 23/57] Added torchvision and torchaudio versions in write_torch_version.py and added specific tested Torch version aws cp --- .../build_portable_linux_pytorch_wheels.yml | 27 +++++++++---------- .../github_actions/write_torch_versions.py | 3 +-- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 75df39a19c2..432b2b5e3ae 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -236,6 +236,8 @@ jobs: S3_BUCKET_PY: "therock-${{ inputs.release_type }}-python" CP_VERSION: ${{ needs.build_pytorch_wheels.outputs.cp_version }} TORCH_VERSION: "${{ needs.build_pytorch_wheels.outputs.torch_version }}" + TORCHAUDIO_VERSION: "${{ needs.build_pytorch_wheels.outputs.torchaudio_version }}" + TORCHVIDEO_VERSION: "${{ needs.build_pytorch_wheels.outputs.torchvideo_version }}" steps: - name: Checkout @@ -248,22 +250,17 @@ jobs: aws-region: us-east-2 role-to-assume: arn:aws:iam::692859939525:role/therock-${{ inputs.release_type }}-releases - - name: Copy only tested PyTorch wheels from staging to release S3 + - name: Copy tested PyTorch wheels from staging to release S3 run: | - echo "Torch version: $TORCH_VERSION" - # Extract the ROCm suffix from the torch version (everything after '+') - rocm_suffix=$(echo "$TORCH_VERSION" | cut -d '+' -f 2) - echo "ROCm suffix: $rocm_suffix" - - for package in torch torchaudio torchvision; do - echo "Copying matching files for $package with +$rocm_suffix..." - aws s3 cp \ - s3://${S3_BUCKET_PY}/${{ inputs.s3_staging_subdir }}/${{ inputs.amdgpu_family }}/ \ - s3://${S3_BUCKET_PY}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/ \ - --recursive \ - --exclude "*" \ - --include "${package}-*+${rocm_suffix}-${CP_VERSION}-linux_x86_64.whl" - done + echo "Copying exact tested wheels to release S3 bucket..." + aws s3 cp \ + s3://${S3_BUCKET_PY}/${{ inputs.s3_staging_subdir }}/${{ inputs.amdgpu_family }}/ \ + s3://${S3_BUCKET_PY}/${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }}/ \ + --recursive \ + --exclude "*" \ + --include "torch-${TORCH_VERSION}-${CP_VERSION}-linux_x86_64.whl" \ + --include "torchaudio-${TORCHAUDIO_VERSION}-${CP_VERSION}-linux_x86_64.whl" \ + --include "torchvideo-${TORCHVIDEO_VERSION}-${CP_VERSION}-linux_x86_64.whl" - name: (Re-)Generate Python package release index run: | diff --git a/build_tools/github_actions/write_torch_versions.py b/build_tools/github_actions/write_torch_versions.py index 560929ef88d..36766b07a0e 100644 --- a/build_tools/github_actions/write_torch_versions.py +++ b/build_tools/github_actions/write_torch_versions.py @@ -11,10 +11,10 @@ import os import glob import platform - from github_actions_utils import * +<<<<<<< HEAD def _log(*args, **kwargs): print(*args, **kwargs) sys.stdout.flush() @@ -108,6 +108,5 @@ def main(argv: list[str]): _log("") gha_set_output(all_versions) - if __name__ == "__main__": main(sys.argv[1:]) From 6a62bf452c8135942e899ef39ff01c80bd518995 Mon Sep 17 00:00:00 2001 From: arravikum Date: Fri, 25 Jul 2025 01:54:52 +0000 Subject: [PATCH 24/57] Added ordering of non-staging and staging subdirs in manage.py --- build_tools/third_party/s3_management/manage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build_tools/third_party/s3_management/manage.py b/build_tools/third_party/s3_management/manage.py index 6489208a88e..f6f7dc2f745 100644 --- a/build_tools/third_party/s3_management/manage.py +++ b/build_tools/third_party/s3_management/manage.py @@ -36,14 +36,14 @@ ACCEPTED_FILE_EXTENSIONS = ("whl", "zip", "tar.gz") PREFIXES = [ "v2/gfx110X-dgpu", - "v2-staging/gfx110X-dgpu", "v2/gfx1151", - "v2-staging/gfx1151", "v2/gfx120X-all", - "v2-staging/gfx120X-all", "v2/gfx94X-dcgpu", - "v2-staging/gfx94X-dcgpu", "v2/gfx950-dcgpu", + "v2-staging/gfx110X-dgpu", + "v2-staging/gfx1151", + "v2-staging/gfx120X-all", + "v2-staging/gfx94X-dcgpu", "v2-staging/gfx950-dcgpu" ] From 3d114d448235065cc2a72661b104b2b1fa815fce Mon Sep 17 00:00:00 2001 From: arravikum Date: Fri, 25 Jul 2025 03:31:29 +0000 Subject: [PATCH 25/57] Explicit usage of bash as shell --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 432b2b5e3ae..da210aabcb5 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -118,6 +118,7 @@ jobs: - name: Select Python version id: select-python + shell: bash run: | cp_version=$(python build_tools/github_actions/python_to_cp_version.py \ --python-version ${{ inputs.python_version }}) From 7b0dcb95102cff3bbd05c6a2920c85fcfe0e7af4 Mon Sep 17 00:00:00 2001 From: arravikum Date: Fri, 25 Jul 2025 09:53:43 +0000 Subject: [PATCH 26/57] Fixed typo in torchvideo to torchvision, split up cp_version output step --- .../build_portable_linux_pytorch_wheels.yml | 17 ++++++++++------- .../.write_torch_versions.py.swp | Bin 0 -> 16384 bytes .../github_actions/write_torch_versions.py | 3 --- 3 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 build_tools/github_actions/.write_torch_versions.py.swp diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index da210aabcb5..825041f1f0f 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -106,7 +106,7 @@ jobs: optional_build_prod_arguments: "" outputs: torch_version: ${{ steps.build-pytorch-wheels.outputs.torch_version }} - cp_version: ${{ steps.select-python.outputs.cp_version }} + cp_version: ${{ steps.output-cp-version.cp_version }} steps: - name: Checkout uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -118,11 +118,14 @@ jobs: - name: Select Python version id: select-python - shell: bash run: | - cp_version=$(python build_tools/github_actions/python_to_cp_version.py \ - --python-version ${{ inputs.python_version }}) - echo "cp_version=$cp_version" >> "$GITHUB_OUTPUT" + python build_tools/github_actions/python_to_cp_version.py \ + --python-version ${{ inputs.python_version }} + + - name: Add cp_version to output + id: output-cp-version + run: + echo "cp_version=${{ env.cp_version }}" >> $GITHUB_OUTPUT - name: Add selected Python version to PATH run: | @@ -238,7 +241,7 @@ jobs: CP_VERSION: ${{ needs.build_pytorch_wheels.outputs.cp_version }} TORCH_VERSION: "${{ needs.build_pytorch_wheels.outputs.torch_version }}" TORCHAUDIO_VERSION: "${{ needs.build_pytorch_wheels.outputs.torchaudio_version }}" - TORCHVIDEO_VERSION: "${{ needs.build_pytorch_wheels.outputs.torchvideo_version }}" + TORCHVISION_VERSION: "${{ needs.build_pytorch_wheels.outputs.torchvision_version }}" steps: - name: Checkout @@ -261,7 +264,7 @@ jobs: --exclude "*" \ --include "torch-${TORCH_VERSION}-${CP_VERSION}-linux_x86_64.whl" \ --include "torchaudio-${TORCHAUDIO_VERSION}-${CP_VERSION}-linux_x86_64.whl" \ - --include "torchvideo-${TORCHVIDEO_VERSION}-${CP_VERSION}-linux_x86_64.whl" + --include "torchvision-${TORCHVISION_VERSION}-${CP_VERSION}-linux_x86_64.whl" - name: (Re-)Generate Python package release index run: | diff --git a/build_tools/github_actions/.write_torch_versions.py.swp b/build_tools/github_actions/.write_torch_versions.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..6064f7be175459a3c4177d70c968b421f45e71f1 GIT binary patch literal 16384 zcmeHOON<=F87|)72Sjk;0HWL|*`C!*+iPS%BMoQ`*jX7b&a#UotX0#Vu9<0VPj{=U zdY8?5;|riTmqFepP$N%{LdUbWr*+b7Poa7IChZ&9^Fm~XbpZ7jGavS^X6UKr(57wi#qEC63IRYw{ zgpJ^*MJ^gjNcGIgor?c#An@RFPb{VFH9rv>)%C%%MHGj=O4C>#>_)0rob`jYiqb?L z+{hy(kd?Q4{<_E|C3@NBPD*T=Wxz5p%|M|#jfeNIJaBmK5M|!)9N>4|xiVeOrdS3n z1C{~HfMvikU>UFsSOzQu|F;aN?0R+{-d$gM*eidpoA|wPxR4BQRu1@-`+-on_cz$Ksybbu4U7ufvx82AJ58t^ONQQ%(S4&aTOP!G5S z1i*8^0>FU(-pJSm;5={?m9xD~hs_yBRDSX}_l z0SY(*{HxlqVH`)d7x+>rKP^;ND0j=zpH2it7^T(s6Hb#)CvMkt_2~@W;@vns>(R+| zu2MzI99k8L%7aJ>x}al+n`2Qd7E^UBEt2qwJWq4C<2)Wo#ltA)Gh0F4m0pN@ANSic zJWPe;NvgPb9*K@|DeCB?v9g{nAl+J%F;Y#La5VF>AeX|YTNkHwBf*@hTu;PV%Q+RO z9^dGRT+ms>67>a-({`Xl=+qgNq$3oapomqA%Dbubx18KV}x2f}tC2VW^p=dn0Y*^&i71BeW zIITzYfzxF;8a73eJVDlogk3YnNFE(J3XHBqd`;*ZS~beybn1}E>Adg8rXKY!L*`hh zcZxh2as@-DW{XlF*GED)PQuJ6?)pE0-y%ux@ zMqV@g*13~yY6kqWcAmETmAvVYUa3$m()5c_VPj$0l`LIzR8-~CuB4O#CYsQep?JWh zZW+{))=T-+(+ICsHljfZ6d1QuGx|U{SG9vWEr;A7`L&G3VeWDFkHk%_h=$deV zM((N5lymni<{52JY0Q&px)vo}PF9rj7<9~RO-6Qm28wFD@d!T|WEpDE)KKR*VSJ>y zZPl1OS+A)x>llHXx-jO~I8e~m_vEHjqVJlChuRkLIM%+TJ3jM_WsZ8mp#^hX$jh%o z;lNl~Z9RoxEEQClslKe`ck;AvEO1T1433S_#kq-w&gKY2rHD#SXrUyi7)p|74y|gd z$&&FwoLa@{`K85$#pBIVMIWn%C_}rbR3t%PV11+INzBx$-J{4=*|awEp;L)G*`BHM zLQ<~k2{fG?t0i)dm$Z;+a@!dgqZ@<1)8l|`W{*(nn-uOBK$u@T0pI}aAFCViBrI?vNoFC1Hw4c9m{T?G<5PD!J7q%rGl z^kQS?4xOqBk1k97_Z#{at-ii}fUUZdv5REtZlyb>>5CK<@Sp=z25QKMC$l(GuG4U8 zXXaM*ith?~fKgiMsDz{YdQydW#_&3EA$#b^97mQ%Sd7kM5i09^c6M!pu1p`->`dm6 zINdbIzxnnH6=6*s^|Lfre3q3L)!GD?c)@jQ@W%#>z^ zu8&OA<+Yh#>O2^%Q&F{CO^#PKMFtZ+d#qZ8*72kqXuCTovsv|3J!E6~DM~RIsA#;U zzF9fMbv8Y=;K9w)hM%Ud-b-_<>1ovLi8N#WFmdQcl?Ipj@rC6duN?KCT3J4|vg|Q> z{y&W8`VK(P|9Fhm$D6qS3sCI?Y?oyOe)j-x;dx#FA Date: Fri, 25 Jul 2025 12:03:26 +0000 Subject: [PATCH 27/57] Commenting out checkut of triton repo --- .../build_portable_linux_pytorch_wheels.yml | 8 ++++---- .../.write_torch_versions.py.swp | Bin 16384 -> 16384 bytes .../github_actions/write_torch_versions.py | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 825041f1f0f..538525ceaca 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -121,7 +121,7 @@ jobs: run: | python build_tools/github_actions/python_to_cp_version.py \ --python-version ${{ inputs.python_version }} - + - name: Add cp_version to output id: output-cp-version run: @@ -133,7 +133,7 @@ jobs: if ! [ -x "${python_dir}/bin/python" ]; then echo "ERROR: Could not find python: ${python_dir}" exit 1 - fi + fi echo "${python_dir}/bin" >> "$GITHUB_PATH" # Github workflows do not have if/else and therefore we define two jobs with different if clauses. @@ -154,7 +154,7 @@ jobs: ./external-builds/pytorch/pytorch_torch_repo.py checkout --gitrepo-origin https://github.com/ROCm/pytorch.git --repo-hashtag ${{ inputs.pytorch_version }} --patchset ${{ inputs.pytorch_patchset }} ./external-builds/pytorch/pytorch_audio_repo.py checkout --require-related-commit ./external-builds/pytorch/pytorch_vision_repo.py checkout --require-related-commit - ./external-builds/pytorch/pytorch_triton_repo.py checkout + # ./external-builds/pytorch/pytorch_triton_repo.py checkout - name: Create pip cache directory run: mkdir -p /tmp/pipcache @@ -238,7 +238,7 @@ jobs: runs-on: ubuntu-24.04 env: S3_BUCKET_PY: "therock-${{ inputs.release_type }}-python" - CP_VERSION: ${{ needs.build_pytorch_wheels.outputs.cp_version }} + CP_VERSION: "${{ needs.build_pytorch_wheels.outputs.cp_version }}" TORCH_VERSION: "${{ needs.build_pytorch_wheels.outputs.torch_version }}" TORCHAUDIO_VERSION: "${{ needs.build_pytorch_wheels.outputs.torchaudio_version }}" TORCHVISION_VERSION: "${{ needs.build_pytorch_wheels.outputs.torchvision_version }}" diff --git a/build_tools/github_actions/.write_torch_versions.py.swp b/build_tools/github_actions/.write_torch_versions.py.swp index 6064f7be175459a3c4177d70c968b421f45e71f1..a51253d33423dd38d740c9be533907080b7b20a0 100644 GIT binary patch delta 408 zcmWmAJ1j$S6vy%3zj{?$~X zOvH7unkAhKEE0pkro>hfzJ^b7mh(Hm9M?6jYc6fu6hp`UYE2Cp=`|wBmWpL3T`29f zNV^zCKl+ftQ%dr&h;DS@w^{mx4-cD|#Tf3B(jAVmj2zY22yU9FifK$Ck9v$?TqA^!0*3L^D1GAv2Ux;9qNrQ!;R5HVSaTd=7{Ftc zA~=MHb*y0(Gw6VYT7&eAGi+c5lbFCD{{LG=a1!*G@PB|8V)Xs1FOBsLi=w?(SMQy7iHz(W^?tr z#KF}~;f8~gqk|#`2Yj79^?CaBJU#W4N|sW|a^mw@vNgZ&-RpUhv|C5>N!hYLFl5d) zNn_|kFM9Cpk)H5~2W%mZ7Tmj~JDlMFam3IK4}M(I3i2372VR}h6{@IU9~&5T%2br_ z*(hCO4<=GbVgfM;=MG**GhA@u!!8}7iVV_-!!eGqg8{U{iwhffsM@Ba1Yril z5T5I$O_&%&3?T&31vhT2(j_X`#v~%>gadZGR8QM1<=ZY>&Dy@^8VOLwvJo{3$%SGe un$8uCY%rw1@nU{>>>>>> c41c84b (Commenting out checkut of triton repo) if __name__ == "__main__": main(sys.argv[1:]) From 1bbd27ff6e457e5ff85a7744c6d5d59c655f4e93 Mon Sep 17 00:00:00 2001 From: arravikum Date: Fri, 25 Jul 2025 12:37:26 +0000 Subject: [PATCH 28/57] Adding locally tested file changes --- build_tools/github_actions/write_torch_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/github_actions/write_torch_versions.py b/build_tools/github_actions/write_torch_versions.py index 294065ff042..655e22c263c 100644 --- a/build_tools/github_actions/write_torch_versions.py +++ b/build_tools/github_actions/write_torch_versions.py @@ -119,7 +119,7 @@ def main(argv: list[str]): torchvision_version_tmp = glob.glob("torchvision-*.whl", root_dir=package_dist_dir) if torchvision_version_tmp: torchvision_version = torchvision_version_tmp[0].split("-")[1] - gha_set_output({"torchaudio_version": torchaudio_version}) + gha_set_output({"torchvision_version": torchvision_version}) >>>>>>> c41c84b (Commenting out checkut of triton repo) From 2f05501890163ec3be5d74b7504196f55ad77038 Mon Sep 17 00:00:00 2001 From: arravikum Date: Fri, 25 Jul 2025 15:45:51 +0000 Subject: [PATCH 29/57] Reverting to pytorch/pytorch repo to verify build --- .../workflows/build_portable_linux_pytorch_wheels.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 538525ceaca..c936f82c376 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -121,7 +121,7 @@ jobs: run: | python build_tools/github_actions/python_to_cp_version.py \ --python-version ${{ inputs.python_version }} - + - name: Add cp_version to output id: output-cp-version run: @@ -151,9 +151,12 @@ jobs: - name: Checkout PyTorch Source Repos from stable branch if: ${{ inputs.pytorch_version != 'nightly' }} run: | - ./external-builds/pytorch/pytorch_torch_repo.py checkout --gitrepo-origin https://github.com/ROCm/pytorch.git --repo-hashtag ${{ inputs.pytorch_version }} --patchset ${{ inputs.pytorch_patchset }} - ./external-builds/pytorch/pytorch_audio_repo.py checkout --require-related-commit - ./external-builds/pytorch/pytorch_vision_repo.py checkout --require-related-commit + ./external-builds/pytorch/pytorch_torch_repo.py checkout + ./external-builds/pytorch/pytorch_audio_repo.py checkout + ./external-builds/pytorch/pytorch_vision_repo.py checkout + # ./external-builds/pytorch/pytorch_torch_repo.py checkout --gitrepo-origin https://github.com/ROCm/pytorch.git --repo-hashtag ${{ inputs.pytorch_version }} --patchset ${{ inputs.pytorch_patchset }} + # ./external-builds/pytorch/pytorch_audio_repo.py checkout --require-related-commit + # ./external-builds/pytorch/pytorch_vision_repo.py checkout --require-related-commit # ./external-builds/pytorch/pytorch_triton_repo.py checkout - name: Create pip cache directory From 16fba9371fa82695b2ecd52f434764aaaaf4e1a9 Mon Sep 17 00:00:00 2001 From: arravikum Date: Fri, 25 Jul 2025 15:46:40 +0000 Subject: [PATCH 30/57] Reverting to pytorch/pytorch repo to verify build --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index c936f82c376..6c4949eaeb3 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -121,7 +121,7 @@ jobs: run: | python build_tools/github_actions/python_to_cp_version.py \ --python-version ${{ inputs.python_version }} - + - name: Add cp_version to output id: output-cp-version run: From d40c0c9a686cf844aee5678bea46978b083f88da Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 29 Jul 2025 15:02:03 +0000 Subject: [PATCH 31/57] Adding changes after rebase, restoring torch checkouts to ROCm/pytorch --- .../build_portable_linux_pytorch_wheels.yml | 13 +++++-------- build_tools/github_actions/write_torch_versions.py | 4 ---- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 6c4949eaeb3..455faf774e2 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -146,18 +146,15 @@ jobs: ./external-builds/pytorch/pytorch_audio_repo.py checkout --repo-hashtag nightly ./external-builds/pytorch/pytorch_vision_repo.py checkout --repo-hashtag nightly ./external-builds/pytorch/pytorch_triton_repo.py checkout --patch --patchset nightly - + # Then the else branch to checkout release version of source code instead. - name: Checkout PyTorch Source Repos from stable branch if: ${{ inputs.pytorch_version != 'nightly' }} run: | - ./external-builds/pytorch/pytorch_torch_repo.py checkout - ./external-builds/pytorch/pytorch_audio_repo.py checkout - ./external-builds/pytorch/pytorch_vision_repo.py checkout - # ./external-builds/pytorch/pytorch_torch_repo.py checkout --gitrepo-origin https://github.com/ROCm/pytorch.git --repo-hashtag ${{ inputs.pytorch_version }} --patchset ${{ inputs.pytorch_patchset }} - # ./external-builds/pytorch/pytorch_audio_repo.py checkout --require-related-commit - # ./external-builds/pytorch/pytorch_vision_repo.py checkout --require-related-commit - # ./external-builds/pytorch/pytorch_triton_repo.py checkout + ./external-builds/pytorch/pytorch_torch_repo.py checkout --gitrepo-origin https://github.com/ROCm/pytorch.git --repo-hashtag ${{ inputs.pytorch_version }} --patchset ${{ inputs.pytorch_patchset }} + ./external-builds/pytorch/pytorch_audio_repo.py checkout --require-related-commit + ./external-builds/pytorch/pytorch_vision_repo.py checkout --require-related-commit + ./external-builds/pytorch/pytorch_triton_repo.py checkout - name: Create pip cache directory run: mkdir -p /tmp/pipcache diff --git a/build_tools/github_actions/write_torch_versions.py b/build_tools/github_actions/write_torch_versions.py index 655e22c263c..d6dbe885474 100644 --- a/build_tools/github_actions/write_torch_versions.py +++ b/build_tools/github_actions/write_torch_versions.py @@ -91,7 +91,6 @@ def get_all_wheel_versions( def main(argv: list[str]): -<<<<<<< HEAD p = argparse.ArgumentParser(prog="write_torch_versions.py") p.add_argument( "--dist-dir", @@ -105,7 +104,6 @@ def main(argv: list[str]): all_versions = get_all_wheel_versions(args.dist_dir) _log("") gha_set_output(all_versions) -======= # Get the torch version from the first torch wheel in PACKAGE_DIST_DIR. package_dist_dir = os.getenv("PACKAGE_DIST_DIR") torch_version = glob.glob("torch-*.whl", root_dir=package_dist_dir)[0].split("-")[1] @@ -121,7 +119,5 @@ def main(argv: list[str]): torchvision_version = torchvision_version_tmp[0].split("-")[1] gha_set_output({"torchvision_version": torchvision_version}) ->>>>>>> c41c84b (Commenting out checkut of triton repo) - if __name__ == "__main__": main(sys.argv[1:]) From 3cc50f8ba1680c9732c3f3c3c97a3aaf4caf0260 Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 29 Jul 2025 15:24:50 +0000 Subject: [PATCH 32/57] Fixed pre-commit issues and trimmer write_torch_versions according to prevs changes --- .../build_portable_linux_pytorch_wheels.yml | 1 - .../github_actions/write_torch_versions.py | 15 +-------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 455faf774e2..4fc6ff5fc1c 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -270,4 +270,3 @@ jobs: run: | pip install boto3 packaging python ./build_tools/third_party/s3_management/manage.py ${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }} - diff --git a/build_tools/github_actions/write_torch_versions.py b/build_tools/github_actions/write_torch_versions.py index d6dbe885474..94f1c8eabe5 100644 --- a/build_tools/github_actions/write_torch_versions.py +++ b/build_tools/github_actions/write_torch_versions.py @@ -104,20 +104,7 @@ def main(argv: list[str]): all_versions = get_all_wheel_versions(args.dist_dir) _log("") gha_set_output(all_versions) - # Get the torch version from the first torch wheel in PACKAGE_DIST_DIR. - package_dist_dir = os.getenv("PACKAGE_DIST_DIR") - torch_version = glob.glob("torch-*.whl", root_dir=package_dist_dir)[0].split("-")[1] - gha_set_output({"torch_version": torch_version}) - - torchaudio_version_tmp = glob.glob("torchaudio-*.whl", root_dir=package_dist_dir) - if torchaudio_version_tmp: - torchaudio_version = torchaudio_version_tmp[0].split("-")[1] - gha_set_output({"torchaudio_version": torchaudio_version}) - - torchvision_version_tmp = glob.glob("torchvision-*.whl", root_dir=package_dist_dir) - if torchvision_version_tmp: - torchvision_version = torchvision_version_tmp[0].split("-")[1] - gha_set_output({"torchvision_version": torchvision_version}) + if __name__ == "__main__": main(sys.argv[1:]) From 746e9ffc1057c39453c8a48cc601a5b89bf74c58 Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 29 Jul 2025 17:27:48 +0000 Subject: [PATCH 33/57] Added / in index_url to get proper index links --- .github/workflows/test_pytorch_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_pytorch_wheels.yml b/.github/workflows/test_pytorch_wheels.yml index 6dbc9b6dff7..cfa05d2d550 100644 --- a/.github/workflows/test_pytorch_wheels.yml +++ b/.github/workflows/test_pytorch_wheels.yml @@ -101,7 +101,7 @@ jobs: run: | python build_tools/setup_venv.py ${VENV_DIR} \ --packages torch==${TORCH_VERSION} \ - --index-url ${{ inputs.cloudfront_url }} \ + --index-url ${{ inputs.cloudfront_url }}/ \ --index-subdir ${{ inputs.amdgpu_family }} \ --activate-in-future-github-actions-steps From 22481634e34f372a4644a6e3721d707011b838a5 Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 29 Jul 2025 18:20:53 +0000 Subject: [PATCH 34/57] Adding quotes for index_url --- .github/workflows/test_pytorch_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_pytorch_wheels.yml b/.github/workflows/test_pytorch_wheels.yml index cfa05d2d550..3eedfef622a 100644 --- a/.github/workflows/test_pytorch_wheels.yml +++ b/.github/workflows/test_pytorch_wheels.yml @@ -101,7 +101,7 @@ jobs: run: | python build_tools/setup_venv.py ${VENV_DIR} \ --packages torch==${TORCH_VERSION} \ - --index-url ${{ inputs.cloudfront_url }}/ \ + --index-url "${{ inputs.cloudfront_url }}/" \ --index-subdir ${{ inputs.amdgpu_family }} \ --activate-in-future-github-actions-steps From b3ffedcdb74b86ec0d462e3d9b340e5bcc4917da Mon Sep 17 00:00:00 2001 From: arravikum Date: Wed, 30 Jul 2025 13:50:15 +0000 Subject: [PATCH 35/57] Reverted test_pytorch_changes and added GITHUB_OUTPUT usage from write_torch_versions.py --- .../build_portable_linux_pytorch_wheels.yml | 9 +++------ .github/workflows/test_pytorch_wheels.yml | 2 +- .../github_actions/.write_torch_versions.py.swp | Bin 16384 -> 0 bytes 3 files changed, 4 insertions(+), 7 deletions(-) delete mode 100644 build_tools/github_actions/.write_torch_versions.py.swp diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 4fc6ff5fc1c..cde72e3f570 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -106,7 +106,9 @@ jobs: optional_build_prod_arguments: "" outputs: torch_version: ${{ steps.build-pytorch-wheels.outputs.torch_version }} - cp_version: ${{ steps.output-cp-version.cp_version }} + cp_version: ${{ steps.select-python.outputs.cp_version }} + torchaudio_version: ${{ steps.build-pytorch-wheels.outputs.torchaudio_version }} + torchvision_version: ${{ steps.build-pytorch-wheels.outputs.torchvision_version }} steps: - name: Checkout uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -122,11 +124,6 @@ jobs: python build_tools/github_actions/python_to_cp_version.py \ --python-version ${{ inputs.python_version }} - - name: Add cp_version to output - id: output-cp-version - run: - echo "cp_version=${{ env.cp_version }}" >> $GITHUB_OUTPUT - - name: Add selected Python version to PATH run: | python_dir="/opt/python/${{ env.cp_version }}" diff --git a/.github/workflows/test_pytorch_wheels.yml b/.github/workflows/test_pytorch_wheels.yml index 3eedfef622a..6dbc9b6dff7 100644 --- a/.github/workflows/test_pytorch_wheels.yml +++ b/.github/workflows/test_pytorch_wheels.yml @@ -101,7 +101,7 @@ jobs: run: | python build_tools/setup_venv.py ${VENV_DIR} \ --packages torch==${TORCH_VERSION} \ - --index-url "${{ inputs.cloudfront_url }}/" \ + --index-url ${{ inputs.cloudfront_url }} \ --index-subdir ${{ inputs.amdgpu_family }} \ --activate-in-future-github-actions-steps diff --git a/build_tools/github_actions/.write_torch_versions.py.swp b/build_tools/github_actions/.write_torch_versions.py.swp deleted file mode 100644 index a51253d33423dd38d740c9be533907080b7b20a0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI2ONNV zdVKZOznXeGPd^ITdfP5SF%4YGUh~)GwaSIZHl|X;ZQQaKI$Az zA~7mOEDua9mi*Wpo|!~W0o0s#O2XV3rd-NV>Pupc}Fc7TV#FLyKc z6F3jP2j7CPz{lWC@CJAtyarwcyTJ%}6g&dXt%hCTf(O9^;P1N_y9iE#Vf zad7E&*acsJ&%sHs6I8)pc=7QkI1k{Rr9^4OBfD3rR@+0^NbU+NIz$6$0)IMjxDR2V34UPc`HiNU-3me99G+T}x3uy<5 z3=>(Hx74Q-PEvP+?E49;!iNJFiYohX1h4X@7aT6rsbJk;5mkMtqR3R_xUrxMIu^Ov z;d)|kAa?|bU*E1%W$Bg@b%h*J6hwuFwatwsue%W+nRlXQT(0BZ#{I$wuLmOLejvFx zfAj%2m;$qi4 z+8(-l!5lOw@@kb^b2^FDE4IH*Bw=3_`7w^ssQk{VO(QWEu%P@$LHXdcV^O`9C=Y!v zHS5yXQdiKZ*A>a~fLMJNu30fg_UKTp!01YN|B8G)ulh9{4($m!9Cv+cswwmeV&MowNJA!dYnQqaJE~L2|LB#2vyLxJ+>sZfB+Jss~Rg##}&@@O(pQu5VPNmNWl@k@9 znR}>d(v846tTW1?)R{d&Fys17PF|$z7|Ix#A8gr$5foJ9qfhV&Ck&B;mWC3?3G*YZ zZPVK2!F&}JS;7olQHim>dX7YC?Q-0SrDzxQ!b5$FbjZ5Lu3U}%7IcS2u1WP#fLUb zE3e6u{3K4j;^6rHy}S1AtfWP>v1@Qc^oy(r-)Rf%ZCn(sZ~_X!Xe`PwQU>n)b)K45R)i>`&M zR>u#zo?cT$7IPBQ?2wCC*EgiISfAoFoV&YV9p0}~iAO=8HjdT7j;kv)oCOveRw|>a z$mqKAT+7pLE;6XH=22;B{9a~k(eCT&6Kpn|v|nUX4J$P;^;o1Qs3H|Uh5^H#GUTe5?Rv?U_P@>(p{U`)@!J7Y+meV)nln?R$-8rSB|nw2YhRB$>YX0`%)cZ>ko52MYOne{dGr EKa8lbod5s; From 25bfa39054b2432d75702d323c3df682ced69c81 Mon Sep 17 00:00:00 2001 From: arravikum Date: Wed, 30 Jul 2025 15:34:04 +0000 Subject: [PATCH 36/57] Added extraction of cp_version from env vars in job output --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index cde72e3f570..ec8ab4cb528 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -106,7 +106,7 @@ jobs: optional_build_prod_arguments: "" outputs: torch_version: ${{ steps.build-pytorch-wheels.outputs.torch_version }} - cp_version: ${{ steps.select-python.outputs.cp_version }} + cp_version: ${{ env.cp_version }} torchaudio_version: ${{ steps.build-pytorch-wheels.outputs.torchaudio_version }} torchvision_version: ${{ steps.build-pytorch-wheels.outputs.torchvision_version }} steps: From a88b0fb5eb852f773a99414c26c2c0a824765698 Mon Sep 17 00:00:00 2001 From: arravikum Date: Wed, 30 Jul 2025 22:17:01 +0000 Subject: [PATCH 37/57] Resolved PR review comments --- .../build_portable_linux_pytorch_wheels.yml | 8 +++++--- .../workflows/release_portable_linux_packages.yml | 12 ++++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index ec8ab4cb528..f472948a72a 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -109,6 +109,7 @@ jobs: cp_version: ${{ env.cp_version }} torchaudio_version: ${{ steps.build-pytorch-wheels.outputs.torchaudio_version }} torchvision_version: ${{ steps.build-pytorch-wheels.outputs.torchvision_version }} + triton_version: ${{ steps.build-pytorch-wheels.outputs.triton_version }} steps: - name: Checkout uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -119,7 +120,6 @@ jobs: git config --global user.email "therockbot@amd.com" - name: Select Python version - id: select-python run: | python build_tools/github_actions/python_to_cp_version.py \ --python-version ${{ inputs.python_version }} @@ -193,7 +193,7 @@ jobs: aws s3 cp ${{ env.PACKAGE_DIST_DIR }}/ s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_staging_subdir }}/${{ inputs.amdgpu_family }}/ \ --recursive --exclude "*" --include "*.whl" - - name: (Re-)Generate Python package release index + - name: (Re-)Generate Python package release index for staging if: ${{ github.repository_owner == 'ROCm' }} run: | pip install boto3 packaging @@ -239,6 +239,7 @@ jobs: TORCH_VERSION: "${{ needs.build_pytorch_wheels.outputs.torch_version }}" TORCHAUDIO_VERSION: "${{ needs.build_pytorch_wheels.outputs.torchaudio_version }}" TORCHVISION_VERSION: "${{ needs.build_pytorch_wheels.outputs.torchvision_version }}" + TRITON_VERSION: "${{ needs.build_pytorch_wheels.outputs.triton_version }}" steps: - name: Checkout @@ -261,7 +262,8 @@ jobs: --exclude "*" \ --include "torch-${TORCH_VERSION}-${CP_VERSION}-linux_x86_64.whl" \ --include "torchaudio-${TORCHAUDIO_VERSION}-${CP_VERSION}-linux_x86_64.whl" \ - --include "torchvision-${TORCHVISION_VERSION}-${CP_VERSION}-linux_x86_64.whl" + --include "torchvision-${TORCHVISION_VERSION}-${CP_VERSION}-linux_x86_64.whl" \ + --include "pytorch_triton_rocm-${TRITON_VERSION}-${CP_VERSION}-linux_x86_64.whl" - name: (Re-)Generate Python package release index run: | diff --git a/.github/workflows/release_portable_linux_packages.yml b/.github/workflows/release_portable_linux_packages.yml index da194854fcc..805d864ca74 100644 --- a/.github/workflows/release_portable_linux_packages.yml +++ b/.github/workflows/release_portable_linux_packages.yml @@ -157,6 +157,12 @@ jobs: echo "Building ${{ env.DIST_ARCHIVE }}" tar cfz "${{ env.DIST_ARCHIVE }}" . + - name: Setup Python + if: ${{ github.repository_owner == 'ROCm' }} + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: 3.12 + - name: Build Python Packages run: | ./build_tools/linux_portable_build.py \ @@ -207,12 +213,6 @@ jobs: --include "*.whl" \ --include "*.tar.gz" - - name: Setup Python - if: ${{ github.repository_owner == 'ROCm' }} - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: 3.12 - - name: (Re-)Generate Python package release index if: ${{ github.repository_owner == 'ROCm' }} run: | From db929bfff2b34d60dd54bba5e43509c3a0dd2428 Mon Sep 17 00:00:00 2001 From: arravikum Date: Thu, 31 Jul 2025 14:10:10 +0000 Subject: [PATCH 38/57] Resolved new PR review comments and added conditional upload if test runner not present --- .../build_portable_linux_pytorch_wheels.yml | 12 +++++++++--- .../workflows/release_portable_linux_packages.yml | 1 - 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index f472948a72a..00aa006f7df 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -105,8 +105,8 @@ jobs: S3_BUCKET_PY: "therock-${{ inputs.release_type }}-python" optional_build_prod_arguments: "" outputs: - torch_version: ${{ steps.build-pytorch-wheels.outputs.torch_version }} cp_version: ${{ env.cp_version }} + torch_version: ${{ steps.build-pytorch-wheels.outputs.torch_version }} torchaudio_version: ${{ steps.build-pytorch-wheels.outputs.torchaudio_version }} torchvision_version: ${{ steps.build-pytorch-wheels.outputs.torchvision_version }} triton_version: ${{ steps.build-pytorch-wheels.outputs.triton_version }} @@ -187,7 +187,7 @@ jobs: run: | python external-builds/pytorch/sanity_check_wheel.py ${{ env.PACKAGE_DIST_DIR }}/ - - name: Upload wheels to staging S3 + - name: Upload wheels to S3 staging if: ${{ github.repository_owner == 'ROCm' }} run: | aws s3 cp ${{ env.PACKAGE_DIST_DIR }}/ s3://${{ env.S3_BUCKET_PY }}/${{ inputs.s3_staging_subdir }}/${{ inputs.amdgpu_family }}/ \ @@ -231,7 +231,13 @@ jobs: upload_pytorch_wheels: name: Upload PyTorch Wheels to S3 needs: [build_pytorch_wheels, generate_target_to_run, test_pytorch_wheels] - if: success() && github.repository_owner == 'ROCm' + if: > + success() && + github.repository_owner == 'ROCm' && + ( + needs.generate_target_to_run.outputs.test_runs_on == '' || + needs.test_pytorch_wheels.result == 'success' + ) runs-on: ubuntu-24.04 env: S3_BUCKET_PY: "therock-${{ inputs.release_type }}-python" diff --git a/.github/workflows/release_portable_linux_packages.yml b/.github/workflows/release_portable_linux_packages.yml index 805d864ca74..7d1bb014ea9 100644 --- a/.github/workflows/release_portable_linux_packages.yml +++ b/.github/workflows/release_portable_linux_packages.yml @@ -158,7 +158,6 @@ jobs: tar cfz "${{ env.DIST_ARCHIVE }}" . - name: Setup Python - if: ${{ github.repository_owner == 'ROCm' }} uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: 3.12 From bdff065bb1b72c8b4754f3695e67b93e1b781cae Mon Sep 17 00:00:00 2001 From: arravikum Date: Thu, 31 Jul 2025 14:30:17 +0000 Subject: [PATCH 39/57] Modified if clause in 'Upload_pytorch_wheels' job --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 00aa006f7df..c40490f50f7 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -230,13 +230,16 @@ jobs: upload_pytorch_wheels: name: Upload PyTorch Wheels to S3 - needs: [build_pytorch_wheels, generate_target_to_run, test_pytorch_wheels] + needs: [build_pytorch_wheels, generate_target_to_run] if: > success() && github.repository_owner == 'ROCm' && ( needs.generate_target_to_run.outputs.test_runs_on == '' || - needs.test_pytorch_wheels.result == 'success' + ( + always() && + needs.test_pytorch_wheels.result == 'success' + ) ) runs-on: ubuntu-24.04 env: From bc63a8720c5b302b1cdec83ba4b5a810a142b4ac Mon Sep 17 00:00:00 2001 From: arravikum Date: Thu, 31 Jul 2025 19:13:17 +0000 Subject: [PATCH 40/57] Resolved upload step conditional logic to remove always() inside if: --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index c40490f50f7..f3579a40fe0 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -232,14 +232,10 @@ jobs: name: Upload PyTorch Wheels to S3 needs: [build_pytorch_wheels, generate_target_to_run] if: > - success() && github.repository_owner == 'ROCm' && ( needs.generate_target_to_run.outputs.test_runs_on == '' || - ( - always() && - needs.test_pytorch_wheels.result == 'success' - ) + needs.test_pytorch_wheels.result == 'success' ) runs-on: ubuntu-24.04 env: From 7192390f63401dfbc10c6061372ced99e52f8621 Mon Sep 17 00:00:00 2001 From: arravikum Date: Thu, 31 Jul 2025 20:29:33 +0000 Subject: [PATCH 41/57] Adding test_pytorch_wheels in needs: of upload_pytorch_wheels job to ensure wait for test wheel to finish --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index f3579a40fe0..4fbc10814b9 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -230,7 +230,7 @@ jobs: upload_pytorch_wheels: name: Upload PyTorch Wheels to S3 - needs: [build_pytorch_wheels, generate_target_to_run] + needs: [build_pytorch_wheels, generate_target_to_run, test_pytorch_wheels] if: > github.repository_owner == 'ROCm' && ( From ee26b7d3607f59f02a54ac979c7363970ea5d991 Mon Sep 17 00:00:00 2001 From: arravikum Date: Fri, 1 Aug 2025 02:12:42 +0000 Subject: [PATCH 42/57] Add check-gate step output variable --- .../build_portable_linux_pytorch_wheels.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 4fbc10814b9..d409b7e5443 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -231,12 +231,7 @@ jobs: upload_pytorch_wheels: name: Upload PyTorch Wheels to S3 needs: [build_pytorch_wheels, generate_target_to_run, test_pytorch_wheels] - if: > - github.repository_owner == 'ROCm' && - ( - needs.generate_target_to_run.outputs.test_runs_on == '' || - needs.test_pytorch_wheels.result == 'success' - ) + if: always() runs-on: ubuntu-24.04 env: S3_BUCKET_PY: "therock-${{ inputs.release_type }}-python" @@ -257,7 +252,18 @@ jobs: aws-region: us-east-2 role-to-assume: arn:aws:iam::692859939525:role/therock-${{ inputs.release_type }}-releases + - name: Check test result gating + id: check-gate + run: | + if [[ "${{ needs.generate_target_to_run.outputs.test_runs_on }}" != "" && "${{ needs.test_pytorch_wheels.result }}" == "failure" ]]; then + echo "::warning::Tests failed. Skipping upload." + echo "upload=false" >> $GITHUB_OUTPUT + else + echo "upload=true" >> $GITHUB_OUTPUT + fi + - name: Copy tested PyTorch wheels from staging to release S3 + if: steps.check-gate.outputs.upload == 'true' run: | echo "Copying exact tested wheels to release S3 bucket..." aws s3 cp \ @@ -271,6 +277,7 @@ jobs: --include "pytorch_triton_rocm-${TRITON_VERSION}-${CP_VERSION}-linux_x86_64.whl" - name: (Re-)Generate Python package release index + if: steps.check-gate.outputs.upload == 'true' run: | pip install boto3 packaging python ./build_tools/third_party/s3_management/manage.py ${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }} From f957f64904885cd8e200d72f82c05e8d76f0b10a Mon Sep 17 00:00:00 2001 From: arravikum Date: Mon, 11 Aug 2025 15:05:38 +0000 Subject: [PATCH 43/57] addressed PR reviews and converted upload flag to env variable --- .../workflows/build_portable_linux_pytorch_wheels.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index d409b7e5443..034595cd3c8 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -247,7 +247,7 @@ jobs: - name: Configure AWS Credentials if: always() - uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1 + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: aws-region: us-east-2 role-to-assume: arn:aws:iam::692859939525:role/therock-${{ inputs.release_type }}-releases @@ -257,13 +257,13 @@ jobs: run: | if [[ "${{ needs.generate_target_to_run.outputs.test_runs_on }}" != "" && "${{ needs.test_pytorch_wheels.result }}" == "failure" ]]; then echo "::warning::Tests failed. Skipping upload." - echo "upload=false" >> $GITHUB_OUTPUT + echo "upload=false" >> $GITHUB_ENV else - echo "upload=true" >> $GITHUB_OUTPUT + echo "upload=true" >> $GITHUB_ENV fi - - name: Copy tested PyTorch wheels from staging to release S3 - if: steps.check-gate.outputs.upload == 'true' + - name: Copy PyTorch wheels from staging to release S3 + if: ${{ env.upload == 'true' }} run: | echo "Copying exact tested wheels to release S3 bucket..." aws s3 cp \ From 971d9f9c993984dcfcdcde6ab5b89227ee1a2961 Mon Sep 17 00:00:00 2001 From: arravikum Date: Mon, 11 Aug 2025 15:40:36 +0000 Subject: [PATCH 44/57] Removed check-gate id from workflow step --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 034595cd3c8..5bdde1ff24e 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -253,7 +253,6 @@ jobs: role-to-assume: arn:aws:iam::692859939525:role/therock-${{ inputs.release_type }}-releases - name: Check test result gating - id: check-gate run: | if [[ "${{ needs.generate_target_to_run.outputs.test_runs_on }}" != "" && "${{ needs.test_pytorch_wheels.result }}" == "failure" ]]; then echo "::warning::Tests failed. Skipping upload." @@ -277,7 +276,7 @@ jobs: --include "pytorch_triton_rocm-${TRITON_VERSION}-${CP_VERSION}-linux_x86_64.whl" - name: (Re-)Generate Python package release index - if: steps.check-gate.outputs.upload == 'true' + if: ${{ env.upload == 'true' }} run: | pip install boto3 packaging python ./build_tools/third_party/s3_management/manage.py ${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }} From 029ea02363afa635841e0c569314cde573dc5d4b Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 12 Aug 2025 15:11:11 +0000 Subject: [PATCH 45/57] Merged condition check for wheels upload and upload step --- .../build_portable_linux_pytorch_wheels.yml | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 5bdde1ff24e..fe466330eff 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -246,23 +246,17 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Configure AWS Credentials - if: always() uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: aws-region: us-east-2 role-to-assume: arn:aws:iam::692859939525:role/therock-${{ inputs.release_type }}-releases - - name: Check test result gating - run: | - if [[ "${{ needs.generate_target_to_run.outputs.test_runs_on }}" != "" && "${{ needs.test_pytorch_wheels.result }}" == "failure" ]]; then - echo "::warning::Tests failed. Skipping upload." - echo "upload=false" >> $GITHUB_ENV - else - echo "upload=true" >> $GITHUB_ENV - fi - - - name: Copy PyTorch wheels from staging to release S3 - if: ${{ env.upload == 'true' }} + #TODO: Move upload wheels to a separate upload wheels script with AWS credential config merged + - name: Copy PyTorch wheels from staging to release S3 (only if all checks pass) + if: > + ${{ needs.build_pytorch_wheels.result == 'success' && + (needs.generate_target_to_run.outputs.test_runs_on == '' || + needs.test_pytorch_wheels.result != 'failure') }} run: | echo "Copying exact tested wheels to release S3 bucket..." aws s3 cp \ @@ -275,8 +269,12 @@ jobs: --include "torchvision-${TORCHVISION_VERSION}-${CP_VERSION}-linux_x86_64.whl" \ --include "pytorch_triton_rocm-${TRITON_VERSION}-${CP_VERSION}-linux_x86_64.whl" + - name: (Re-)Generate Python package release index - if: ${{ env.upload == 'true' }} + if: > + ${{ needs.build_pytorch_wheels.result == 'success' && + (needs.generate_target_to_run.outputs.test_runs_on == '' || + needs.test_pytorch_wheels.result != 'failure') }} run: | pip install boto3 packaging python ./build_tools/third_party/s3_management/manage.py ${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }} From c0ed8f14231f4c904425d492f07a1536ecea8cd4 Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 12 Aug 2025 16:42:52 +0000 Subject: [PATCH 46/57] Revert "Merged condition check for wheels upload and upload step" This reverts commit 30dc4515f546e10ac0dd432a1e1f68c327e927a2. --- .../build_portable_linux_pytorch_wheels.yml | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index fe466330eff..5bdde1ff24e 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -246,17 +246,23 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Configure AWS Credentials + if: always() uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: aws-region: us-east-2 role-to-assume: arn:aws:iam::692859939525:role/therock-${{ inputs.release_type }}-releases - #TODO: Move upload wheels to a separate upload wheels script with AWS credential config merged - - name: Copy PyTorch wheels from staging to release S3 (only if all checks pass) - if: > - ${{ needs.build_pytorch_wheels.result == 'success' && - (needs.generate_target_to_run.outputs.test_runs_on == '' || - needs.test_pytorch_wheels.result != 'failure') }} + - name: Check test result gating + run: | + if [[ "${{ needs.generate_target_to_run.outputs.test_runs_on }}" != "" && "${{ needs.test_pytorch_wheels.result }}" == "failure" ]]; then + echo "::warning::Tests failed. Skipping upload." + echo "upload=false" >> $GITHUB_ENV + else + echo "upload=true" >> $GITHUB_ENV + fi + + - name: Copy PyTorch wheels from staging to release S3 + if: ${{ env.upload == 'true' }} run: | echo "Copying exact tested wheels to release S3 bucket..." aws s3 cp \ @@ -269,12 +275,8 @@ jobs: --include "torchvision-${TORCHVISION_VERSION}-${CP_VERSION}-linux_x86_64.whl" \ --include "pytorch_triton_rocm-${TRITON_VERSION}-${CP_VERSION}-linux_x86_64.whl" - - name: (Re-)Generate Python package release index - if: > - ${{ needs.build_pytorch_wheels.result == 'success' && - (needs.generate_target_to_run.outputs.test_runs_on == '' || - needs.test_pytorch_wheels.result != 'failure') }} + if: ${{ env.upload == 'true' }} run: | pip install boto3 packaging python ./build_tools/third_party/s3_management/manage.py ${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }} From 8d6b63ec6fab9eb6d7678ce5933fd9e144207d85 Mon Sep 17 00:00:00 2001 From: arravikum Date: Tue, 12 Aug 2025 17:13:04 +0000 Subject: [PATCH 47/57] Added needs.pytorch_build_wheels.results to upload check --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 5bdde1ff24e..c18f7de4330 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -254,7 +254,7 @@ jobs: - name: Check test result gating run: | - if [[ "${{ needs.generate_target_to_run.outputs.test_runs_on }}" != "" && "${{ needs.test_pytorch_wheels.result }}" == "failure" ]]; then + if [[ "${{ needs.generate_target_to_run.outputs.test_runs_on }}" != "" && "${{ needs.test_pytorch_wheels.result }}" == "failure" && "${{ needs.build_pytorch_wheels.result }}" != "success" ]]; then echo "::warning::Tests failed. Skipping upload." echo "upload=false" >> $GITHUB_ENV else From 1aa14739995e992c581402c96511f7c4050adfc2 Mon Sep 17 00:00:00 2001 From: arravikum Date: Wed, 13 Aug 2025 14:46:59 +0000 Subject: [PATCH 48/57] Added a condition to skip upload even if needs.test_pytorch_wheels.result is skipped if test_runs_on is populated --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index c18f7de4330..b1a0a710acf 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -254,13 +254,16 @@ jobs: - name: Check test result gating run: | - if [[ "${{ needs.generate_target_to_run.outputs.test_runs_on }}" != "" && "${{ needs.test_pytorch_wheels.result }}" == "failure" && "${{ needs.build_pytorch_wheels.result }}" != "success" ]]; then - echo "::warning::Tests failed. Skipping upload." + if [[ "${{ needs.generate_target_to_run.outputs.test_runs_on }}" != "" \ + && ( "${{ needs.test_pytorch_wheels.result }}" == "failure" || "${{ needs.test_pytorch_wheels.result }}" == "skipped" ) \ + && "${{ needs.build_pytorch_wheels.result }}" != "success" ]]; then + echo "::warning::Tests failed or skipped. Skipping upload." echo "upload=false" >> $GITHUB_ENV else echo "upload=true" >> $GITHUB_ENV fi + - name: Copy PyTorch wheels from staging to release S3 if: ${{ env.upload == 'true' }} run: | From be258df1e5074082e396a35c2247f13925b786bf Mon Sep 17 00:00:00 2001 From: arravikum Date: Thu, 21 Aug 2025 14:15:53 +0000 Subject: [PATCH 49/57] This is a dummy commit From cb3f485b9901c691d53464317eeec99c5cd91c3c Mon Sep 17 00:00:00 2001 From: arravikum Date: Fri, 22 Aug 2025 15:15:06 +0000 Subject: [PATCH 50/57] Removing gating check step in job, enforcing promotion of wheels to v2 only if tests pass even for missing CI runners case --- .../build_portable_linux_pytorch_wheels.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index b1a0a710acf..ac822169d04 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -231,7 +231,6 @@ jobs: upload_pytorch_wheels: name: Upload PyTorch Wheels to S3 needs: [build_pytorch_wheels, generate_target_to_run, test_pytorch_wheels] - if: always() runs-on: ubuntu-24.04 env: S3_BUCKET_PY: "therock-${{ inputs.release_type }}-python" @@ -246,26 +245,12 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Configure AWS Credentials - if: always() uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: aws-region: us-east-2 role-to-assume: arn:aws:iam::692859939525:role/therock-${{ inputs.release_type }}-releases - - name: Check test result gating - run: | - if [[ "${{ needs.generate_target_to_run.outputs.test_runs_on }}" != "" \ - && ( "${{ needs.test_pytorch_wheels.result }}" == "failure" || "${{ needs.test_pytorch_wheels.result }}" == "skipped" ) \ - && "${{ needs.build_pytorch_wheels.result }}" != "success" ]]; then - echo "::warning::Tests failed or skipped. Skipping upload." - echo "upload=false" >> $GITHUB_ENV - else - echo "upload=true" >> $GITHUB_ENV - fi - - - name: Copy PyTorch wheels from staging to release S3 - if: ${{ env.upload == 'true' }} run: | echo "Copying exact tested wheels to release S3 bucket..." aws s3 cp \ @@ -279,7 +264,6 @@ jobs: --include "pytorch_triton_rocm-${TRITON_VERSION}-${CP_VERSION}-linux_x86_64.whl" - name: (Re-)Generate Python package release index - if: ${{ env.upload == 'true' }} run: | pip install boto3 packaging python ./build_tools/third_party/s3_management/manage.py ${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }} From c4dc878b5833e4fc5e903243dc1bfa6b2ace2d8b Mon Sep 17 00:00:00 2001 From: arravikum Date: Mon, 25 Aug 2025 18:56:52 +0000 Subject: [PATCH 51/57] Revert "Removing gating check step in job, enforcing promotion of wheels to v2 only if tests pass even for missing CI runners case" This reverts commit bc9d2097f640bd902903adfb8fe764a012b4c50b. --- .../build_portable_linux_pytorch_wheels.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index ac822169d04..b1a0a710acf 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -231,6 +231,7 @@ jobs: upload_pytorch_wheels: name: Upload PyTorch Wheels to S3 needs: [build_pytorch_wheels, generate_target_to_run, test_pytorch_wheels] + if: always() runs-on: ubuntu-24.04 env: S3_BUCKET_PY: "therock-${{ inputs.release_type }}-python" @@ -245,12 +246,26 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Configure AWS Credentials + if: always() uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: aws-region: us-east-2 role-to-assume: arn:aws:iam::692859939525:role/therock-${{ inputs.release_type }}-releases + - name: Check test result gating + run: | + if [[ "${{ needs.generate_target_to_run.outputs.test_runs_on }}" != "" \ + && ( "${{ needs.test_pytorch_wheels.result }}" == "failure" || "${{ needs.test_pytorch_wheels.result }}" == "skipped" ) \ + && "${{ needs.build_pytorch_wheels.result }}" != "success" ]]; then + echo "::warning::Tests failed or skipped. Skipping upload." + echo "upload=false" >> $GITHUB_ENV + else + echo "upload=true" >> $GITHUB_ENV + fi + + - name: Copy PyTorch wheels from staging to release S3 + if: ${{ env.upload == 'true' }} run: | echo "Copying exact tested wheels to release S3 bucket..." aws s3 cp \ @@ -264,6 +279,7 @@ jobs: --include "pytorch_triton_rocm-${TRITON_VERSION}-${CP_VERSION}-linux_x86_64.whl" - name: (Re-)Generate Python package release index + if: ${{ env.upload == 'true' }} run: | pip install boto3 packaging python ./build_tools/third_party/s3_management/manage.py ${{ inputs.s3_subdir }}/${{ inputs.amdgpu_family }} From 91e4976a003f9bb62de31727f9a8170fb836d8d8 Mon Sep 17 00:00:00 2001 From: arravikum Date: Mon, 25 Aug 2025 19:14:59 +0000 Subject: [PATCH 52/57] Re-worked upload logic to be more readable --- .../build_portable_linux_pytorch_wheels.yml | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index b1a0a710acf..8278f98ecc4 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -252,18 +252,27 @@ jobs: aws-region: us-east-2 role-to-assume: arn:aws:iam::692859939525:role/therock-${{ inputs.release_type }}-releases - - name: Check test result gating + - name: Determine upload flag + env: + BUILD_RESULT: ${{ needs.build_pytorch_wheels.result }} + TEST_RESULT: ${{ needs.test_pytorch_wheels.result }} + TEST_RUNS_ON: ${{ needs.generate_target_to_run.outputs.test_runs_on }} run: | - if [[ "${{ needs.generate_target_to_run.outputs.test_runs_on }}" != "" \ - && ( "${{ needs.test_pytorch_wheels.result }}" == "failure" || "${{ needs.test_pytorch_wheels.result }}" == "skipped" ) \ - && "${{ needs.build_pytorch_wheels.result }}" != "success" ]]; then - echo "::warning::Tests failed or skipped. Skipping upload." - echo "upload=false" >> $GITHUB_ENV + # 1) If the build failed → upload=false + if [[ "$BUILD_RESULT" != "success" ]]; then + echo "::warning::Build failed. Skipping upload." + echo "upload=false" >> "$GITHUB_ENV" + + # 2) Else if there was a test runner AND tests failed or were skipped → upload=false + elif [[ -n "$TEST_RUNS_ON" && ( "$TEST_RESULT" == "failure" || "$TEST_RESULT" == "skipped" ) ]]; then + echo "::warning::Tests failed or were skipped (runner present). Skipping upload." + echo "upload=false" >> "$GITHUB_ENV" + + # 3) Else → upload=true else - echo "upload=true" >> $GITHUB_ENV + echo "upload=true" >> "$GITHUB_ENV" fi - - name: Copy PyTorch wheels from staging to release S3 if: ${{ env.upload == 'true' }} run: | From 84b034eeb4da4ac57f242658a36fddf45f54a578 Mon Sep 17 00:00:00 2001 From: arravikum Date: Wed, 27 Aug 2025 21:02:31 +0000 Subject: [PATCH 53/57] Added upload_always flag for arch families that dont have a runner as an override and utilizing the same to decide upload to v2 --- .../build_portable_linux_pytorch_wheels.yml | 11 ++++-- .../github_actions/amdgpu_family_matrix.py | 17 +++------ .../github_actions/configure_target_run.py | 35 +++++++++++++++++++ 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index 8278f98ecc4..d41d281b5e5 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -143,7 +143,7 @@ jobs: ./external-builds/pytorch/pytorch_audio_repo.py checkout --repo-hashtag nightly ./external-builds/pytorch/pytorch_vision_repo.py checkout --repo-hashtag nightly ./external-builds/pytorch/pytorch_triton_repo.py checkout --patch --patchset nightly - + # Then the else branch to checkout release version of source code instead. - name: Checkout PyTorch Source Repos from stable branch if: ${{ inputs.pytorch_version != 'nightly' }} @@ -252,11 +252,13 @@ jobs: aws-region: us-east-2 role-to-assume: arn:aws:iam::692859939525:role/therock-${{ inputs.release_type }}-releases + - name: Determine upload flag env: BUILD_RESULT: ${{ needs.build_pytorch_wheels.result }} TEST_RESULT: ${{ needs.test_pytorch_wheels.result }} TEST_RUNS_ON: ${{ needs.generate_target_to_run.outputs.test_runs_on }} + ALWAYS_UPLOAD: ${{ needs.generate_target_to_run.outputs.always_upload }} run: | # 1) If the build failed → upload=false if [[ "$BUILD_RESULT" != "success" ]]; then @@ -268,7 +270,12 @@ jobs: echo "::warning::Tests failed or were skipped (runner present). Skipping upload." echo "upload=false" >> "$GITHUB_ENV" - # 3) Else → upload=true + # 3) Else if ALWAYS_UPLOAD is not set and there was no test runner → upload=false + elif [[ -z "$ALWAYS_UPLOAD" && -z "$TEST_RUNS_ON" ]]; then + echo "::warning::No test runner and ALWAYS_UPLOAD not set. Skipping upload." + echo "upload=false" >> "$GITHUB_ENV" + + # 4) Otherwise → upload=true else echo "upload=true" >> "$GITHUB_ENV" fi diff --git a/build_tools/github_actions/amdgpu_family_matrix.py b/build_tools/github_actions/amdgpu_family_matrix.py index 57116572eb9..bda9f1f3d90 100644 --- a/build_tools/github_actions/amdgpu_family_matrix.py +++ b/build_tools/github_actions/amdgpu_family_matrix.py @@ -10,13 +10,11 @@ } }, "gfx110x": { - "linux": { - "test-runs-on": "", - "family": "gfx110X-dgpu", - }, + "linux": {"test-runs-on": "", "family": "gfx110X-dgpu", "always_upload": True}, "windows": { "test-runs-on": "", "family": "gfx110X-dgpu", + "always_upload": True, }, }, } @@ -29,10 +27,7 @@ } }, "gfx115x": { - "linux": { - "test-runs-on": "", - "family": "gfx1151", - }, + "linux": {"test-runs-on": "", "family": "gfx1151", "always_upload": True}, "windows": { "test-runs-on": "windows-strix-halo-gpu-rocm", "family": "gfx1151", @@ -42,11 +37,9 @@ "linux": { "test-runs-on": "", # removed due to machine issues, label is "linux-rx9070-gpu-rocm" "family": "gfx120X-all", + "always_upload": True, }, - "windows": { - "test-runs-on": "", - "family": "gfx120X-all", - }, + "windows": {"test-runs-on": "", "family": "gfx120X-all", "always_upload": True}, }, } diff --git a/build_tools/github_actions/configure_target_run.py b/build_tools/github_actions/configure_target_run.py index 65f662ea0dc..83cba0954d7 100644 --- a/build_tools/github_actions/configure_target_run.py +++ b/build_tools/github_actions/configure_target_run.py @@ -47,6 +47,38 @@ def get_runner_label(target: str, platform: str) -> str: print(f" Found runner: '{test_runs_on_machine}'") return test_runs_on_machine + +def get_upload_label(target: str, platform: str) -> str: + print(f"Searching for a runner for target '{target}' on platform '{platform}'") + amdgpu_family_info_matrix = ( + amdgpu_family_info_matrix_presubmit | amdgpu_family_info_matrix_postsubmit + ) + for key, info_for_key in amdgpu_family_info_matrix.items(): + print(f"Cheecking key '{key}' with info:\n {info_for_key}") + platform_for_key = info_for_key.get(platform) + + if not platform_for_key: + # Some AMDGPU families are only supported on certain platforms. + print(f" Skipping since this entry has no platform '{platform}'") + continue + + # Check against both the inner "family" and the outer "key". If neither + # match then skip. Workflows are expected to use the inner "family" + # but manually triggered runs may use the outer "key" instead, so we'll + # be a bit lenient here. + # This needs a rework, see https://github.com/ROCm/TheRock/issues/1097. + family_for_platform = platform_for_key.get("family") + if target != family_for_platform and key not in target.lower(): + print( + f" Skipping since the target '{target}' does not match the family '{family_for_platform}'" + ) + continue + + # If there is no test machine available and always_upload flag is True for GPU family and platform, output always_upload as True + always_upload = platform_for_key.get("always_upload") + if always_upload: + print(f" always_upload: True") + return always_upload return "" @@ -54,6 +86,9 @@ def main(target: str, platform: str): runner_label = get_runner_label(target, platform) if runner_label: gha_set_output({"test-runs-on": runner_label}) + upload_label = get_upload_label(target, platform) + if upload_label: + gha_set_output({"always_upload": upload_label}) if __name__ == "__main__": From d7d4e0106ee0d0bb71e21865017cd9b9a961c8f3 Mon Sep 17 00:00:00 2001 From: arravikum Date: Wed, 27 Aug 2025 21:41:35 +0000 Subject: [PATCH 54/57] Adding always_upload in ouptuts for generate_target_to_run job --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index d41d281b5e5..f3afcb9b717 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -204,6 +204,7 @@ jobs: runs-on: ubuntu-24.04 outputs: test_runs_on: ${{ steps.configure.outputs.test-runs-on }} + always_upload: ${{ steps.configure.outputs.always_upload }} steps: - name: Checking out repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 From a68c3ded2fd2453ad33f6fe6120a8c2981105910 Mon Sep 17 00:00:00 2001 From: arravikum Date: Thu, 28 Aug 2025 00:34:01 +0000 Subject: [PATCH 55/57] Addressing PR comments, changed 'always_upload' flag to 'bypass_tests_for_releases' --- .../build_portable_linux_pytorch_wheels.yml | 12 ++++++------ .../github_actions/amdgpu_family_matrix.py | 18 +++++++++++++++--- .../github_actions/configure_target_run.py | 12 ++++++------ 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index f3afcb9b717..c3c38db8d0a 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -130,7 +130,7 @@ jobs: if ! [ -x "${python_dir}/bin/python" ]; then echo "ERROR: Could not find python: ${python_dir}" exit 1 - fi + fi echo "${python_dir}/bin" >> "$GITHUB_PATH" # Github workflows do not have if/else and therefore we define two jobs with different if clauses. @@ -204,7 +204,7 @@ jobs: runs-on: ubuntu-24.04 outputs: test_runs_on: ${{ steps.configure.outputs.test-runs-on }} - always_upload: ${{ steps.configure.outputs.always_upload }} + bypass_tests_for_releases: ${{ steps.configure.outputs.bypass_tests_for_releases }} steps: - name: Checking out repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -259,7 +259,7 @@ jobs: BUILD_RESULT: ${{ needs.build_pytorch_wheels.result }} TEST_RESULT: ${{ needs.test_pytorch_wheels.result }} TEST_RUNS_ON: ${{ needs.generate_target_to_run.outputs.test_runs_on }} - ALWAYS_UPLOAD: ${{ needs.generate_target_to_run.outputs.always_upload }} + BYPASS_TESTS_FOR_RELEASES: ${{ needs.generate_target_to_run.outputs.bypass_tests_for_releases }} run: | # 1) If the build failed → upload=false if [[ "$BUILD_RESULT" != "success" ]]; then @@ -271,9 +271,9 @@ jobs: echo "::warning::Tests failed or were skipped (runner present). Skipping upload." echo "upload=false" >> "$GITHUB_ENV" - # 3) Else if ALWAYS_UPLOAD is not set and there was no test runner → upload=false - elif [[ -z "$ALWAYS_UPLOAD" && -z "$TEST_RUNS_ON" ]]; then - echo "::warning::No test runner and ALWAYS_UPLOAD not set. Skipping upload." + # 3) Else if BYPASS_TESTS_FOR_RELEASES is not set and there was no test runner → upload=false + elif [[ -z "$BYPASS_TESTS_FOR_RELEASES" && -z "$TEST_RUNS_ON" ]]; then + echo "::warning::No test runner and BYPASS_TESTS_FOR_RELEASES not set. Skipping upload." echo "upload=false" >> "$GITHUB_ENV" # 4) Otherwise → upload=true diff --git a/build_tools/github_actions/amdgpu_family_matrix.py b/build_tools/github_actions/amdgpu_family_matrix.py index bda9f1f3d90..9a8d9ea3613 100644 --- a/build_tools/github_actions/amdgpu_family_matrix.py +++ b/build_tools/github_actions/amdgpu_family_matrix.py @@ -10,7 +10,11 @@ } }, "gfx110x": { - "linux": {"test-runs-on": "", "family": "gfx110X-dgpu", "always_upload": True}, + "linux": { + "test-runs-on": "", + "family": "gfx110X-dgpu", + "always_upload": True, + }, "windows": { "test-runs-on": "", "family": "gfx110X-dgpu", @@ -27,7 +31,11 @@ } }, "gfx115x": { - "linux": {"test-runs-on": "", "family": "gfx1151", "always_upload": True}, + "linux": { + "test-runs-on": "", + "family": "gfx1151", + "always_upload": True, + }, "windows": { "test-runs-on": "windows-strix-halo-gpu-rocm", "family": "gfx1151", @@ -39,7 +47,11 @@ "family": "gfx120X-all", "always_upload": True, }, - "windows": {"test-runs-on": "", "family": "gfx120X-all", "always_upload": True}, + "windows": { + "test-runs-on": "", + "family": "gfx120X-all", + "always_upload": True, + }, }, } diff --git a/build_tools/github_actions/configure_target_run.py b/build_tools/github_actions/configure_target_run.py index 83cba0954d7..725b9a8a6cc 100644 --- a/build_tools/github_actions/configure_target_run.py +++ b/build_tools/github_actions/configure_target_run.py @@ -74,11 +74,11 @@ def get_upload_label(target: str, platform: str) -> str: ) continue - # If there is no test machine available and always_upload flag is True for GPU family and platform, output always_upload as True - always_upload = platform_for_key.get("always_upload") - if always_upload: - print(f" always_upload: True") - return always_upload + # If there is no test machine available and bypass_tests_for_releases flag is True for GPU family and platform, output bypass_tests_for_releases as True + bypass_tests_for_releases = platform_for_key.get("bypass_tests_for_releases") + if bypass_tests_for_releases: + print(f" bypass_tests_for_releases: True") + return bypass_tests_for_releases return "" @@ -88,7 +88,7 @@ def main(target: str, platform: str): gha_set_output({"test-runs-on": runner_label}) upload_label = get_upload_label(target, platform) if upload_label: - gha_set_output({"always_upload": upload_label}) + gha_set_output({"bypass_tests_for_releases": upload_label}) if __name__ == "__main__": From 2f8f638fe0384a46b3b85d7dacd772da75e16d7f Mon Sep 17 00:00:00 2001 From: arravikum Date: Thu, 28 Aug 2025 02:08:45 +0000 Subject: [PATCH 56/57] Updated 'bypass_tests_for_releases' in amdgpu_family_matrix.py --- build_tools/github_actions/amdgpu_family_matrix.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build_tools/github_actions/amdgpu_family_matrix.py b/build_tools/github_actions/amdgpu_family_matrix.py index 9a8d9ea3613..d9ef80304b7 100644 --- a/build_tools/github_actions/amdgpu_family_matrix.py +++ b/build_tools/github_actions/amdgpu_family_matrix.py @@ -13,12 +13,12 @@ "linux": { "test-runs-on": "", "family": "gfx110X-dgpu", - "always_upload": True, + "bypass_tests_for_releases": True, }, "windows": { "test-runs-on": "", "family": "gfx110X-dgpu", - "always_upload": True, + "bypass_tests_for_releases": True, }, }, } @@ -34,7 +34,7 @@ "linux": { "test-runs-on": "", "family": "gfx1151", - "always_upload": True, + "bypass_tests_for_releases": True, }, "windows": { "test-runs-on": "windows-strix-halo-gpu-rocm", @@ -45,12 +45,12 @@ "linux": { "test-runs-on": "", # removed due to machine issues, label is "linux-rx9070-gpu-rocm" "family": "gfx120X-all", - "always_upload": True, + "bypass_tests_for_releases": True, }, "windows": { "test-runs-on": "", "family": "gfx120X-all", - "always_upload": True, + "bypass_tests_for_releases": True, }, }, } From 576595ceb049303ed15755d590ffced6b75b4824 Mon Sep 17 00:00:00 2001 From: arravikum Date: Thu, 28 Aug 2025 13:21:57 +0000 Subject: [PATCH 57/57] Adding missed return '' in configure_target_run.py and updated job name --- .github/workflows/build_portable_linux_pytorch_wheels.yml | 2 +- build_tools/github_actions/configure_target_run.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_portable_linux_pytorch_wheels.yml b/.github/workflows/build_portable_linux_pytorch_wheels.yml index c3c38db8d0a..08a60786ac6 100644 --- a/.github/workflows/build_portable_linux_pytorch_wheels.yml +++ b/.github/workflows/build_portable_linux_pytorch_wheels.yml @@ -230,7 +230,7 @@ jobs: pytorch_version: ${{ inputs.pytorch_version }} upload_pytorch_wheels: - name: Upload PyTorch Wheels to S3 + name: Release PyTorch Wheels to S3 needs: [build_pytorch_wheels, generate_target_to_run, test_pytorch_wheels] if: always() runs-on: ubuntu-24.04 diff --git a/build_tools/github_actions/configure_target_run.py b/build_tools/github_actions/configure_target_run.py index 725b9a8a6cc..1e008a2558d 100644 --- a/build_tools/github_actions/configure_target_run.py +++ b/build_tools/github_actions/configure_target_run.py @@ -46,6 +46,7 @@ def get_runner_label(target: str, platform: str) -> str: if test_runs_on_machine: print(f" Found runner: '{test_runs_on_machine}'") return test_runs_on_machine + return "" def get_upload_label(target: str, platform: str) -> str: