diff --git a/.github/actions/setup_test_environment/action.yml b/.github/actions/setup_test_environment/action.yml index b3cc6323b13..eebf3604c12 100644 --- a/.github/actions/setup_test_environment/action.yml +++ b/.github/actions/setup_test_environment/action.yml @@ -15,6 +15,9 @@ inputs: type: string PLATFORM: type: string + BUCKET: + type: string + default: "therock-artifacts" runs: using: "composite" @@ -56,6 +59,7 @@ runs: AMDGPU_FAMILIES: ${{ inputs.AMDGPU_FAMILIES }} FETCH_ARTIFACT_ARGS: ${{ inputs.FETCH_ARTIFACT_ARGS }} GITHUB_TOKEN: ${{ github.token }} + BUCKET: ${{ inputs.BUCKET }} run: | python ./build_tools/install_rocm_from_artifacts.py \ --run-id=${ARTIFACT_RUN_ID} \ diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml index 1e41dd736ec..ff5a674d4f3 100644 --- a/.github/workflows/build_linux_packages.yml +++ b/.github/workflows/build_linux_packages.yml @@ -11,6 +11,8 @@ on: expect_failure: type: boolean default: false + extra_cmake_options: + type: string workflow_call: inputs: @@ -21,6 +23,8 @@ on: type: string expect_failure: type: boolean + extra_cmake_options: + type: string # See the details regarding permissions from the link: # https://github.com/aws-actions/configure-aws-credentials?tab=readme-ov-file#oidc @@ -44,9 +48,20 @@ jobs: CCACHE_MAXSIZE: "700M" AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} TEATIME_FORCE_INTERACTIVE: 0 + BUCKET: ${{ github.event.repository.name == 'TheRock' && 'therock-artifacts' || 'therock-artifacts-external' }} steps: - name: "Checking out repository" uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: "ROCm/TheRock" + + - name: "Checking out repository for rocm-libraries" + if: ${{ github.repository == 'ROCm/rocm-libraries' }} + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: "ROCm/rocm-libraries" + ref: ${{ github.head_ref || github.ref_name }} + path: "rocm-libraries" - name: Runner Health Settings run: | @@ -85,6 +100,7 @@ jobs: run: | # Generate a new build id. package_version="${{ inputs.package_version }}" + extra_cmake_options="${{ inputs.extra_cmake_options }}" echo "Building package ${package_version}" # Build. @@ -93,10 +109,13 @@ jobs: -DTHEROCK_AMDGPU_FAMILIES=${{env.AMDGPU_FAMILIES}} \ -DTHEROCK_PACKAGE_VERSION="${package_version}" \ -DTHEROCK_VERBOSE=ON \ - -DBUILD_TESTING=ON + -DBUILD_TESTING=ON \ + ${extra_cmake_options} + cmake --build build --target therock-archives therock-dist - name: Test Packaging + if: ${{ github.event.repository.name == 'TheRock' }} run: | ctest --test-dir build --output-on-failure @@ -145,28 +164,28 @@ jobs: # TODO: Move to script - name: Upload Artifacts run: | - aws s3 cp build/artifacts/ s3://therock-artifacts/${{github.run_id}}-linux/ \ + aws s3 cp build/artifacts/ s3://${{env.BUCKET}}/${{github.run_id}}-linux/ \ --recursive --no-follow-symlinks \ --exclude "*" \ --include "*.tar.xz*" - aws s3 cp build/artifacts/index.html s3://therock-artifacts/${{github.run_id}}-linux/index-${{env.AMDGPU_FAMILIES}}.html + aws s3 cp build/artifacts/index.html s3://${{env.BUCKET}}/${{github.run_id}}-linux/index-${{env.AMDGPU_FAMILIES}}.html - name: Upload Logs if: always() run: | python3 build_tools/upload_logs_to_s3.py \ --build-dir=build \ - --s3-base-path="s3://therock-artifacts/${{github.run_id}}-linux/logs/${{env.AMDGPU_FAMILIES}}" + --s3-base-path="s3://${{env.BUCKET}}/${{github.run_id}}-linux/logs/${{env.AMDGPU_FAMILIES}}" - name: Add Links to Job Summary if: always() run: | - LOG_URL="https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-linux/logs/${{env.AMDGPU_FAMILIES}}/index.html" + LOG_URL="https://${{env.BUCKET}}.s3.us-east-2.amazonaws.com/${{github.run_id}}-linux/logs/${{env.AMDGPU_FAMILIES}}/index.html" echo "[Build Logs](${LOG_URL})" >> $GITHUB_STEP_SUMMARY ARTIFACT_INDEX=build/artifacts/index.html if [ -f "${ARTIFACT_INDEX}" ]; then - ARTIFACT_URL="https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-linux/index-${{env.AMDGPU_FAMILIES}}.html" + ARTIFACT_URL="https://${{env.BUCKET}}.s3.us-east-2.amazonaws.com/${{github.run_id}}-linux/index-${{env.AMDGPU_FAMILIES}}.html" echo "[Artifacts](${ARTIFACT_URL})" >> $GITHUB_STEP_SUMMARY else diff --git a/.github/workflows/test_rocprim.yml b/.github/workflows/test_rocprim.yml index bddf193ca9c..c41a7d925db 100644 --- a/.github/workflows/test_rocprim.yml +++ b/.github/workflows/test_rocprim.yml @@ -52,6 +52,8 @@ jobs: steps: - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: "ROCm/TheRock" - name: Run setup test environment workflow uses: './.github/actions/setup_test_environment' @@ -62,6 +64,7 @@ jobs: VENV_DIR: ${{ env.VENV_DIR }} FETCH_ARTIFACT_ARGS: "--prim --tests" PLATFORM: ${{ inputs.platform }} + BUCKET: ${{ github.repository == 'ROCm/TheRock' && 'therock-artifacts' || 'therock-artifacts-external' }} - name: Run rocprim tests env: diff --git a/build_tools/fetch_artifacts.py b/build_tools/fetch_artifacts.py index 096e4e3d388..7e05d2624ff 100644 --- a/build_tools/fetch_artifacts.py +++ b/build_tools/fetch_artifacts.py @@ -6,6 +6,7 @@ # but those artifacts may not have all required dependencies. import argparse +import os import platform import shutil import subprocess @@ -14,6 +15,8 @@ GENERIC_VARIANT = "generic" PLATFORM = platform.system().lower() +BUCKET = os.getenv("BUCKET", "therock-artifacts") + def log(*args, **kwargs): print(*args, **kwargs) @@ -32,7 +35,7 @@ def s3_bucket_exists(run_id): "aws", "s3", "ls", - f"s3://therock-artifacts/{run_id}-{PLATFORM}", + f"s3://{BUCKET}/{run_id}-{PLATFORM}", "--no-sign-request", ] process = subprocess.run(cmd, check=False, stdout=subprocess.DEVNULL) @@ -45,7 +48,7 @@ def s3_exec(variant, package, run_id, build_dir): "aws", "s3", "cp", - f"s3://therock-artifacts/{run_id}-{PLATFORM}/{package}_{variant}.tar.xz", + f"s3://{BUCKET}/{run_id}-{PLATFORM}/{package}_{variant}.tar.xz", str(build_dir), "--no-sign-request", ] diff --git a/build_tools/github_actions/monorepo_configure_ci.py b/build_tools/github_actions/monorepo_configure_ci.py new file mode 100644 index 00000000000..c1180344cfc --- /dev/null +++ b/build_tools/github_actions/monorepo_configure_ci.py @@ -0,0 +1,26 @@ +""" +This script determines which build flag and tests to run based on SUBTREES + +Required environment variables: + - SUBTREES +""" + +from configure_ci import set_github_output +import json +from monorepo_map import monorepo_map +import os + +SUBTREES = os.getenv("SUBTREES", "") + + +def run(): + subtrees = SUBTREES.split("\n") + jobs = [] + for subtree in subtrees: + if subtree in monorepo_map: + jobs.append(monorepo_map.get(subtree)) + set_github_output({"jobs": json.dumps(jobs)}) + + +if __name__ == "__main__": + run() diff --git a/build_tools/github_actions/monorepo_map.py b/build_tools/github_actions/monorepo_map.py new file mode 100644 index 00000000000..37dfb17f5bc --- /dev/null +++ b/build_tools/github_actions/monorepo_map.py @@ -0,0 +1,9 @@ +""" +This dictionary is used to map specific file directory changes to the corresponding build flag and tests +""" +monorepo_map = { + "projects/rocprim": { + "flag": "-DTHEROCK_ENABLE_PRIM=ON -DTHEROCK_ENABLE_ALL=OFF", + "test": "test_rocprim", + } +}