Adding monorepo configurations to TheRock

.github/actions/setup_test_environment/action.yml

@@ -15,6 +15,9 @@ inputs:
     type: string
   PLATFORM:
     type: string
+  BUCKET:
+    type: string
+    default: "therock-artifacts"
 
 runs:
   using: "composite"
@@ -56,6 +59,7 @@ runs:
         AMDGPU_FAMILIES: ${{ inputs.AMDGPU_FAMILIES }}
         FETCH_ARTIFACT_ARGS: ${{ inputs.FETCH_ARTIFACT_ARGS }}
         GITHUB_TOKEN: ${{ github.token }}
+        BUCKET: ${{ inputs.BUCKET }}
       run: |
         python ./build_tools/install_rocm_from_artifacts.py \
           --run-id=${ARTIFACT_RUN_ID} \

.github/workflows/build_linux_packages.yml

@@ -11,6 +11,8 @@ on:
       expect_failure:
         type: boolean
         default: false
+      extra_cmake_options:
+        type: string
 
   workflow_call:
     inputs:
@@ -21,6 +23,8 @@ on:
         type: string
       expect_failure:
         type: boolean
+      extra_cmake_options:
+        type: string
 
 # See the details regarding permissions from the link:
 # https://github.com/aws-actions/configure-aws-credentials?tab=readme-ov-file#oidc
@@ -44,9 +48,20 @@ jobs:
       CCACHE_MAXSIZE: "700M"
       AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }}
       TEATIME_FORCE_INTERACTIVE: 0
+      BUCKET: ${{ github.event.repository.name == 'TheRock'  && 'therock-artifacts' || 'therock-artifacts-external' }}
     steps:
       - name: "Checking out repository"
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          repository: "ROCm/TheRock"
+
+      - name: "Checking out repository for rocm-libraries"
+        if: ${{ github.repository == 'ROCm/rocm-libraries' }}
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          repository: "ROCm/rocm-libraries"
+          ref: ${{ github.head_ref || github.ref_name }}
+          path: "rocm-libraries"
 
       - name: Runner Health Settings
         run: |
@@ -85,6 +100,7 @@ jobs:
         run: |
           # Generate a new build id.
           package_version="${{ inputs.package_version }}"
+          extra_cmake_options="${{ inputs.extra_cmake_options }}"
           echo "Building package ${package_version}"
 
           # Build.
@@ -93,10 +109,13 @@ jobs:
             -DTHEROCK_AMDGPU_FAMILIES=${{env.AMDGPU_FAMILIES}} \
             -DTHEROCK_PACKAGE_VERSION="${package_version}" \
             -DTHEROCK_VERBOSE=ON \
-            -DBUILD_TESTING=ON
+            -DBUILD_TESTING=ON \
+            ${extra_cmake_options}
+
           cmake --build build --target therock-archives therock-dist
 
       - name: Test Packaging
+        if: ${{ github.event.repository.name == 'TheRock' }}
         run: |
           ctest --test-dir build --output-on-failure
 
@@ -145,28 +164,28 @@ jobs:
       # TODO: Move to script
       - name: Upload Artifacts
         run: |
-          aws s3 cp build/artifacts/ s3://therock-artifacts/${{github.run_id}}-linux/ \
+          aws s3 cp build/artifacts/ s3://${{env.BUCKET}}/${{github.run_id}}-linux/ \
             --recursive --no-follow-symlinks \
             --exclude "*" \
             --include "*.tar.xz*"
-          aws s3 cp build/artifacts/index.html s3://therock-artifacts/${{github.run_id}}-linux/index-${{env.AMDGPU_FAMILIES}}.html
+          aws s3 cp build/artifacts/index.html s3://${{env.BUCKET}}/${{github.run_id}}-linux/index-${{env.AMDGPU_FAMILIES}}.html
 
       - name: Upload Logs
         if: always()
         run: |
           python3 build_tools/upload_logs_to_s3.py \
             --build-dir=build \
-            --s3-base-path="s3://therock-artifacts/${{github.run_id}}-linux/logs/${{env.AMDGPU_FAMILIES}}"
+            --s3-base-path="s3://${{env.BUCKET}}/${{github.run_id}}-linux/logs/${{env.AMDGPU_FAMILIES}}"
 
       - name: Add Links to Job Summary
         if: always()
         run: |
-          LOG_URL="https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-linux/logs/${{env.AMDGPU_FAMILIES}}/index.html"
+          LOG_URL="https://${{env.BUCKET}}.s3.us-east-2.amazonaws.com/${{github.run_id}}-linux/logs/${{env.AMDGPU_FAMILIES}}/index.html"
           echo "[Build Logs](${LOG_URL})" >> $GITHUB_STEP_SUMMARY
 
           ARTIFACT_INDEX=build/artifacts/index.html
           if [ -f "${ARTIFACT_INDEX}" ]; then
-            ARTIFACT_URL="https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-linux/index-${{env.AMDGPU_FAMILIES}}.html"
+            ARTIFACT_URL="https://${{env.BUCKET}}.s3.us-east-2.amazonaws.com/${{github.run_id}}-linux/index-${{env.AMDGPU_FAMILIES}}.html"
 
             echo "[Artifacts](${ARTIFACT_URL})" >> $GITHUB_STEP_SUMMARY
           else

.github/workflows/test_rocprim.yml

@@ -52,6 +52,8 @@ jobs:
     steps:
       - name: Checkout Repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          repository: "ROCm/TheRock"
 
       - name: Run setup test environment workflow
         uses: './.github/actions/setup_test_environment'
@@ -62,6 +64,7 @@ jobs:
           VENV_DIR: ${{ env.VENV_DIR }}
           FETCH_ARTIFACT_ARGS: "--prim --tests"
           PLATFORM: ${{ inputs.platform }}
+          BUCKET: ${{ github.repository == 'ROCm/TheRock'  && 'therock-artifacts' || 'therock-artifacts-external' }}
 
       - name: Run rocprim tests
         env:

build_tools/fetch_artifacts.py

@@ -6,6 +6,7 @@
 # but those artifacts may not have all required dependencies.
 
 import argparse
+import os
 import platform
 import shutil
 import subprocess
@@ -14,6 +15,8 @@
 GENERIC_VARIANT = "generic"
 PLATFORM = platform.system().lower()
 
+BUCKET = os.getenv("BUCKET", "therock-artifacts")
+
 
 def log(*args, **kwargs):
     print(*args, **kwargs)
@@ -32,7 +35,7 @@ def s3_bucket_exists(run_id):
         "aws",
         "s3",
         "ls",
-        f"s3://therock-artifacts/{run_id}-{PLATFORM}",
+        f"s3://{BUCKET}/{run_id}-{PLATFORM}",
         "--no-sign-request",
     ]
     process = subprocess.run(cmd, check=False, stdout=subprocess.DEVNULL)
@@ -45,7 +48,7 @@ def s3_exec(variant, package, run_id, build_dir):
         "aws",
         "s3",
         "cp",
-        f"s3://therock-artifacts/{run_id}-{PLATFORM}/{package}_{variant}.tar.xz",
+        f"s3://{BUCKET}/{run_id}-{PLATFORM}/{package}_{variant}.tar.xz",
         str(build_dir),
         "--no-sign-request",
     ]

build_tools/github_actions/monorepo_configure_ci.py

@@ -0,0 +1,26 @@
+"""
+This script determines which build flag and tests to run based on SUBTREES
+
+Required environment variables:
+  - SUBTREES
+"""
+
+from configure_ci import set_github_output
+import json
+from monorepo_map import monorepo_map
+import os
+
+SUBTREES = os.getenv("SUBTREES", "")
+
+
+def run():
+    subtrees = SUBTREES.split("\n")
+    jobs = []
+    for subtree in subtrees:
+        if subtree in monorepo_map:
+            jobs.append(monorepo_map.get(subtree))
+    set_github_output({"jobs": json.dumps(jobs)})
+
+
+if __name__ == "__main__":
+    run()

build_tools/github_actions/monorepo_map.py

@@ -0,0 +1,9 @@
+"""
+This dictionary is used to map specific file directory changes to the corresponding build flag and tests
+"""
+monorepo_map = {
+    "projects/rocprim": {
+        "flag": "-DTHEROCK_ENABLE_PRIM=ON -DTHEROCK_ENABLE_ALL=OFF",
+        "test": "test_rocprim",
+    }
+}