[ci] Reorganizing artifact upload

.github/actions/setup_test_environment/action.yml

@@ -15,9 +15,6 @@ inputs:
     type: string
   PLATFORM:
     type: string
-  BUCKET:
-    type: string
-    default: "therock-artifacts"
 
 runs:
   using: "composite"
@@ -59,7 +56,6 @@ runs:
         AMDGPU_FAMILIES: ${{ inputs.AMDGPU_FAMILIES }}
         FETCH_ARTIFACT_ARGS: ${{ inputs.FETCH_ARTIFACT_ARGS }}
         GITHUB_TOKEN: ${{ github.token }}
-        BUCKET: ${{ inputs.BUCKET }}
       run: |
         python ./build_tools/install_rocm_from_artifacts.py \
           --run-id=${ARTIFACT_RUN_ID} \

.github/workflows/build_linux_packages.yml

@@ -145,48 +145,36 @@ jobs:
           aws-region: us-east-2
           role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts
 
-      # TODO: Move to script
-      - name: Create Index Files
+      - name: Create Logs index Files
         if: always()
         run: |
-          curl --silent --fail --show-error --location \
-            https://raw.githubusercontent.com/joshbrunty/Indexer/6d8cbfd15d3853b482e6a49f2d875ded9188b721/indexer.py \
-            --output build/indexer.py
-          python build/indexer.py -f '*.tar.xz*' build/artifacts/
-          python3 build_tools/create_log_index.py \
+          python3 build_tools/github_actions/create_log_index.py \
             --build-dir=build \
             --amdgpu-family=${{ env.AMDGPU_FAMILIES }}
 
-      # TODO: Move to script
-      - name: Upload Artifacts
+      - name: Upload artifacts
+        if: always()
         run: |
-          aws s3 cp build/artifacts/ s3://${{env.BUCKET}}/${{github.run_id}}-linux/ \
-            --recursive --no-follow-symlinks \
-            --exclude "*" \
-            --include "*.tar.xz*"
-          aws s3 cp build/artifacts/index.html s3://${{env.BUCKET}}/${{github.run_id}}-linux/index-${{env.AMDGPU_FAMILIES}}.html
+          python build_tools/github_actions/upload_build_artifacts.py \
+            --run-id ${{ github.run_id }} \
+            --amdgpu-family ${{ env.AMDGPU_FAMILIES }} \
+            --build-dir build/
 
       - name: Upload Logs
         if: always()
         run: |
-          python3 build_tools/upload_logs_to_s3.py \
+          python3 build_tools/github_actions/upload_build_logs_to_s3.py \
             --build-dir=build \
-            --s3-base-path="s3://${{env.BUCKET}}/${{github.run_id}}-linux/logs/${{env.AMDGPU_FAMILIES}}"
+            --run-id ${{ github.run_id }} \
+            --amdgpu-family ${{ env.AMDGPU_FAMILIES }}
 
       - name: Add Links to Job Summary
         if: always()
         run: |
-          LOG_URL="https://${{env.BUCKET}}.s3.us-east-2.amazonaws.com/${{github.run_id}}-linux/logs/${{env.AMDGPU_FAMILIES}}/index.html"
-          echo "[Build Logs](${LOG_URL})" >> $GITHUB_STEP_SUMMARY
-
-          ARTIFACT_INDEX=build/artifacts/index.html
-          if [ -f "${ARTIFACT_INDEX}" ]; then
-            ARTIFACT_URL="https://${{env.BUCKET}}.s3.us-east-2.amazonaws.com/${{github.run_id}}-linux/index-${{env.AMDGPU_FAMILIES}}.html"
-
-            echo "[Artifacts](${ARTIFACT_URL})" >> $GITHUB_STEP_SUMMARY
-          else
-            echo "[INFO] No artifacts index found. Skipping artifact link."
-          fi
+          python build_tools/github_actions/upload_build_summary.py \
+            --run-id ${{ github.run_id }} \
+            --amdgpu-family ${{ env.AMDGPU_FAMILIES }} \
+            --build-dir build/
 
       - name: Save cache
         uses: actions/cache/save@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2

.github/workflows/build_windows_packages.yml

@@ -199,52 +199,36 @@ jobs:
           aws-region: us-east-2
           role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts
 
-      # TODO: Move to script
-      - name: Create Index Files
+      - name: Create Logs index Files
         if: always()
         run: |
-          curl --silent --fail --show-error --location \
-            https://raw.githubusercontent.com/joshbrunty/Indexer/6d8cbfd15d3853b482e6a49f2d875ded9188b721/indexer.py \
-            --output ${{ env.BUILD_DIR_BASH }}/indexer.py
-          python ${{ env.BUILD_DIR_BASH }}/indexer.py -f '*.tar.xz*' ${{ env.BUILD_DIR_BASH }}/artifacts/
-          python build_tools/create_log_index.py \
+          python3 build_tools/github_actions/create_log_index.py \
             --build-dir=${{ env.BUILD_DIR_BASH }} \
             --amdgpu-family=${{ env.AMDGPU_FAMILIES }}
 
-      # TODO: Move to script
-      - name: Upload Artifacts
-        shell: powershell
+      - name: Upload artifacts
+        if: always()
         run: |
-          $Env:PATH += ";C:\Program Files\Amazon\AWSCLIV2"
-          aws s3 cp ${{ env.BASE_BUILD_DIR_POWERSHELL }}\artifacts s3://therock-artifacts/${{github.run_id}}-windows/ `
-            --recursive --no-follow-symlinks `
-            --exclude "*" `
-            --include "*.tar.xz*"
-          aws s3 cp ${{ env.BASE_BUILD_DIR_POWERSHELL }}\artifacts\index.html s3://therock-artifacts/${{github.run_id}}-windows/index-${{env.AMDGPU_FAMILIES}}.html
+          python build_tools/github_actions/upload_build_artifacts.py \
+            --run-id ${{ github.run_id }} \
+            --amdgpu-family ${{ env.AMDGPU_FAMILIES }} \
+            --build-dir ${{ env.BUILD_DIR_BASH }}
 
       - name: Upload Logs
         if: always()
-        shell: powershell
         run: |
-          $Env:PATH += ";C:\Program Files\Amazon\AWSCLIV2"
-          python3 build_tools/upload_logs_to_s3.py `
-            --build-dir=${{ env.BASE_BUILD_DIR_POWERSHELL }} `
-            --s3-base-path="s3://therock-artifacts/${{github.run_id}}-windows/logs/${{env.AMDGPU_FAMILIES}}"
+          python3 build_tools/github_actions/upload_build_logs_to_s3.py \
+            --build-dir=${{ env.BUILD_DIR_BASH }} \
+            --run-id ${{ github.run_id }} \
+            --amdgpu-family ${{ env.AMDGPU_FAMILIES }}
 
       - name: Add Links to Job Summary
         if: always()
         run: |
-          LOG_URL="https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-windows/logs/${{env.AMDGPU_FAMILIES}}/index.html"
-          echo "[Build Logs](${LOG_URL})" >> $GITHUB_STEP_SUMMARY
-
-          ARTIFACT_INDEX="${{ env.BASE_BUILD_DIR_POWERSHELL }}/artifacts/index.html"
-          if [ -f "${ARTIFACT_INDEX}" ]; then
-            ARTIFACT_URL="https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-windows/index-${{env.AMDGPU_FAMILIES}}.html"
-
-            echo "[Artifacts](${ARTIFACT_URL})" >> $GITHUB_STEP_SUMMARY
-          else
-            echo "[INFO] No artifacts index found. Skipping artifact link."
-          fi
+          python build_tools/github_actions/upload_build_summary.py \
+            --run-id ${{ github.run_id }} \
+            --amdgpu-family ${{ env.AMDGPU_FAMILIES }} \
+            --build-dir ${{ env.BUILD_DIR_BASH }}
 
       - name: Save cache
         uses: actions/cache/save@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2

.github/workflows/test_rocprim.yml

@@ -64,7 +64,6 @@ jobs:
           VENV_DIR: ${{ env.VENV_DIR }}
           FETCH_ARTIFACT_ARGS: "--prim --tests"
           PLATFORM: ${{ inputs.platform }}
-          BUCKET: ${{ github.repository == 'ROCm/TheRock'  && 'therock-artifacts' || 'therock-artifacts-external' }}
 
       - name: Run rocprim tests
         env:

build_tools/fetch_artifacts.py

@@ -8,14 +8,20 @@
 import argparse
 import concurrent.futures
 from html.parser import HTMLParser
+from pathlib import Path
 import platform
 from shutil import copyfileobj
 import sys
 import urllib.request
 
+THEROCK_DIR = Path(__file__).resolve().parent.parent
+
+# Importing build_artifact_upload.py
+sys.path.append(str(THEROCK_DIR / "build_tools" / "github_actions"))
+from upload_build_artifacts import retrieve_bucket_info
+
 GENERIC_VARIANT = "generic"
 PLATFORM = platform.system().lower()
-BUCKET_URL = "https://therock-artifacts.s3.us-east-2.amazonaws.com"
 
 
 class FetchArtifactException(Exception):
@@ -57,7 +63,10 @@ def log(*args, **kwargs):
 
 def retrieve_s3_artifacts(run_id, amdgpu_family):
     """Checks that the AWS S3 bucket exists and returns artifact names."""
-    index_page_url = f"{BUCKET_URL}/{run_id}-{PLATFORM}/index-{amdgpu_family}.html"
+    EXTERNAL_REPO, BUCKET = retrieve_bucket_info()
+    BUCKET_URL = f"https://{BUCKET}.s3.amazonaws.com/{EXTERNAL_REPO}{run_id}-{PLATFORM}"
+    index_page_url = f"{BUCKET_URL}/index-{amdgpu_family}.html"
+    log(f"Retrieving artifacts from {index_page_url}")
     request = urllib.request.Request(index_page_url)
     try:
         with urllib.request.urlopen(request) as response:
@@ -89,6 +98,8 @@ def collect_artifacts_urls(
     existing_artifacts: set[str],
 ) -> list[str]:
     """Collects S3 artifact URLs to execute later in parallel."""
+    EXTERNAL_REPO, BUCKET = retrieve_bucket_info()
+    BUCKET_URL = f"https://{BUCKET}.s3.us-east-2.amazonaws.com/{EXTERNAL_REPO}{run_id}-{PLATFORM}"
     artifacts_to_retrieve = []
     for artifact in artifacts:
         file_name = f"{artifact}_{variant}.tar.xz"
@@ -98,7 +109,7 @@ def collect_artifacts_urls(
             artifacts_to_retrieve.append(
                 (
                     f"{build_dir}/{file_name}",
-                    f"{BUCKET_URL}/{run_id}-{PLATFORM}/{file_name}",
+                    f"{BUCKET_URL}/{file_name}",
                 )
             )
 

build_tools/create_log_index.py → build_tools/github_actions/create_log_index.py

@@ -7,6 +7,8 @@
 import platform
 import argparse
 
+THEROCK_DIR = Path(__file__).resolve().parent.parent.parent
+
 
 def log(*args):
     print(*args)
@@ -25,8 +27,7 @@ def index_log_files(build_dir: Path, amdgpu_family: str):
     log_dir = build_dir / "logs"
     index_file = log_dir / "index.html"
 
-    # TODO: Fork indexer.py locally to avoid relying on an external GitHub source at runtime.
-    indexer_path = build_dir / "indexer.py"
+    indexer_path = THEROCK_DIR / "third-party" / "indexer" / "indexer.py"
 
     if log_dir.is_dir():
         log(f"[INFO] Found '{log_dir}' directory. Indexing '*.log' files...")

build_tools/github_actions/upload_build_artifacts.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""
+upload_build_artifacts.py
+
+Uploads build artifacts to AWS S3 bucket
+"""
+
+import argparse
+import logging
+import os
+from pathlib import Path
+import platform
+import shlex
+import subprocess
+import sys
+
+logging.basicConfig(level=logging.INFO)
+
+THEROCK_DIR = Path(__file__).resolve().parent.parent.parent
+PLATFORM = platform.system().lower()
+
+# Importing indexer.py
+sys.path.append(str(THEROCK_DIR / "third-party" / "indexer"))
+from indexer import process_dir
+
+
+def exec(cmd: list[str], cwd: Path):
+    logging.info(f"++ Exec [{cwd}]$ {shlex.join(cmd)}")
+    subprocess.run(cmd, check=True)
+
+
+def retrieve_bucket_info() -> tuple[str, str]:
+    github_repository = os.getenv("GITHUB_REPOSITORY", "ROCm/TheRock")
+    owner, repo_name = github_repository.split("/")
+    external_repo = (
+        "" if repo_name == "TheRock" and owner == "ROCm" else f"{owner}-{repo_name}/"
+    )
+    bucket = (
+        "therock-artifacts"
+        if repo_name == "TheRock" and owner == "ROCm"
+        else "therock-artifacts-external"
+    )
+    return (external_repo, bucket)
+
+
+def create_index_file(args: argparse.Namespace):
+    logging.info("Creating index file")
+    build_dir = args.build_dir / "artifacts"
+
+    indexer_args = argparse.Namespace()
+    indexer_args.filter = "*.tar.xz*"
+    indexer_args.output_file = "index.html"
+    indexer_args.verbose = False
+    indexer_args.recursive = False
+    process_dir(build_dir, indexer_args)
+
+
+def upload_artifacts(args: argparse.Namespace, bucket_uri: str):
+    logging.info("Uploading artifacts to S3")
+    build_dir = args.build_dir
+    amdgpu_family = args.amdgpu_family
+
+    # Uploading artifacts to S3 bucket
+    cmd = [
+        "aws",
+        "s3",
+        "cp",
+        str(build_dir / "artifacts"),
+        bucket_uri,
+        "--recursive",
+        "--no-follow-symlinks",
+        "--exclude",
+        "*",
+        "--include",
+        "*.tar.xz*",
+    ]
+    exec(cmd, cwd=Path.cwd())
+
+    # Uploading index.html to S3 bucket
+    cmd = [
+        "aws",
+        "s3",
+        "cp",
+        str(build_dir / "artifacts" / "index.html"),
+        f"{bucket_uri}/index-{amdgpu_family}.html",
+    ]
+    exec(cmd, cwd=Path.cwd())
+
+
+def run(args: argparse.Namespace):
+    external_repo_path, bucket = retrieve_bucket_info()
+    run_id = args.run_id
+    bucket_uri = f"s3://{bucket}/{external_repo_path}{run_id}-{PLATFORM}"
+
+    create_index_file(args)
+    upload_artifacts(args, bucket_uri)
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(prog="artifact_upload")
+    parser.add_argument(
+        "--run-id", type=str, required=True, help="GitHub run ID of this workflow run"
+    )
+
+    parser.add_argument(
+        "--amdgpu-family", type=str, required=True, help="AMD GPU family to upload"
+    )
+
+    parser.add_argument(
+        "--build-dir",
+        type=Path,
+        required=True,
+        help="Path to the build directory of TheRock",
+    )
+
+    args = parser.parse_args(argv)
+    run(args)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])