diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml index 4454fac5114..1e41dd736ec 100644 --- a/.github/workflows/build_linux_packages.yml +++ b/.github/workflows/build_linux_packages.yml @@ -22,6 +22,8 @@ on: expect_failure: type: boolean +# See the details regarding permissions from the link: +# https://github.com/aws-actions/configure-aws-credentials?tab=readme-ov-file#oidc permissions: contents: read @@ -122,6 +124,7 @@ jobs: key: linux-build-packages-manylinux-v2-${{ github.sha }} - name: Configure AWS Credentials + if: always() uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 with: aws-region: us-east-2 @@ -129,13 +132,15 @@ jobs: # TODO: Move to script - name: Create Index Files + if: always() run: | curl --silent --fail --show-error --location \ https://raw.githubusercontent.com/joshbrunty/Indexer/6d8cbfd15d3853b482e6a49f2d875ded9188b721/indexer.py \ --output build/indexer.py python build/indexer.py -f '*.tar.xz*' build/artifacts/ - python build/indexer.py -f '*.log' build/logs/ - sed -i 's,a href="..",a href="../../index-${{env.AMDGPU_FAMILIES}}.html",g' build/logs/index.html + python3 build_tools/create_log_index.py \ + --build-dir=build \ + --amdgpu-family=${{ env.AMDGPU_FAMILIES }} # TODO: Move to script - name: Upload Artifacts @@ -146,17 +151,24 @@ jobs: --include "*.tar.xz*" aws s3 cp build/artifacts/index.html s3://therock-artifacts/${{github.run_id}}-linux/index-${{env.AMDGPU_FAMILIES}}.html - # TODO: Move to script - name: Upload Logs + if: always() run: | - aws s3 cp build/logs/ s3://therock-artifacts/${{github.run_id}}-linux/logs/${{env.AMDGPU_FAMILIES}}/ \ - --recursive \ - --content-type="text/plain" \ - --exclude "*" \ - --include "*.log" - aws s3 cp build/logs/index.html s3://therock-artifacts/${{github.run_id}}-linux/logs/${{env.AMDGPU_FAMILIES}}/ + python3 build_tools/upload_logs_to_s3.py \ + --build-dir=build \ + --s3-base-path="s3://therock-artifacts/${{github.run_id}}-linux/logs/${{env.AMDGPU_FAMILIES}}" - name: Add Links to Job Summary + if: always() run: | - echo '* [Artifacts](https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-linux/index-${{env.AMDGPU_FAMILIES}}.html)' >> $GITHUB_STEP_SUMMARY - echo '* [Build logs](https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-linux/logs/${{env.AMDGPU_FAMILIES}}/index.html)' >> $GITHUB_STEP_SUMMARY + LOG_URL="https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-linux/logs/${{env.AMDGPU_FAMILIES}}/index.html" + echo "[Build Logs](${LOG_URL})" >> $GITHUB_STEP_SUMMARY + + ARTIFACT_INDEX=build/artifacts/index.html + if [ -f "${ARTIFACT_INDEX}" ]; then + ARTIFACT_URL="https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-linux/index-${{env.AMDGPU_FAMILIES}}.html" + + echo "[Artifacts](${ARTIFACT_URL})" >> $GITHUB_STEP_SUMMARY + else + echo "[INFO] No artifacts index found. Skipping artifact link." + fi diff --git a/.github/workflows/build_windows_packages.yml b/.github/workflows/build_windows_packages.yml index 0a86061aa6d..1c01de73c12 100644 --- a/.github/workflows/build_windows_packages.yml +++ b/.github/workflows/build_windows_packages.yml @@ -197,6 +197,7 @@ jobs: get-disk | Select-object @{Name="Size(GB)";Expression={$_.Size/1GB}} | Write-Host - name: Configure AWS Credentials + if: always() uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 with: aws-region: us-east-2 @@ -204,13 +205,15 @@ jobs: # TODO: Move to script - name: Create Index Files + if: always() run: | curl --silent --fail --show-error --location \ https://raw.githubusercontent.com/joshbrunty/Indexer/6d8cbfd15d3853b482e6a49f2d875ded9188b721/indexer.py \ --output ${{ env.BUILD_DIR_BASH }}/indexer.py python ${{ env.BUILD_DIR_BASH }}/indexer.py -f '*.tar.xz*' ${{ env.BUILD_DIR_BASH }}/artifacts/ - python ${{ env.BUILD_DIR_BASH }}/indexer.py -f '*.log' ${{ env.BUILD_DIR_BASH }}/logs/ - sed -i 's,a href="..",a href="../../index-${{env.AMDGPU_FAMILIES}}.html",g' ${{ env.BUILD_DIR_BASH }}/logs/index.html + python build_tools/create_log_index.py \ + --build-dir=${{ env.BUILD_DIR_BASH }} \ + --amdgpu-family=${{ env.AMDGPU_FAMILIES }} # TODO: Move to script - name: Upload Artifacts @@ -223,19 +226,26 @@ jobs: --include "*.tar.xz*" aws s3 cp ${{ env.BASE_BUILD_DIR_POWERSHELL }}\artifacts\index.html s3://therock-artifacts/${{github.run_id}}-windows/index-${{env.AMDGPU_FAMILIES}}.html - # TODO: Move to script - name: Upload Logs + if: always() shell: powershell run: | $Env:PATH += ";C:\Program Files\Amazon\AWSCLIV2" - aws s3 cp ${{ env.BASE_BUILD_DIR_POWERSHELL }}\logs s3://therock-artifacts/${{github.run_id}}-windows/logs/${{env.AMDGPU_FAMILIES}}/ ` - --recursive ` - --content-type="text/plain" ` - --exclude "*" ` - --include "*.log" - aws s3 cp ${{ env.BASE_BUILD_DIR_POWERSHELL }}\logs\index.html s3://therock-artifacts/${{github.run_id}}-windows/logs/${{env.AMDGPU_FAMILIES}}/ + python3 build_tools/upload_logs_to_s3.py ` + --build-dir=${{ env.BASE_BUILD_DIR_POWERSHELL }} ` + --s3-base-path="s3://therock-artifacts/${{github.run_id}}-windows/logs/${{env.AMDGPU_FAMILIES}}" - name: Add Links to Job Summary + if: always() run: | - echo '* [Artifacts](https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-windows/index-${{env.AMDGPU_FAMILIES}}.html)' >> $GITHUB_STEP_SUMMARY - echo '* [Build logs](https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-windows/logs/${{env.AMDGPU_FAMILIES}}/index.html)' >> $GITHUB_STEP_SUMMARY + LOG_URL="https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-windows/logs/${{env.AMDGPU_FAMILIES}}/index.html" + echo "[Build Logs](${LOG_URL})" >> $GITHUB_STEP_SUMMARY + + ARTIFACT_INDEX="${{ env.BASE_BUILD_DIR_POWERSHELL }}/artifacts/index.html" + if [ -f "${ARTIFACT_INDEX}" ]; then + ARTIFACT_URL="https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-windows/index-${{env.AMDGPU_FAMILIES}}.html" + + echo "[Artifacts](${ARTIFACT_URL})" >> $GITHUB_STEP_SUMMARY + else + echo "[INFO] No artifacts index found. Skipping artifact link." + fi diff --git a/build_tools/create_log_index.py b/build_tools/create_log_index.py new file mode 100644 index 00000000000..b7d0d374a71 --- /dev/null +++ b/build_tools/create_log_index.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +import os +import subprocess +from pathlib import Path +import sys +import platform +import argparse + + +def log(*args): + print(*args) + sys.stdout.flush() + + +def is_windows(): + return platform.system().lower() == "windows" + + +def normalize_path(p: Path) -> str: + return str(p).replace("\\", "/") if is_windows() else str(p) + + +def index_log_files(build_dir: Path, amdgpu_family: str): + log_dir = build_dir / "logs" + index_file = log_dir / "index.html" + + # TODO: Fork indexer.py locally to avoid relying on an external GitHub source at runtime. + indexer_path = build_dir / "indexer.py" + + if log_dir.is_dir(): + log(f"[INFO] Found '{log_dir}' directory. Indexing '*.log' files...") + subprocess.run( + ["python", str(indexer_path), "-f", "*.log", normalize_path(log_dir)], + check=True, + ) + else: + log(f"[WARN] Log directory '{log_dir}' not found. Skipping indexing.") + return + + if index_file.exists(): + log( + f"[INFO] Rewriting links in '{index_file}' with AMDGPU_FAMILIES={amdgpu_family}..." + ) + content = index_file.read_text() + updated = content.replace( + 'a href=".."', f'a href="../../index-{amdgpu_family}.html"' + ) + index_file.write_text(updated) + log("[INFO] Log index links updated.") + else: + log(f"[WARN] '{index_file}' not found. Skipping link rewrite.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Create HTML index for log files.") + parser.add_argument( + "--build-dir", + type=Path, + default=Path(os.getenv("BUILD_DIR", "build")), + help="Build directory containing logs (default: 'build' or $BUILD_DIR)", + ) + parser.add_argument( + "--amdgpu-family", + type=str, + default=os.getenv("AMDGPU_FAMILIES"), + help="AMDGPU family name (default: $AMDGPU_FAMILIES)", + ) + args = parser.parse_args() + + if not args.amdgpu_family: + log("[ERROR] --amdgpu-family not provided and AMDGPU_FAMILIES env var not set") + sys.exit(1) + + index_log_files(args.build_dir, args.amdgpu_family) diff --git a/build_tools/upload_logs_to_s3.py b/build_tools/upload_logs_to_s3.py new file mode 100644 index 00000000000..f5456893119 --- /dev/null +++ b/build_tools/upload_logs_to_s3.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +""" +upload_logs_to_s3.py + +Uploads log files and index.html to an S3 bucket using the AWS CLI. +""" + +import os +import sys +import glob +import shutil +import argparse +import subprocess +from pathlib import Path + + +def log(*args, **kwargs): + print(*args, **kwargs) + sys.stdout.flush() + + +def check_aws_cli_available(): + if not shutil.which("aws"): + log("[ERROR] AWS CLI not found in PATH.") + sys.exit(1) + + +def run_aws_cp(source_path: Path, s3_destination: str, content_type: str = None): + if source_path.is_dir(): + cmd = ["aws", "s3", "cp", str(source_path), s3_destination, "--recursive"] + else: + cmd = ["aws", "s3", "cp", str(source_path), s3_destination] + + if content_type: + cmd += ["--content-type", content_type] + try: + log(f"[INFO] Running: {' '.join(cmd)}") + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as e: + log(f"[ERROR] Failed to upload {source_path} to {s3_destination}: {e}") + + +def upload_logs_to_s3(s3_base_path: str, build_dir: Path): + log_dir = build_dir / "logs" + + if not log_dir.is_dir(): + log(f"[INFO] Log directory {log_dir} not found. Skipping upload.") + return + + # Upload .log files + log_files = list(log_dir.glob("*.log")) + if not log_files: + log("[WARN] No .log files found. Skipping log upload.") + else: + run_aws_cp(log_dir, s3_base_path, content_type="text/plain") + + # Upload index.html + index_path = log_dir / "index.html" + if index_path.is_file(): + index_s3_dest = f"{s3_base_path}/index.html" + run_aws_cp(index_path, index_s3_dest, content_type="text/html") + log(f"[INFO] Uploaded {index_path} to {index_s3_dest}") + else: + log(f"[INFO] No index.html found at {log_dir}. Skipping index upload.") + + +def main(): + check_aws_cli_available() + + repo_root = Path(__file__).resolve().parent.parent + default_build_dir = repo_root / "build" + + parser = argparse.ArgumentParser(description="Upload logs to S3.") + parser.add_argument( + "--build-dir", + type=Path, + default=default_build_dir, + help="Path to the build directory (default: /build)", + ) + parser.add_argument( + "--s3-base-path", + type=str, + required=True, + help="Base S3 path to upload logs to, e.g. s3://bucket/run-id-platform/logs/family", + ) + args = parser.parse_args() + + upload_logs_to_s3(args.s3_base_path, args.build_dir) + + +if __name__ == "__main__": + main()