diff --git a/.github/workflows/build_linux_packages.yml b/.github/workflows/build_linux_packages.yml index 276e5427e6f..2b7f07b72c8 100644 --- a/.github/workflows/build_linux_packages.yml +++ b/.github/workflows/build_linux_packages.yml @@ -43,11 +43,17 @@ jobs: strategy: fail-fast: true env: + BASE_BUILD_DIR: build + LOG_DIR: build/logs CACHE_DIR: ${{ github.workspace }}/.container-cache CCACHE_DIR: "${{ github.workspace }}/.container-cache/ccache" CCACHE_MAXSIZE: "700M" AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} TEATIME_FORCE_INTERACTIVE: 0 + TEATIME_S3_UPLOAD: "1" + TEATIME_S3_BUCKET: "therock-artifacts" + TEATIME_S3_SUBDIR: "${{ github.run_id }}-linux/logs/${{ inputs.amdgpu_families }}" + TEATIME_FAIL_ON_UPLOAD_ERROR: "1" BUCKET: ${{ github.event.repository.name == 'TheRock' && 'therock-artifacts' || 'therock-artifacts-external' }} steps: - name: "Checking out repository" @@ -91,9 +97,18 @@ jobs: run: | ./build_tools/fetch_sources.py --jobs 12 - - name: Install python deps + - name: Install python and dev deps run: | pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Configure AWS Credentials + if: always() + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 + with: + aws-region: us-east-2 + role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts + role-duration-seconds: 14400 # 4 hours - name: Build Projects id: build @@ -135,24 +150,10 @@ jobs: echo "-------------" ccache -s - - name: Configure AWS Credentials - if: always() - uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 - with: - aws-region: us-east-2 - role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts - - # TODO: Move to script - - name: Create Index Files + - name: Create Artifact Index if: always() run: | - curl --silent --fail --show-error --location \ - https://raw.githubusercontent.com/joshbrunty/Indexer/6d8cbfd15d3853b482e6a49f2d875ded9188b721/indexer.py \ - --output build/indexer.py - python build/indexer.py -f '*.tar.xz*' build/artifacts/ - python3 build_tools/create_log_index.py \ - --build-dir=build \ - --amdgpu-family=${{ env.AMDGPU_FAMILIES }} + python third-party/indexer/indexer.py -f '*.tar.xz*' ${{ env.BASE_BUILD_DIR }}/artifacts/ # TODO: Move to script - name: Upload Artifacts @@ -163,12 +164,10 @@ jobs: --include "*.tar.xz*" aws s3 cp build/artifacts/index.html s3://${{env.BUCKET}}/${{github.run_id}}-linux/index-${{env.AMDGPU_FAMILIES}}.html - - name: Upload Logs + - name: Index and Upload Logs if: always() run: | - python3 build_tools/upload_logs_to_s3.py \ - --build-dir=build \ - --s3-base-path="s3://${{env.BUCKET}}/${{github.run_id}}-linux/logs/${{env.AMDGPU_FAMILIES}}" + python3 build_tools/teatime.py --label "index_and_upload" --log-timestamps build/logs/temp.log - name: Add Links to Job Summary if: always() diff --git a/.github/workflows/build_windows_packages.yml b/.github/workflows/build_windows_packages.yml index b3accb0a1f2..ff3113b3fe5 100644 --- a/.github/workflows/build_windows_packages.yml +++ b/.github/workflows/build_windows_packages.yml @@ -39,17 +39,23 @@ jobs: strategy: fail-fast: true env: - BASE_BUILD_DIR_POWERSHELL: B:\tmpbuild + # Base build dir for Windows + BASE_BUILD_DIR: B:\tmpbuild + LOG_DIR: B:\tmpbuild\logs CACHE_DIR: "${{github.workspace}}/.cache" CCACHE_DIR: "${{github.workspace}}/.cache/ccache" CCACHE_MAXSIZE: "4000M" TEATIME_FORCE_INTERACTIVE: 0 AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }} + TEATIME_S3_UPLOAD: "1" + TEATIME_S3_BUCKET: "therock-artifacts" + TEATIME_S3_SUBDIR: "${{ github.run_id }}-windows/logs/${{ inputs.amdgpu_families }}" + TEATIME_FAIL_ON_UPLOAD_ERROR: "1" steps: - name: "Create build dir" shell: powershell run: | - $buildDir = "$env:BASE_BUILD_DIR_POWERSHELL\" + $buildDir = "$env:BASE_BUILD_DIR\" echo "BUILD_DIR_POWERSHELL=$buildDir" >> $env:GITHUB_ENV mkdir "$buildDir" Write-Host "Generated Build Directory: $buildDir" @@ -67,9 +73,10 @@ jobs: with: python-version: "3.12" - - name: Install python deps + - name: Install python and dev deps run: | pip install -r requirements.txt + pip install -r requirements-dev.txt - name: Install requirements run: | @@ -133,6 +140,14 @@ jobs: path: amdgpu-windows-interop lfs: true + - name: Configure AWS Credentials + if: always() + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 + with: + aws-region: us-east-2 + role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts + role-duration-seconds: 14400 # 4 hours + - name: Configure Projects run: | # clear cache before build and after download @@ -189,44 +204,26 @@ jobs: $fsout | % {$_.Used/=1GB;$_.Free/=1GB;$_} | Write-Host get-disk | Select-object @{Name="Size(GB)";Expression={$_.Size/1GB}} | Write-Host - - name: Configure AWS Credentials - if: always() - uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0 - with: - aws-region: us-east-2 - role-to-assume: arn:aws:iam::692859939525:role/therock-artifacts - - # TODO: Move to script - - name: Create Index Files + - name: Create Artifact Index if: always() run: | - curl --silent --fail --show-error --location \ - https://raw.githubusercontent.com/joshbrunty/Indexer/6d8cbfd15d3853b482e6a49f2d875ded9188b721/indexer.py \ - --output ${{ env.BUILD_DIR_BASH }}/indexer.py - python ${{ env.BUILD_DIR_BASH }}/indexer.py -f '*.tar.xz*' ${{ env.BUILD_DIR_BASH }}/artifacts/ - python build_tools/create_log_index.py \ - --build-dir=${{ env.BUILD_DIR_BASH }} \ - --amdgpu-family=${{ env.AMDGPU_FAMILIES }} + python third-party/indexer/indexer.py -f '*.tar.xz*' ${{ env.BUILD_DIR_BASH }}/artifacts/ # TODO: Move to script - name: Upload Artifacts shell: powershell run: | $Env:PATH += ";C:\Program Files\Amazon\AWSCLIV2" - aws s3 cp ${{ env.BASE_BUILD_DIR_POWERSHELL }}\artifacts s3://therock-artifacts/${{github.run_id}}-windows/ ` + aws s3 cp ${{ env.BASE_BUILD_DIR }}\artifacts s3://therock-artifacts/${{github.run_id}}-windows/ ` --recursive --no-follow-symlinks ` --exclude "*" ` --include "*.tar.xz*" - aws s3 cp ${{ env.BASE_BUILD_DIR_POWERSHELL }}\artifacts\index.html s3://therock-artifacts/${{github.run_id}}-windows/index-${{env.AMDGPU_FAMILIES}}.html + aws s3 cp ${{ env.BASE_BUILD_DIR }}\artifacts\index.html s3://therock-artifacts/${{github.run_id}}-windows/index-${{env.AMDGPU_FAMILIES}}.html - - name: Upload Logs + - name: Index and Upload Logs if: always() - shell: powershell run: | - $Env:PATH += ";C:\Program Files\Amazon\AWSCLIV2" - python3 build_tools/upload_logs_to_s3.py ` - --build-dir=${{ env.BASE_BUILD_DIR_POWERSHELL }} ` - --s3-base-path="s3://therock-artifacts/${{github.run_id}}-windows/logs/${{env.AMDGPU_FAMILIES}}" + python3 build_tools/teatime.py --label "index_and_upload" --log-timestamps build/logs/temp.log - name: Add Links to Job Summary if: always() @@ -234,7 +231,7 @@ jobs: LOG_URL="https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-windows/logs/${{env.AMDGPU_FAMILIES}}/index.html" echo "[Build Logs](${LOG_URL})" >> $GITHUB_STEP_SUMMARY - ARTIFACT_INDEX="${{ env.BASE_BUILD_DIR_POWERSHELL }}/artifacts/index.html" + ARTIFACT_INDEX="${{ env.BASE_BUILD_DIR }}/artifacts/index.html" if [ -f "${ARTIFACT_INDEX}" ]; then ARTIFACT_URL="https://therock-artifacts.s3.us-east-2.amazonaws.com/${{github.run_id}}-windows/index-${{env.AMDGPU_FAMILIES}}.html" diff --git a/build_tools/create_log_index.py b/build_tools/create_log_index.py index b7d0d374a71..ffdbc155eb3 100644 --- a/build_tools/create_log_index.py +++ b/build_tools/create_log_index.py @@ -21,22 +21,41 @@ def normalize_path(p: Path) -> str: return str(p).replace("\\", "/") if is_windows() else str(p) -def index_log_files(build_dir: Path, amdgpu_family: str): - log_dir = build_dir / "logs" +def get_indexer_path() -> Path: + """ + Resolve to the local third-party/indexer/indexer.py copy. + """ + indexer_path = ( + Path(__file__).resolve().parent.parent / "third-party/indexer/indexer.py" + ) + if indexer_path.is_file(): + log(f"[INFO] Using bundled indexer.py: {indexer_path}") + return indexer_path + else: + log(f"[ERROR] Bundled indexer.py not found at: {indexer_path}") + sys.exit(2) + + +def index_log_files(log_dir: Path, amdgpu_family: str): index_file = log_dir / "index.html" - # TODO: Fork indexer.py locally to avoid relying on an external GitHub source at runtime. - indexer_path = build_dir / "indexer.py" + if not log_dir.is_dir(): + log(f"[WARN] Log directory '{log_dir}' not found. Skipping indexing.") + return + + indexer_path = get_indexer_path() + log( + f"[INFO] Found '{log_dir}' directory. Indexing '*.log' files using indexer: {indexer_path}" + ) - if log_dir.is_dir(): - log(f"[INFO] Found '{log_dir}' directory. Indexing '*.log' files...") + try: subprocess.run( - ["python", str(indexer_path), "-f", "*.log", normalize_path(log_dir)], + [sys.executable, str(indexer_path), "-f", "*.log", normalize_path(log_dir)], check=True, ) - else: - log(f"[WARN] Log directory '{log_dir}' not found. Skipping indexing.") - return + except subprocess.CalledProcessError as e: + log(f"[ERROR] Failed to run indexer.py: {e}") + sys.exit(2) if index_file.exists(): log( @@ -46,19 +65,23 @@ def index_log_files(build_dir: Path, amdgpu_family: str): updated = content.replace( 'a href=".."', f'a href="../../index-{amdgpu_family}.html"' ) - index_file.write_text(updated) + + # Ensure full write and flush to disk + with open(index_file, "w", encoding="utf-8") as f: + f.write(updated) + f.flush() + os.fsync(f.fileno()) + log("[INFO] Log index links updated.") - else: - log(f"[WARN] '{index_file}' not found. Skipping link rewrite.") if __name__ == "__main__": parser = argparse.ArgumentParser(description="Create HTML index for log files.") parser.add_argument( - "--build-dir", + "--log-dir", type=Path, - default=Path(os.getenv("BUILD_DIR", "build")), - help="Build directory containing logs (default: 'build' or $BUILD_DIR)", + default=Path(os.getenv("LOG_DIR", "build/logs")), + help="Directory containing log files (default: 'build/logs' or $LOG_DIR)", ) parser.add_argument( "--amdgpu-family", @@ -72,4 +95,4 @@ def index_log_files(build_dir: Path, amdgpu_family: str): log("[ERROR] --amdgpu-family not provided and AMDGPU_FAMILIES env var not set") sys.exit(1) - index_log_files(args.build_dir, args.amdgpu_family) + index_log_files(args.log_dir, args.amdgpu_family) diff --git a/build_tools/teatime.py b/build_tools/teatime.py index 9ce34ed808b..f4584015c8e 100755 --- a/build_tools/teatime.py +++ b/build_tools/teatime.py @@ -25,6 +25,18 @@ * --log-timestamps: Log lines will be written with a starting column of the time in seconds since start, and a header/trailer will be added with more timing information. +* --skip-index: Suppresses automatic indexing and uploading of logs to S3. + +Environment Variables: +* TEATIME_FORCE_INTERACTIVE: If set to 1, forces console output regardless of + --interactive flag. +* TEATIME_S3_UPLOAD: If "1", enables automatic S3 upload and log indexing. +* TEATIME_S3_BUCKET: S3 bucket name for log uploads. +* TEATIME_S3_SUBDIR: Subdirectory in the bucket for organizing uploads. +* TEATIME_FAIL_ON_UPLOAD_ERROR: If "1", treat S3 upload or indexing errors as fatal. +* BASE_BUILD_DIR: Root of the build directory. Used to find logs and pass to + indexing/upload scripts. +* AMDGPU_FAMILIES: Required by `create_log_index.py` to organize logs. CI systems can set `TEATIME_LABEL_GH_GROUP=1` in the environment, which will cause labeled console output to be printed using GitHub Actions group markers @@ -46,6 +58,7 @@ class OutputSink: def __init__(self, args: argparse.Namespace): self.start_time = time.time() self.interactive: bool = args.interactive + self.skip_index = args.skip_index if self.interactive: self.out = sys.stdout.buffer else: @@ -81,6 +94,33 @@ def __init__(self, args: argparse.Namespace): self.log_file = open(self.log_path, "wb") self.log_timestamps: bool = args.log_timestamps + # S3 upload configuration + try: + self.upload_to_s3 = bool(int(os.getenv("TEATIME_S3_UPLOAD", "0"))) + except ValueError: + print( + "warning: TEATIME_S3_UPLOAD env var must be an integer " + "(skipping S3 log upload)", + file=sys.stderr, + ) + self.upload_to_s3 = False + + self.s3_bucket = os.getenv("TEATIME_S3_BUCKET") + if not self.s3_bucket and self.upload_to_s3: + print( + "warning: TEATIME_S3_BUCKET is not set (S3 upload will likely fail)", + file=sys.stderr, + ) + + self.s3_subdir = os.getenv("TEATIME_S3_SUBDIR") + if not self.s3_subdir and self.upload_to_s3: + print( + "warning: TEATIME_S3_SUBDIR is not set (S3 upload will likely fail)", + file=sys.stderr, + ) + + self.log_dir = os.getenv("LOG_DIR", "build/logs") + def start(self): if self.gh_group_label is not None: self.out.write(b"::group::" + self.gh_group_label + b"\n") @@ -95,6 +135,7 @@ def finish(self): f"END\t{end_time}\t{end_time - self.start_time}\n".encode() ) self.log_file.close() + if self.gh_group_label is not None: self.out.write(b"::endgroup::\n") elif self.interactive_prefix is not None and self.label is not None: @@ -103,6 +144,79 @@ def finish(self): b"[" + self.label + b" completed in " + run_pretty.encode() + b"]\n" ) + # Call coordinate_index_and_logs if indexing is enabled. + if ( + not self.skip_index + and self.upload_to_s3 + and self.s3_bucket + and self.log_dir + ): + self.coordinate_index_and_logs() + + def coordinate_index_and_logs(self): + # Determine paths + log_dir = self.log_dir + amdgpu_family = os.getenv("AMDGPU_FAMILIES") + + # Step 1: Run create_log_index.py (if AMDGPU_FAMILIES is defined) + if amdgpu_family: + try: + index_script = ( + Path(__file__).resolve().parent.parent + / "build_tools" + / "create_log_index.py" + ) + print(f"[TEATIME] Indexing logs for AMDGPU_FAMILIES={amdgpu_family}") + subprocess.run( + [ + sys.executable, + str(index_script), + "--log-dir", + str(log_dir), + "--amdgpu-family", + amdgpu_family, + ], + check=True, + ) + except subprocess.CalledProcessError as e: + print(f"[WARN] create_log_index.py failed: {e}", file=sys.stderr) + except Exception as e: + print( + f"[WARN] Unexpected error during log indexing: {e}", file=sys.stderr + ) + else: + print("[WARN] AMDGPU_FAMILIES not set; skipping log indexing") + + # Step 2: S3 upload using --bucket and --subdir + try: + upload_script = ( + Path(__file__).resolve().parent.parent + / "build_tools" + / "upload_logs_to_s3.py" + ) + print(f"[TEATIME] Uploading logs to s3://{self.s3_bucket}/{self.s3_subdir}") + subprocess.run( + [ + sys.executable, + str(upload_script), + "--log-dir", + str(log_dir), + "--bucket", + self.s3_bucket, + "--subdir", + self.s3_subdir, + ], + check=True, + ) + except subprocess.CalledProcessError as e: + print(f"[WARN] Log upload failed: {e}", file=sys.stderr) + if os.getenv("TEATIME_FAIL_ON_UPLOAD_ERROR") == "1": + raise + except Exception as e: + print(f"[WARN] Unexpected error during log upload: {e}", file=sys.stderr) + if os.getenv("TEATIME_FAIL_ON_UPLOAD_ERROR") == "1": + raise + def writeline(self, line: bytes): if self.interactive_prefix is not None: self.out.write(self.interactive_prefix) @@ -178,6 +292,9 @@ def main(cl_args: list[str]): default=False, help="Log timestamps along with log lines to the log file", ) + p.add_argument( + "--skip-index", action="store_true", help="Skip indexing and uploading logs" + ) p.add_argument("file", type=Path, help="Also log output to this file") args = p.parse_args(cl_args) diff --git a/build_tools/upload_logs_to_s3.py b/build_tools/upload_logs_to_s3.py index f5456893119..a770e8241ab 100644 --- a/build_tools/upload_logs_to_s3.py +++ b/build_tools/upload_logs_to_s3.py @@ -2,16 +2,17 @@ """ upload_logs_to_s3.py -Uploads log files and index.html to an S3 bucket using the AWS CLI. +Uploads log files and index.html to an S3 bucket using boto3. """ import os import sys -import glob -import shutil import argparse -import subprocess from pathlib import Path +import time + +import boto3 +from botocore.exceptions import ClientError def log(*args, **kwargs): @@ -19,29 +20,30 @@ def log(*args, **kwargs): sys.stdout.flush() -def check_aws_cli_available(): - if not shutil.which("aws"): - log("[ERROR] AWS CLI not found in PATH.") - sys.exit(1) - +def upload_file_boto3(file_path: Path, bucket: str, key: str, content_type: str = None): + s3 = boto3.client("s3") + extra_args = {"ContentType": content_type} if content_type else {} -def run_aws_cp(source_path: Path, s3_destination: str, content_type: str = None): - if source_path.is_dir(): - cmd = ["aws", "s3", "cp", str(source_path), s3_destination, "--recursive"] + for attempt in range(3): + if not file_path.exists() or file_path.stat().st_size == 0: + log(f"[WARN] Attempt {attempt+1}: index.html is missing or empty") + time.sleep(1) + else: + break else: - cmd = ["aws", "s3", "cp", str(source_path), s3_destination] + log(f"[ERROR] index.html still missing or empty after retries: {file_path}") + return - if content_type: - cmd += ["--content-type", content_type] try: - log(f"[INFO] Running: {' '.join(cmd)}") - subprocess.run(cmd, check=True) - except subprocess.CalledProcessError as e: - log(f"[ERROR] Failed to upload {source_path} to {s3_destination}: {e}") + log(f"[INFO] Uploading {file_path} to s3://{bucket}/{key}") + s3.upload_file(str(file_path), bucket, key, ExtraArgs=extra_args) + except ClientError as e: + log(f"[ERROR] Failed to upload {file_path} to s3://{bucket}/{key}: {e}") + else: + log(f"[INFO] Successfully uploaded {file_path} to s3://{bucket}/{key}") -def upload_logs_to_s3(s3_base_path: str, build_dir: Path): - log_dir = build_dir / "logs" +def upload_logs_to_s3(bucket: str, subdir: str, log_dir: Path): if not log_dir.is_dir(): log(f"[INFO] Log directory {log_dir} not found. Skipping upload.") @@ -52,40 +54,29 @@ def upload_logs_to_s3(s3_base_path: str, build_dir: Path): if not log_files: log("[WARN] No .log files found. Skipping log upload.") else: - run_aws_cp(log_dir, s3_base_path, content_type="text/plain") + for file_path in log_files: + key = f"{subdir}/{file_path.name}" + upload_file_boto3(file_path, bucket, key, content_type="text/plain") # Upload index.html index_path = log_dir / "index.html" if index_path.is_file(): - index_s3_dest = f"{s3_base_path}/index.html" - run_aws_cp(index_path, index_s3_dest, content_type="text/html") - log(f"[INFO] Uploaded {index_path} to {index_s3_dest}") + key = f"{subdir}/index.html" + upload_file_boto3(index_path, bucket, key, content_type="text/html") else: log(f"[INFO] No index.html found at {log_dir}. Skipping index upload.") def main(): - check_aws_cli_available() - - repo_root = Path(__file__).resolve().parent.parent - default_build_dir = repo_root / "build" + default = Path(os.getenv("LOG_DIR", "build/logs")) parser = argparse.ArgumentParser(description="Upload logs to S3.") - parser.add_argument( - "--build-dir", - type=Path, - default=default_build_dir, - help="Path to the build directory (default: /build)", - ) - parser.add_argument( - "--s3-base-path", - type=str, - required=True, - help="Base S3 path to upload logs to, e.g. s3://bucket/run-id-platform/logs/family", - ) - args = parser.parse_args() + parser.add_argument("--bucket", required=True, help="S3 bucket name") + parser.add_argument("--subdir", required=True, help="Subdirectory in the bucket") + parser.add_argument("--log-dir", type=Path, default=default) - upload_logs_to_s3(args.s3_base_path, args.build_dir) + args = parser.parse_args() + upload_logs_to_s3(args.bucket, args.subdir, args.log_dir) if __name__ == "__main__": diff --git a/cmake/therock_subproject.cmake b/cmake/therock_subproject.cmake index 9d8769f1b9e..633aecce540 100644 --- a/cmake/therock_subproject.cmake +++ b/cmake/therock_subproject.cmake @@ -106,6 +106,7 @@ function(therock_subproject_log_command out_var) "${Python3_EXECUTABLE}" "${THEROCK_SOURCE_DIR}/build_tools/teatime.py" "--log-timestamps" + "--skip-index" ) if(ARG_LABEL) list(APPEND command "--label" "${ARG_LABEL}") diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000000..c740658dec1 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +boto3==1.38.23