diff --git a/.github/workflows/multi_arch_build_native_linux_packages.yml b/.github/workflows/multi_arch_build_native_linux_packages.yml new file mode 100644 index 00000000000..b7f1af6c4c6 --- /dev/null +++ b/.github/workflows/multi_arch_build_native_linux_packages.yml @@ -0,0 +1,187 @@ +name: Build Multi-Arch Native Linux Packages + +on: + workflow_call: + inputs: + dist_amdgpu_families: + description: "Semicolon-separated list of all GPU families (e.g., 'gfx94X-dcgpu;gfx120X-all')" + required: true + type: string + artifact_group: + type: string + description: "Artifact group (e.g. multi-arch-release)" + required: true + artifact_run_id: + description: "Workflow run id to download the artifacts from" + required: true + type: string + rocm_version: + description: "ROCm version to append to the package (8.0.0, 8.0.1rc1, ...)" + required: true + type: string + native_package_type: + description: "Package type (deb or rpm)" + required: true + type: string + package_suffix: + description: "The suffix to be added to package name (asan, tsan, static or rpath)" + required: false + type: string + default: "" + release_type: + description: "The type of release to build ('dev', 'nightly', or 'release'). Empty string for CI builds." + required: false + type: string + default: "" + repository: + description: "Repository to checkout. Otherwise, defaults to `github.repository`" + type: string + required: false + ref: + description: "Branch, tag or SHA to checkout. Defaults to the reference or SHA that triggered the workflow" + type: string + required: false + +permissions: + id-token: write + contents: read + +run-name: Build ${{ inputs.native_package_type }} packages (${{ inputs.rocm_version }}${{ inputs.release_type && format(', {0}', inputs.release_type) || '' }}) + +jobs: + build_native_packages: + name: Build ${{ inputs.native_package_type }} packages + runs-on: ${{ github.repository_owner == 'ROCm' && 'azure-linux-scale-rocm' || 'ubuntu-24.04' }} + env: + ARTIFACT_RUN_ID: ${{ inputs.artifact_run_id }} + DIST_AMDGPU_FAMILIES: ${{ inputs.dist_amdgpu_families }} + PACKAGE_SUFFIX: ${{ inputs.package_suffix }} + OUTPUT_DIR: ${{ github.workspace }}/output + ARTIFACTS_DIR: ${{ github.workspace }}/output/artifacts + PACKAGE_DIST_DIR: ${{ github.workspace }}/output/packages + RELEASE_TYPE: ${{ inputs.release_type }} + steps: + - name: "Checking out repository" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref || '' }} + + - name: Set up Python + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: '3.12' + + - name: Install Python requirements + run: | + pip install pyelftools boto3 jinja2 + + - name: Install System requirements + run: | + # Install the needed tools for creating rpm / deb packages + # Also install tools for creating repo files + sudo apt update + sudo apt install -y llvm + sudo apt install -y rpm debhelper-compat build-essential + sudo apt install -y dpkg-dev createrepo-c + + - name: Determine S3 bucket and prefix + id: s3_config + run: | + python ./build_tools/packaging/linux/get_s3_config.py \ + --release-type "${{ env.RELEASE_TYPE }}" \ + --repository "${{ github.repository }}" \ + --is-fork "${{ github.event.pull_request.head.repo.fork || 'false' }}" \ + --pkg-type "${{ inputs.native_package_type }}" \ + --artifact-id "${{ env.ARTIFACT_RUN_ID }}" \ + --rocm-version "${{ inputs.rocm_version }}" \ + --output-format github >> $GITHUB_OUTPUT + + - name: Determine IAM role + id: iam_role + run: | + # ================================================================ + # IAM Role Selection Logic + # ================================================================ + # Determines which AWS IAM role to assume based on job_type from s3_config step. + # + # Role Mapping: + # ├─ IF job_type == "ci" + # │ └─ Use: therock-ci role + # │ (For all CI buckets: therock-ci-artifacts, therock-ci-artifacts-external, therock-artifacts-internal) + # │ + # └─ ELSE (job_type == dev/nightly/prerelease/release) + # └─ Use: therock-${job_type} role + # (For package buckets: therock-dev-packages, therock-nightly-packages, etc.) + # + # ================================================================ + + JOB_TYPE="${{ steps.s3_config.outputs.job_type }}" + + if [[ "${JOB_TYPE}" == "ci" ]]; then + # CI builds use the shared CI role (for all artifact buckets) + IAM_ROLE="arn:aws:iam::692859939525:role/therock-ci" + echo "✓ Using CI role: ${IAM_ROLE}" + else + # Release builds use release-type-specific roles (for package buckets) + IAM_ROLE="arn:aws:iam::692859939525:role/therock-${JOB_TYPE}" + echo "✓ Using release-type role: ${IAM_ROLE}" + fi + + echo "iam_role=${IAM_ROLE}" >> $GITHUB_OUTPUT + + - name: Fetch Artifacts for all GPU families + run: | + echo "Fetching artifacts for build ${{ env.ARTIFACT_RUN_ID }}" + + # Convert semicolon-separated to comma-separated + FAMILIES_CSV="${DIST_AMDGPU_FAMILIES//;/,}" + echo "Fetching artifacts for GPU families: ${FAMILIES_CSV}" + + # Create artifacts directory + mkdir -p "${{ env.ARTIFACTS_DIR }}" + + python ./build_tools/fetch_artifacts.py \ + --run-id="${{ env.ARTIFACT_RUN_ID }}" \ + --run-github-repo="${{ github.repository }}" \ + --artifact-group="${{ inputs.artifact_group }}" \ + --platform="linux" \ + --amdgpu-targets="${FAMILIES_CSV}" \ + --output-dir="${{ env.ARTIFACTS_DIR }}" + + - name: Build Packages + id: build-packages + run: | + echo "Building ${{ inputs.native_package_type }} packages for all GPU families" + echo "Families: ${{ env.DIST_AMDGPU_FAMILIES }}" + + # Pass the target families as-is (semicolon-separated) + # build_package.py's normalize_target_list() handles all separators + python ./build_tools/packaging/linux/build_package.py \ + --dest-dir ${{ env.PACKAGE_DIST_DIR }} \ + --rocm-version ${{ inputs.rocm_version }} \ + --target "${{ env.DIST_AMDGPU_FAMILIES }}" \ + --artifacts-dir ${{ env.ARTIFACTS_DIR }} \ + --pkg-type ${{ inputs.native_package_type }} \ + --version-suffix ${{ env.ARTIFACT_RUN_ID }} \ + --enable_kpack=false + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 + with: + aws-region: us-east-2 + role-to-assume: ${{ steps.iam_role.outputs.iam_role }} + + - name: Upload Package repo to S3 + id: upload-packages + run: | + echo "Uploading to S3 bucket: ${{ steps.s3_config.outputs.s3_bucket }}" + echo "Using prefix: ${{ steps.s3_config.outputs.s3_prefix }}" + echo "Job type: ${{ steps.s3_config.outputs.job_type }}" + + python ./build_tools/packaging/linux/upload_package_repo.py \ + --pkg-type ${{ inputs.native_package_type }} \ + --s3-bucket ${{ steps.s3_config.outputs.s3_bucket }} \ + --artifact-id ${{ env.ARTIFACT_RUN_ID }} \ + --job ${{ steps.s3_config.outputs.job_type }} \ + --s3-prefix "${{ steps.s3_config.outputs.s3_prefix }}" diff --git a/build_tools/packaging/linux/build_package.py b/build_tools/packaging/linux/build_package.py index 0881b689b7c..fe94f259768 100755 --- a/build_tools/packaging/linux/build_package.py +++ b/build_tools/packaging/linux/build_package.py @@ -23,6 +23,7 @@ import shutil import subprocess import sys +import traceback from datetime import datetime, timezone from email.utils import format_datetime @@ -56,8 +57,10 @@ def create_deb_package(pkg_name, config: PackageConfig): print(f"Package Name: {pkg_name}") # Non-versioned packages are not required for RPATH packages + # In multi-arch mode, only create non-versioned packages for generic architecture if not config.enable_rpath: - create_nonversioned_deb_package(pkg_name, config) + if not config.enable_kpack or config.gfx_arch == GFX_GENERIC: + create_nonversioned_deb_package(pkg_name, config) create_versioned_deb_package(pkg_name, config) output_list = move_packages_to_destination(pkg_name, config) @@ -132,13 +135,21 @@ def create_versioned_deb_package(pkg_name, config: PackageConfig): sourcedir_list = [] dir_list = filter_components_fromartifactory( - pkg_name, config.artifacts_dir, config.gfx_arch + pkg_name, config.artifacts_dir, config.gfx_arch, config.enable_kpack ) sourcedir_list.extend(dir_list) print(f"sourcedir_list:\n {sourcedir_list}") if not sourcedir_list and not is_meta: - sys.exit(f"{pkg_name}: Empty sourcedir_list and not a meta package, exiting") + if config.enable_kpack: + print( + f"ERROR: {pkg_name}: Empty sourcedir_list and not a meta package, skipping" + ) + return [] + else: + sys.exit( + f"{pkg_name}: Empty sourcedir_list and not a meta package, exiting" + ) if not sourcedir_list: print(f"{pkg_name} is a Meta package") @@ -263,6 +274,14 @@ def generate_rules_file(pkg_info, deb_dir, config: PackageConfig): # Get package name for changelog installation pkg_name = update_package_name(pkg_info.get("Package"), config) + # Disable debian dh_strip for multi-arch builds + # WORKAROUND: dh_strip's debugedit incorrectly truncates ELF files with + # unconventional layouts (e.g., program headers at end of file). + # This causes "program header goes past the end of the file" errors. + # See: https://github.com/ROCm/TheRock/issues/4047 + if config.enable_kpack: + disable_dh_strip = True + env = Environment(loader=FileSystemLoader(str(SCRIPT_DIR))) template = env.get_template("template/debian_rules.j2") # Prepare context dictionary @@ -291,47 +310,32 @@ def generate_control_file(pkg_info, deb_dir, config: PackageConfig): """ print_function_name() control_file = Path(deb_dir) / "control" - pkg_name = pkg_info.get("Package") - provides = "" - replaces = "" - conflicts = "" - debrecommends = "" - debsuggests = "" + is_meta = is_meta_package(pkg_info) - if config.versioned_pkg: - recommends_list = pkg_info.get("DEBRecommends", []) - debrecommends = convert_to_versiondependency(recommends_list, config) - suggests_list = pkg_info.get("DEBSuggests", []) - debsuggests = convert_to_versiondependency(suggests_list, config) + # Initialize optional fields + provides = replaces = conflicts = "" + debrecommends = debsuggests = "" - depends_list = pkg_info.get("DEBDepends", []) + if config.versioned_pkg: + # Get -> Filter -> Transform + debrecommends = process_dependency_field(pkg_info, "DEBRecommends", config) + debsuggests = process_dependency_field(pkg_info, "DEBSuggests", config) + depends = process_dependency_field( + pkg_info, "DEBDepends", config, use_multiarch=True + ) else: - depends_list = [pkg_name] - provides_list = [ - debian_replace_devel_name(pkg) - for pkg in (pkg_info.get("Provides", []) or []) - ] - provides = ", ".join(provides_list) - replaces_list = [ - debian_replace_devel_name(pkg) - for pkg in (pkg_info.get("Replaces", []) or []) - ] - replaces = ", ".join(replaces_list) - conflicts_list = [ - debian_replace_devel_name(pkg) - for pkg in (pkg_info.get("Conflicts", []) or []) - ] - conflicts = ", ".join(conflicts_list) - - depends = convert_to_versiondependency(depends_list, config) - if is_meta_package(pkg_info): - depends = append_version_suffix(depends, config) + # Get -> Transform -> Join + provides = process_name_field(pkg_info, "Provides", debian_replace_devel_name) + replaces = process_name_field(pkg_info, "Replaces", debian_replace_devel_name) + conflicts = process_name_field(pkg_info, "Conflicts", debian_replace_devel_name) + # Non-versioned package depends on versioned package itself + depends = resolve_versioned_dependencies([pkg_name], config, is_meta) pkg_name = update_package_name(pkg_name, config) + env = Environment(loader=FileSystemLoader(str(SCRIPT_DIR))) template = env.get_template("template/debian_control.j2") - # Prepare your context dictionary context = { "source": pkg_name, "depends": depends, @@ -478,8 +482,11 @@ def create_rpm_package(pkg_name, config: PackageConfig): print_function_name() print(f"Package Name: {pkg_name}") + # Non-versioned packages are not required for RPATH packages + # In multi-arch mode, only create non-versioned packages for generic architecture if not config.enable_rpath: - create_nonversioned_rpm_package(pkg_name, config) + if not config.enable_kpack or config.gfx_arch == GFX_GENERIC: + create_nonversioned_rpm_package(pkg_name, config) create_versioned_rpm_package(pkg_name, config) output_list = move_packages_to_destination(pkg_name, config) @@ -547,15 +554,12 @@ def generate_spec_file(pkg_name, specfile, config: PackageConfig): os.makedirs(os.path.dirname(specfile), exist_ok=True) pkg_info = get_package_info(pkg_name) - # populate packge version details version = f"{config.rocm_version}" - # TBD: Whether to use component version details? - # version = pkg_info.get("Version") - provides = "" - obsoletes = "" - conflicts = "" - rpmrecommends = "" - rpmsuggests = "" + is_meta = is_meta_package(pkg_info) + + # Initialize optional fields + provides = obsoletes = conflicts = "" + rpmrecommends = rpmsuggests = "" sourcedir_list = [] rpm_scripts = [] # amdrocm-debugger: Exclude libpython requirements @@ -564,21 +568,32 @@ def generate_spec_file(pkg_name, specfile, config: PackageConfig): exclude_libpython_requires = pkg_name == "amdrocm-debugger" if config.versioned_pkg: - recommends_list = pkg_info.get("RPMRecommends", []) - rpmrecommends = convert_to_versiondependency(recommends_list, config) - suggests_list = pkg_info.get("RPMSuggests", []) - rpmsuggests = convert_to_versiondependency(suggests_list, config) - - requires_list = pkg_info.get("RPMRequires", []) + # Get -> Filter -> Transform + rpmrecommends = process_dependency_field(pkg_info, "RPMRecommends", config) + rpmsuggests = process_dependency_field(pkg_info, "RPMSuggests", config) + requires = process_dependency_field( + pkg_info, "RPMRequires", config, use_multiarch=True + ) dir_list = filter_components_fromartifactory( - pkg_name, config.artifacts_dir, config.gfx_arch + pkg_name, config.artifacts_dir, config.gfx_arch, config.enable_kpack ) sourcedir_list.extend(dir_list) # Filter out non-existing directories sourcedir_list = [path for path in sourcedir_list if os.path.isdir(path)] + # Warn if we have no artifacts for non-meta packages + if not sourcedir_list and not is_meta: + if config.enable_kpack: + print( + f"WARNING: {pkg_name}: Empty sourcedir_list and not a meta package, creating empty RPM" + ) + else: + sys.exit( + f"{pkg_name}: Empty sourcedir_list and not a meta package, exiting" + ) + if is_postinstallscripts_available(pkg_info): rpm_scripts = generate_rpm_postscripts(pkg_info, config) @@ -586,24 +601,17 @@ def generate_spec_file(pkg_name, specfile, config: PackageConfig): for path in sourcedir_list: convert_runpath_to_rpath(path) else: - # Provides, Obsoletes and Conflicts field is only valid - # for non-versioned packages - provides = ", ".join(pkg_info.get("Provides", []) or []) - obsoletes = ", ".join(pkg_info.get("Obsoletes", []) or []) - conflicts = ", ".join(pkg_info.get("Conflicts", []) or []) - requires_list = [pkg_name] - - requires = convert_to_versiondependency(requires_list, config) - if is_meta_package(pkg_info): - requires = append_version_suffix(requires, config) - - # Update package name with version details and gfxarch + # Get -> Transform -> Join (no transform needed for RPM) + provides = process_name_field(pkg_info, "Provides") + obsoletes = process_name_field(pkg_info, "Obsoletes") + conflicts = process_name_field(pkg_info, "Conflicts") + # Non-versioned package requires versioned package itself + requires = resolve_versioned_dependencies([pkg_name], config, is_meta) + pkg_name = update_package_name(pkg_name, config) env = Environment(loader=FileSystemLoader(str(SCRIPT_DIR))) template = env.get_template("template/rpm_specfile.j2") - - # Prepare your context dictionary context = { "pkg_name": pkg_name, "version": version, @@ -738,33 +746,49 @@ def parse_input_package_list(pkg_name, artifact_dir): return pkg_list, skipped_list -def clean_package_build_dir(config: PackageConfig): - """Clean the package build directories +def normalize_target_list(targets: list[str]) -> list[str]: + """Normalize target list by splitting on semicolons, commas, or spaces. - If artifactory directory is provided, clean the same as well + Accepts targets in multiple formats: + - Space-separated CLI args: ['gfx94X-dcgpu', 'gfx120X-all'] + - Single comma-separated string: ['gfx94X-dcgpu,gfx120X-all,gfx1151'] + - Single semicolon-separated string: ['gfx94X-dcgpu;gfx120X-all;gfx1151'] + - Mixed: ['gfx94X-dcgpu;gfx120X-all', 'gfx1151'] - Parameters: - config: Configuration object containing package metadata - - Returns: None + Returns a flat list of individual target names. """ - print_function_name() - PYCACHE_DIR = Path(SCRIPT_DIR) / "__pycache__" - remove_dir(PYCACHE_DIR) + normalized = [] + for target in targets: + # Split by semicolon first, then comma, then whitespace + if ";" in target: + normalized.extend(target.split(";")) + elif "," in target: + normalized.extend(target.split(",")) + else: + # Could be space-separated or single value + normalized.extend(target.split()) - # NOTE: Remove only the build directory - # Make sure the destination directory is not removed - remove_dir(Path(config.dest_dir) / config.pkg_type) - # TBD: - # Currently RPATH packages are created by modifying the artifacts dir - # So artifacts dir clean up is required - # remove_dir(artifacts_dir) + # Remove empty strings and strip whitespace + return [t.strip() for t in normalized if t.strip()] def run(args: argparse.Namespace): # Set the global variables dest_dir = Path(args.dest_dir).expanduser().resolve() + # Normalize target list to handle various input formats + normalized_targets = normalize_target_list(args.target) + + # Configure architecture based on multi-arch mode + if args.enable_kpack: + # Multi-arch mode: use generic default, targets for gfxarch packages + default_gfx_arch = GFX_GENERIC + gfxarch_list = normalized_targets + else: + # Single-arch mode: use first target as default, no additional arch list + default_gfx_arch = normalized_targets[0] + gfxarch_list = [] + # Split version passed to use only major and minor version for prefix folder # Split by dot and take first two components parts = args.rocm_version.split(".") @@ -791,8 +815,10 @@ def run(args: argparse.Namespace): rocm_version=args.rocm_version, version_suffix=args.version_suffix, install_prefix=prefix, - gfx_arch=args.target, + gfx_arch=default_gfx_arch, enable_rpath=args.rpath_pkg, + enable_kpack=args.enable_kpack, + gfxarch_list=gfxarch_list, ) # Clean the packaging build directories @@ -809,18 +835,45 @@ def run(args: argparse.Namespace): f"Invalid package type: {config.pkg_type}. Must be 'deb' or 'rpm'." ) + current_pkg_idx = 0 try: built_pkglist = [] - for pkg_name in pkg_list: + failed_pkglist = [] + + for current_pkg_idx, pkg_name in enumerate(pkg_list): print(f"Create {pkg_type} package.") - if pkg_type == "rpm": - output_list = create_rpm_package(pkg_name, config) - else: - output_list = create_deb_package(pkg_name, config) - if output_list: - built_pkglist.extend(output_list) - print(f"Built package List: {built_pkglist}") + pkg_info = get_package_info(pkg_name) + # Check the package is marked as gfxarch package OR meta package + if is_gfxarch_package(pkg_info, config.enable_kpack) or is_meta_package( + pkg_info + ): + # Use all gfxarch values + loop_list = gfxarch_list + [default_gfx_arch] + else: + # Only use default architecture + loop_list = [default_gfx_arch] + + pkg_built = False + for gfxarch in loop_list: + config.gfx_arch = gfxarch + if pkg_type == "rpm": + output_list = create_rpm_package(pkg_name, config) + else: + output_list = create_deb_package(pkg_name, config) + + if output_list: + built_pkglist.extend(output_list) + pkg_built = True + print(f"Built package List: {built_pkglist}") + else: + # Add failed architecture variant to failed list + variant_name = ( + f"{pkg_name}-{gfxarch}" + if gfxarch != default_gfx_arch + else pkg_name + ) + failed_pkglist.append(variant_name) # Clean the build directories clean_package_build_dir(config) @@ -829,17 +882,28 @@ def run(args: argparse.Namespace): total=pkg_list, built=built_pkglist, skipped=skipped_list, + failed=failed_pkglist, ) # Print build summary print_build_summary(config, pkglist_status) - except SystemExit: + except SystemExit as e: # Build aborted somewhere inside create_* functions - print("\n❌ Build aborted due to an error.\n") + tb = traceback.extract_tb(sys.exc_info()[2]) + if tb: + filename, line_no, func, text = tb[-1] + print(f"\n❌ Build aborted due to an error at {filename}:{line_no}: {e}\n") + else: + print(f"\n❌ Build aborted due to an error: {e}\n") + # Record failed package and all pending packages + failed_pkglist.append(pkg_list[current_pkg_idx]) + pending_pkgs = pkg_list[current_pkg_idx + 1 :] + failed_pkglist.extend(pending_pkgs) pkglist_status = PackageList( total=pkg_list, built=built_pkglist, skipped=skipped_list, + failed=failed_pkglist, ) print_build_summary(config, pkglist_status) # Stop the program @@ -865,8 +929,9 @@ def main(argv: list[str]): p.add_argument( "--target", type=str, + nargs="+", required=True, - help="Graphics architecture used for the artifacts", + help="Graphics architecture(s) used for the artifacts (can specify multiple)", ) p.add_argument( @@ -899,6 +964,12 @@ def main(argv: list[str]): help="Enable rpath-pkg mode", ) + p.add_argument( + "--enable-kpack", + action="store_true", + help="Enable multi-architecture package generation", + ) + p.add_argument( "--clean-build", action="store_true", diff --git a/build_tools/packaging/linux/get_s3_config.py b/build_tools/packaging/linux/get_s3_config.py new file mode 100644 index 00000000000..ec78cea96e3 --- /dev/null +++ b/build_tools/packaging/linux/get_s3_config.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 + +# SPDX-License-Identifier: MIT +# Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. + +""" +Determine S3 bucket and prefix for ROCm package uploads. + +This script implements the S3 bucket selection logic for native Linux packages, +determining the appropriate bucket and prefix based on release type, repository, +and fork status. + +Decision Tree: + ├─ IF release_type is set (dev/nightly/prerelease/release) + │ └─ Use: therock-${release_type}-packages bucket + │ ├─ prerelease/release → prefix: v3/packages/ + │ └─ dev/nightly → prefix: v3/packages//- + │ + ├─ ELSE IF fork PR OR non-ROCm/TheRock repository + │ └─ Use: therock-ci-artifacts-external bucket (external CI) + │ └─ prefix: v3/packages//- + │ + └─ ELSE (default: ROCm/TheRock non-fork) + └─ Use: therock-ci-artifacts bucket (internal CI) + └─ prefix: v3/packages//- + +Usage: + # GitHub Actions output format + python get_s3_config.py \\ + --release-type dev \\ + --repository "ROCm/TheRock" \\ + --is-fork false \\ + --pkg-type deb \\ + --artifact-id 12345678 \\ + --output-format github + + # Environment variables format + python get_s3_config.py \\ + --release-type "" \\ + --repository "ROCm/TheRock" \\ + --is-fork false \\ + --pkg-type rpm \\ + --artifact-id 12345678 \\ + --output-format env + + # JSON format + python get_s3_config.py \\ + --release-type nightly \\ + --repository "ROCm/TheRock" \\ + --is-fork false \\ + --pkg-type deb \\ + --artifact-id 12345678 \\ + --output-format json +""" + +import argparse +import json +import re +import sys +from datetime import datetime +from typing import Optional, Tuple + + +def extract_date_from_version(version: Optional[str]) -> str: + """ + Extract date from ROCm package version string. + + Supports various version formats: + - Debian dev: 8.1.0~dev20251203 → 20251203 + - Debian nightly: 8.1.0~20251203 → 20251203 + - RPM: 8.1.0~20251203gf689a8e → 20251203 + - Wheel/alpha: 7.10.0a20251021 → 20251021 + + Falls back to current date if no date is found in the version string. + + Args: + version: ROCm package version string (may be None) + + Returns: + Date string in YYYYMMDD format + """ + if not version: + return datetime.now().strftime("%Y%m%d") + + # Look for 8-digit date pattern (YYYYMMDD) + # Common patterns: ~dev20251203, ~20251203, a20251021, ~20251203gf689a8e + match = re.search(r"(\d{8})", version) + if match: + return match.group(1) + + # No date found, fall back to current date + return datetime.now().strftime("%Y%m%d") + + +def determine_s3_config( + release_type: str, + repository: str, + is_fork: bool, + pkg_type: str, + artifact_id: str, + rocm_version: Optional[str] = None, +) -> Tuple[str, str, str]: + """ + Determine S3 bucket, prefix, and job type based on inputs. + + Args: + release_type: Release type ('dev', 'nightly', 'prerelease', 'release', 'ci', or empty) + repository: GitHub repository name (e.g., 'ROCm/TheRock') + is_fork: Whether this is a fork PR + pkg_type: Package type ('deb' or 'rpm') + artifact_id: Artifact/run ID for versioning + rocm_version: ROCm package version string (for date extraction) + + Returns: + Tuple of (s3_bucket, s3_prefix, job_type) + """ + # Extract date from version for consistency between version and S3 path + yyyymmdd = extract_date_from_version(rocm_version) + + # Branch 1: Release-type-specific package buckets (dev/nightly/prerelease/release) + # Note: 'ci' or empty string should fall through to CI bucket logic below + if release_type and release_type not in ("", "ci"): + s3_bucket = f"therock-{release_type}-packages" + + if release_type in ("prerelease", "release"): + # Prerelease/Release packages go to stable prefix (no date subfolder) + s3_prefix = f"v3/packages/{pkg_type}" + job_type = release_type + print(f"✓ Using release-type bucket: {s3_bucket}", file=sys.stderr) + else: + # Dev/Nightly packages go to dated subfolder for versioning + s3_prefix = f"v3/packages/{pkg_type}/{yyyymmdd}-{artifact_id}" + job_type = release_type + print(f"✓ Using release-type bucket: {s3_bucket}", file=sys.stderr) + + # Branch 2: Fork PRs or external repositories + elif is_fork or repository != "ROCm/TheRock": + s3_bucket = "therock-ci-artifacts-external" + s3_prefix = f"v3/packages/{pkg_type}/{yyyymmdd}-{artifact_id}" + job_type = "ci" + print(f"✓ Using external bucket: {s3_bucket}", file=sys.stderr) + + # Branch 3: Default - ROCm/TheRock non-fork (normal CI builds) + else: + s3_bucket = "therock-ci-artifacts" + s3_prefix = f"v3/packages/{pkg_type}/{yyyymmdd}-{artifact_id}" + job_type = "ci" + print(f"✓ Using default CI bucket: {s3_bucket}", file=sys.stderr) + + print(f"S3 bucket: {s3_bucket}", file=sys.stderr) + print(f"S3 prefix: {s3_prefix}", file=sys.stderr) + print(f"Job type: {job_type}", file=sys.stderr) + + return s3_bucket, s3_prefix, job_type + + +def main(): + parser = argparse.ArgumentParser( + description="Determine S3 bucket and prefix for ROCm package uploads", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + + parser.add_argument( + "--release-type", + required=True, + help="Release type ('dev', 'nightly', 'prerelease', 'release', 'ci', or empty string for CI builds)", + ) + parser.add_argument( + "--repository", + required=True, + help="GitHub repository name (e.g., 'ROCm/TheRock')", + ) + parser.add_argument( + "--is-fork", + required=True, + help="Whether this is a fork PR ('true' or 'false')", + ) + parser.add_argument( + "--pkg-type", + required=True, + choices=["deb", "rpm"], + help="Package type", + ) + parser.add_argument( + "--artifact-id", + required=True, + help="Artifact/run ID for versioning", + ) + parser.add_argument( + "--rocm-version", + required=False, + default=None, + help="ROCm package version string (for date extraction, e.g., '8.1.0~dev20251203')", + ) + parser.add_argument( + "--output-format", + choices=["env", "json", "github"], + default="env", + help="Output format: 'env' for shell variables, 'json' for JSON, 'github' for GitHub Actions", + ) + + args = parser.parse_args() + + # Convert string to boolean + is_fork = args.is_fork.lower() in ("true", "1", "yes") + + # Determine S3 configuration + s3_bucket, s3_prefix, job_type = determine_s3_config( + release_type=args.release_type, + repository=args.repository, + is_fork=is_fork, + pkg_type=args.pkg_type, + artifact_id=args.artifact_id, + rocm_version=args.rocm_version, + ) + + # Output in requested format + if args.output_format == "json": + output = { + "s3_bucket": s3_bucket, + "s3_prefix": s3_prefix, + "job_type": job_type, + } + print(json.dumps(output, indent=2)) + elif args.output_format == "github": + # GitHub Actions output format + print(f"s3_bucket={s3_bucket}") + print(f"s3_prefix={s3_prefix}") + print(f"job_type={job_type}") + else: # env format + print(f"export S3_BUCKET={s3_bucket}") + print(f"export S3_PREFIX={s3_prefix}") + print(f"export JOB_TYPE={job_type}") + + +if __name__ == "__main__": + main() diff --git a/build_tools/packaging/linux/package.json b/build_tools/packaging/linux/package.json index b245c53d193..fe9010c5f12 100644 --- a/build_tools/packaging/linux/package.json +++ b/build_tools/packaging/linux/package.json @@ -1746,12 +1746,19 @@ "Components": [ "test" ] + }, + { + "Name": "rccl-tests", + "Components": [ + "test" + ] } ] } ], - "Gfxarch": "True" + "Gfxarch": "True", + "DisablePackaging": "True" }, { diff --git a/build_tools/packaging/linux/packaging_summary.py b/build_tools/packaging/linux/packaging_summary.py index 99fe4f8f67a..f99395c38e4 100644 --- a/build_tools/packaging/linux/packaging_summary.py +++ b/build_tools/packaging/linux/packaging_summary.py @@ -15,6 +15,8 @@ class PackageList: built: List[str] # Base packages that were skipped skipped: List[str] + # Base packages that failed to produce any output + failed: List[str] = field(default_factory=list) def write_build_manifest(config: PackageConfig, pkg_list: PackageList): @@ -32,9 +34,8 @@ def write_build_manifest(config: PackageConfig, pkg_list: PackageList): manifest_file = Path(config.dest_dir) / "built_packages.txt" total_basepkg = len(pkg_list.total) + len(pkg_list.skipped) - expected = 2 * len(pkg_list.total) built = len(pkg_list.built) - failed = expected - built + failed = len(pkg_list.failed) try: with open(manifest_file, "w", encoding="utf-8") as f: @@ -47,9 +48,6 @@ def write_build_manifest(config: PackageConfig, pkg_list: PackageList): ) f.write(f"# Total base packages: {total_basepkg}\n") f.write(f"# Skipped base packages: {len(pkg_list.skipped)}\n") - f.write( - f"# Total packages attempted (versioned + non-versioned): {expected}\n" - ) f.write(f"# Successfully built: {built}\n") f.write(f"# Failed to build: {failed}\n") f.write(f"\n") @@ -59,6 +57,11 @@ def write_build_manifest(config: PackageConfig, pkg_list: PackageList): for pkg in sorted(pkg_list.built): f.write(f"{pkg}\n") + if pkg_list.failed: + f.write(f"\n# Failed Packages:\n") + for pkg in sorted(pkg_list.failed): + f.write(f"{pkg}\n") + if pkg_list.skipped: f.write(f"\n# Skipped Packages:\n") f.write( @@ -86,13 +89,11 @@ def print_build_status(config: PackageConfig, pkg_list: PackageList): print("=" * 80) total_basepkg = len(pkg_list.total) + len(pkg_list.skipped) - expected = 2 * len(pkg_list.total) built = len(pkg_list.built) - failed = expected - built + failed = len(pkg_list.failed) print(f"\nTotal base packages: {total_basepkg} ") print(f"⏭️ Skipped base packages: {len(pkg_list.skipped)}") - print(f"Total packages attempted (versioned + non-versioned): {expected}") print(f"✅ Successfully built: {built}") print(f"❌ Failed to build: {failed}") @@ -100,6 +101,11 @@ def print_build_status(config: PackageConfig, pkg_list: PackageList): for pkg in sorted(pkg_list.built): print(f" - {pkg}") + if pkg_list.failed: + print(f"\n❌ Failed packages") + for pkg in sorted(pkg_list.failed): + print(f" - {pkg}") + if pkg_list.skipped: print(f"\n⏭️ Skipped packages") print(f" (Base package names from package.json)") diff --git a/build_tools/packaging/linux/packaging_utils.py b/build_tools/packaging/linux/packaging_utils.py index 879a0ab2c33..84d086cf12d 100644 --- a/build_tools/packaging/linux/packaging_utils.py +++ b/build_tools/packaging/linux/packaging_utils.py @@ -15,15 +15,22 @@ from pathlib import Path +# Constants +# Used for creating a generic package in Multi arch mode +GFX_GENERIC = "gfx_generic" + + # User inputs required for packaging # dest_dir - For saving the rpm/deb packages # pkg_type - Package type DEB or RPM # rocm_version - Used along with package name # version_suffix - Used along with package name # install_prefix - Install prefix for the package -# gfx_arch - gfxarch used for building artifacts +# gfx_arch - gfxarch used for building package # enable_rpath - To enable RPATH packages # versioned_pkg - Used to indicate versioned or non versioned packages +# enable_kpack - To enable multi-architecture support +# gfxarch_list - List of all architectures for multi-arch mode @dataclass class PackageConfig: artifacts_dir: Path @@ -35,6 +42,8 @@ class PackageConfig: gfx_arch: str enable_rpath: bool = field(default=False) versioned_pkg: bool = field(default=True) + enable_kpack: bool = field(default=False) + gfxarch_list: list = field(default_factory=list) SCRIPT_DIR = Path(__file__).resolve().parent @@ -197,19 +206,21 @@ def is_packaging_disabled(pkg_info): return is_key_defined(pkg_info, "Disablepackaging") -def is_gfxarch_package(pkg_info): +def is_gfxarch_package(pkg_info, enable_kpack=False): """Check whether the package is associated with a graphics architecture Parameters: pkg_info (dict): A dictionary containing package details. + enable_kpack (bool): Enable multi-architecture support. Returns: bool : True if Gfxarch is set, else False. - #False if devel package + False if devel package when enable_kpack is True """ - # Disabling this for time being as per the requirements - # if pkgname.endswith("-devel"): - # return False + if enable_kpack: + pkgname = pkg_info.get("Package", "") + if pkgname.endswith("-devel"): + return False return is_key_defined(pkg_info, "Gfxarch") @@ -254,7 +265,7 @@ def get_package_list(artifact_dir): try: dir_entries = os.listdir(artifact_dir) except FileNotFoundError: - sys.exit(f"{artifact_dir}: Artifactory directory doesn not exist, Exiting") + sys.exit(f"{artifact_dir}: Artifactory directory does not exist, Exiting") for pkg_info in data: pkg_name = pkg_info["Package"] @@ -377,14 +388,44 @@ def update_package_name(pkg_name, config: PackageConfig): updated_pkgname += pkg_suffix - if is_gfxarch_package(pkg_info): - # Remove -dcgpu from gfx_arch - gfx_arch = config.gfx_arch.lower().split("-", 1)[0] - updated_pkgname += "-" + gfx_arch + if is_gfxarch_package(pkg_info, config.enable_kpack): + # For multi-arch mode, skip appending gfx_generic + if config.enable_kpack and config.gfx_arch == GFX_GENERIC: + pass # Don't append gfx_generic in multi-arch mode + else: + # Remove -dcgpu from gfx_arch + gfx_arch = config.gfx_arch.lower().split("-", 1)[0] + updated_pkgname += "-" + gfx_arch return updated_pkgname +def expand_metapackage_to_all_archs(pkg_name, gfxarch_list, config: PackageConfig): + """Expand a generic metapackage dependency to include all architecture-specific variants. + + For example, if pkg_name is "amdrocm-core" and gfxarch_list is ["gfx94x", "gfx1150"], + this returns a list: ["amdrocm-core-gfx94x", "amdrocm-core-gfx1150"] + + Parameters: + pkg_name: Base package name (e.g., "amdrocm-core") + gfxarch_list: List of architecture targets + config: Configuration object containing package metadata + + Returns: List of architecture-specific package names + """ + arch_specific_packages = [] + local_config = copy.deepcopy(config) + local_config.versioned_pkg = True + + for gfx_arch in gfxarch_list: + local_config.gfx_arch = gfx_arch + # update_package_name will append version and gfx_arch + arch_pkg = update_package_name(pkg_name, local_config) + arch_specific_packages.append(arch_pkg) + + return arch_specific_packages + + def debian_replace_devel_name(pkg_name): """Replace '-devel' with '-dev' in the package name. @@ -405,7 +446,65 @@ def debian_replace_devel_name(pkg_name): return pkg_name -def convert_to_versiondependency(dependency_list, config: PackageConfig): +def process_name_field( + pkg_info: dict, + field_key: str, + transform_fn=None, +) -> str: + """Process a name field: get -> transform -> join. + + For non-dependency fields: Provides, Replaces, Conflicts, Obsoletes + + Parameters: + pkg_info: Package details from JSON + field_key: Key to extract (e.g., "Provides", "Conflicts") + transform_fn: Optional function to transform each name + + Returns: Comma-separated string of names + """ + name_list = pkg_info.get(field_key, []) or [] + if transform_fn: + name_list = [transform_fn(name) for name in name_list] + return ", ".join(name_list) + + +def process_dependency_field( + pkg_info: dict, + field_key: str, + config: PackageConfig, + use_multiarch: bool = False, +) -> str: + """Process a dependency field with 3-step pattern. + + Works for: DEBDepends, DEBRecommends, DEBSuggests, + RPMRequires, RPMRecommends, RPMSuggests + + Parameters: + pkg_info: Package details from JSON + field_key: Key to extract (e.g., "DEBDepends", "RPMRecommends") + config: Configuration object containing package metadata + use_multiarch: If True, apply multi-arch expansion for main dependencies + + Returns: Comma-separated string of versioned dependencies + """ + is_meta = is_meta_package(pkg_info) + # Step 1 & 2: Get + Filter + if use_multiarch: + dep_list = get_dependency_list_for_multiarch(pkg_info, field_key, config) + else: + dep_list = pkg_info.get(field_key, []) or [] + + # Return empty string if no dependencies + if not dep_list: + return "" + + # Step 3: Transform + return resolve_versioned_dependencies(dep_list, config, is_meta) + + +def convert_to_versiondependency( + dependency_list, config: PackageConfig, preserve_arch=False +): """Change ROCm package dependencies to versioned ones. If a package depends on any packages listed in `pkg_list`, @@ -414,6 +513,7 @@ def convert_to_versiondependency(dependency_list, config: PackageConfig): Parameters: dependency_list : List of dependent packages config: Configuration object containing package metadata + preserve_arch: If True, preserve the gfx_arch from config instead of forcing generic Returns: A string of comma separated versioned packages """ @@ -423,6 +523,10 @@ def convert_to_versiondependency(dependency_list, config: PackageConfig): local_config = copy.deepcopy(config) local_config.versioned_pkg = True + # In multi-arch mode, dependencies should always point to generic packages + # UNLESS preserve_arch is True (for arch-specific metapackages) + if config.enable_kpack and not preserve_arch: + local_config.gfx_arch = GFX_GENERIC pkg_list, skipped_list = get_package_list(config.artifacts_dir) filtered_deps = [] @@ -524,7 +628,9 @@ def move_packages_to_destination(pkg_name, config: PackageConfig): return output_packages -def filter_components_fromartifactory(pkg_name, artifacts_dir, gfx_arch): +def filter_components_fromartifactory( + pkg_name, artifacts_dir, gfx_arch, enable_kpack=False +): """Get the list of Artifactory directories required for creating the package. The `package.json` file defines the required artifactories for each package. @@ -533,6 +639,7 @@ def filter_components_fromartifactory(pkg_name, artifacts_dir, gfx_arch): pkg_name : package name artifacts_dir : Directory where artifacts are saved gfx_arch : graphics architecture + enable_kpack : enable multi-architecture support Returns: List of directories """ @@ -541,7 +648,16 @@ def filter_components_fromartifactory(pkg_name, artifacts_dir, gfx_arch): pkg_info = get_package_info(pkg_name) sourcedir_list = [] - dir_suffix = gfx_arch if is_gfxarch_package(pkg_info) else "generic" + if enable_kpack: + dir_suffix = ( + gfx_arch + if (is_gfxarch_package(pkg_info, enable_kpack) and gfx_arch != GFX_GENERIC) + else "generic" + ) + else: + dir_suffix = ( + gfx_arch if is_gfxarch_package(pkg_info, enable_kpack) else "generic" + ) artifactory = pkg_info.get("Artifactory") if artifactory is None: @@ -594,3 +710,187 @@ def filter_components_fromartifactory(pkg_name, artifacts_dir, gfx_arch): continue return sourcedir_list + + +def clean_package_build_dir(config: PackageConfig): + """Clean the package build directories + + If artifactory directory is provided, clean the same as well + + Parameters: + config: Configuration object containing package metadata + + Returns: None + """ + print_function_name() + PYCACHE_DIR = Path(SCRIPT_DIR) / "__pycache__" + remove_dir(PYCACHE_DIR) + + # NOTE: Remove only the build directory + # Make sure the destination directory is not removed + remove_dir(Path(config.dest_dir) / config.pkg_type) + # TBD: + # Currently RPATH packages are created by modifying the artifacts dir + # So artifacts dir clean up is required + # remove_dir(artifacts_dir) + + +def resolve_versioned_dependencies(dep_list, config: PackageConfig, is_meta): + """Resolve a dependency list into a versioned dependency string. + + Handles three cases based on multi-arch mode and package type: + - Generic metapackages in multi-arch mode: dependencies are already expanded + and versioned, so just join and add version suffix. + - Arch-specific metapackages in multi-arch mode: convert dependencies while + preserving architecture, then add version suffix. + - Normal path: convert dependencies and conditionally add version suffix + for metapackages. + + Parameters: + dep_list: List of dependency package names + config: Configuration object containing package metadata + is_meta: Whether this is a metapackage + + Returns: A comma-separated string of versioned dependencies + """ + if ( + config.versioned_pkg + and config.enable_kpack + and is_meta + and config.gfx_arch == GFX_GENERIC + ): + # dep_list already contains versioned arch-specific package names + # Just add version suffix and join + deps = append_version_suffix(", ".join(dep_list), config) + elif config.enable_kpack and is_meta and config.gfx_arch != GFX_GENERIC: + # Arch-specific metapackage: preserve architecture for gfxarch dependencies + deps = convert_to_versiondependency(dep_list, config, preserve_arch=True) + deps = append_version_suffix(deps, config) + elif config.enable_kpack and not is_meta and config.gfx_arch != GFX_GENERIC: + # Gfx-specific non-meta package: + # dep_list[0] is the versioned-dependency (resolved as generic) + # dep_list[1:] are gfxarch dependencies (resolved with arch suffix) + if not dep_list: + deps = "" + else: + version_deps = convert_to_versiondependency([dep_list[0]], config) + if len(dep_list) > 1: + gfx_deps = convert_to_versiondependency( + dep_list[1:], config, preserve_arch=True + ) + deps = f"{version_deps}, {gfx_deps}" + else: + deps = version_deps + else: + # Normal path: convert dependencies and add version suffix + deps = convert_to_versiondependency(dep_list, config) + if is_meta: + deps = append_version_suffix(deps, config) + return deps + + +def has_artifact_for_arch(pkg_name, artifacts_dir, gfx_arch): + """Check if a package has artifacts available for a specific architecture. + + Parameters: + pkg_name: Package name to check + artifacts_dir: Directory where artifacts are stored + gfx_arch: Graphics architecture to check for + + Returns: True if artifacts exist for the architecture, False otherwise + """ + pkg_info = get_package_info(pkg_name) + if pkg_info is None: + return False + + # Non-gfxarch packages don't need arch-specific artifacts + if not is_gfxarch_package(pkg_info, enable_kpack=True): + return True + + # Meta packages don't have their own artifacts + if is_meta_package(pkg_info): + return True + + artifactory = pkg_info.get("Artifactory") + if artifactory is None: + return False + + # Check if at least one required artifact directory exists for this architecture + for artifact in artifactory: + artifact_prefix = artifact["Artifact"] + # Check for artifact-specific gfxarch override + if "Artifact_Gfxarch" in artifact: + is_gfxarch = str(artifact["Artifact_Gfxarch"]).lower() == "true" + artifact_suffix = gfx_arch if is_gfxarch else "generic" + else: + artifact_suffix = gfx_arch + + for subdir in artifact["Artifact_Subdir"]: + component_list = subdir["Components"] + for component in component_list: + source_dir = ( + Path(artifacts_dir) + / f"{artifact_prefix}_{component}_{artifact_suffix}" + ) + if source_dir.exists(): + return True + + return False + + +def get_dependency_list_for_multiarch(pkg_info, dep_key, config: PackageConfig): + """Determine the appropriate dependency list for multi-arch mode. + + Parameters: + pkg_info: Package details from JSON + dep_key: Dependency key ("DEBDepends" or "RPMRequires") + config: Configuration object containing package metadata + + Returns: List of dependency package names + """ + pkg_name = pkg_info.get("Package") + is_meta = is_meta_package(pkg_info) + + if config.enable_kpack and is_meta: + # For metapackages in multi-arch mode: + # - Generic variant depends on all arch-specific variants + # - Arch-specific variants depend on actual runtime packages + if config.gfx_arch == GFX_GENERIC: + # Generic metapackage: depend on all arch-specific metapackages + return expand_metapackage_to_all_archs( + pkg_name, config.gfxarch_list, config + ) + else: + # Arch-specific metapackage: depend on actual runtime packages + # Filter out dependencies that don't have artifacts for this architecture + dep_list = pkg_info.get(dep_key, []) + return [ + dep + for dep in dep_list + if has_artifact_for_arch(dep, config.artifacts_dir, config.gfx_arch) + ] + elif config.enable_kpack and config.gfx_arch == GFX_GENERIC: + # Generic package in multi-arch mode: + # Only include non-gfxarch dependencies + # Gfxarch deps are pulled via the gfx-specific package + dep_list = pkg_info.get(dep_key, []) + return [ + dep + for dep in dep_list + if not is_gfxarch_package(get_package_info(dep) or {}, config.enable_kpack) + ] + elif config.enable_kpack and config.gfx_arch != GFX_GENERIC: + # Gfx-specific package in multi-arch mode: + # Depend on generic self + gfxarch dependencies with arch suffix + # Filter out dependencies that don't have artifacts for this architecture + dep_list = pkg_info.get(dep_key, []) + gfxarch_deps = [ + dep + for dep in dep_list + if is_gfxarch_package(get_package_info(dep) or {}, config.enable_kpack) + and has_artifact_for_arch(dep, config.artifacts_dir, config.gfx_arch) + ] + return [pkg_name] + gfxarch_deps + else: + # Single-arch mode: use full dependencies + return pkg_info.get(dep_key, []) diff --git a/build_tools/packaging/linux/tests/get_s3_config_test.py b/build_tools/packaging/linux/tests/get_s3_config_test.py new file mode 100644 index 00000000000..bed5cf10106 --- /dev/null +++ b/build_tools/packaging/linux/tests/get_s3_config_test.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 + +# SPDX-License-Identifier: MIT +# Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. + +import os +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + +# Add parent directory to path to import the module +sys.path.insert(0, os.fspath(Path(__file__).parent.parent)) +import get_s3_config + + +class ExtractDateFromVersionTest(unittest.TestCase): + """Tests for date extraction from ROCm package versions.""" + + def test_deb_dev_version(self): + """Test extracting date from Debian dev version.""" + date = get_s3_config.extract_date_from_version("8.1.0~dev20251203") + self.assertEqual(date, "20251203") + + def test_deb_nightly_version(self): + """Test extracting date from Debian nightly version.""" + date = get_s3_config.extract_date_from_version("8.1.0~20251203") + self.assertEqual(date, "20251203") + + def test_rpm_dev_version(self): + """Test extracting date from RPM dev version with git SHA.""" + date = get_s3_config.extract_date_from_version("8.1.0~20251203gf689a8e") + self.assertEqual(date, "20251203") + + def test_wheel_nightly_version(self): + """Test extracting date from wheel nightly (alpha) version.""" + date = get_s3_config.extract_date_from_version("7.10.0a20251021") + self.assertEqual(date, "20251021") + + @patch("get_s3_config.datetime") + def test_version_without_date_uses_current(self, mock_datetime): + """Test fallback to current date when version has no date.""" + mock_now = mock_datetime.now.return_value + mock_now.strftime.return_value = "20260312" + + date = get_s3_config.extract_date_from_version("8.1.0") + self.assertEqual(date, "20260312") + mock_now.strftime.assert_called_once_with("%Y%m%d") + + def test_prerelease_version_without_date(self): + """Test prerelease version without date falls back to current date.""" + date = get_s3_config.extract_date_from_version("8.1.0~pre2") + # Should be 8 digits (YYYYMMDD) + self.assertEqual(len(date), 8) + self.assertTrue(date.isdigit()) + + def test_release_version_without_date(self): + """Test release version without date falls back to current date.""" + date = get_s3_config.extract_date_from_version("8.1.0") + # Should be 8 digits (YYYYMMDD) + self.assertEqual(len(date), 8) + self.assertTrue(date.isdigit()) + + +class DetermineS3ConfigReleaseTypeTest(unittest.TestCase): + """Tests for S3 config with different release types.""" + + def test_dev_release_type(self): + """Test dev release type uses dev-packages bucket.""" + bucket, prefix, job_type = get_s3_config.determine_s3_config( + release_type="dev", + repository="ROCm/TheRock", + is_fork=False, + pkg_type="deb", + artifact_id="12345678", + rocm_version="8.1.0~dev20251203", + ) + self.assertEqual(bucket, "therock-dev-packages") + self.assertEqual(prefix, "v3/packages/deb/20251203-12345678") + self.assertEqual(job_type, "dev") + + def test_nightly_release_type(self): + """Test nightly release type uses nightly-packages bucket.""" + bucket, prefix, job_type = get_s3_config.determine_s3_config( + release_type="nightly", + repository="ROCm/TheRock", + is_fork=False, + pkg_type="rpm", + artifact_id="87654321", + rocm_version="8.1.0~20251203", + ) + self.assertEqual(bucket, "therock-nightly-packages") + self.assertEqual(prefix, "v3/packages/rpm/20251203-87654321") + self.assertEqual(job_type, "nightly") + + def test_prerelease_release_type(self): + """Test prerelease uses prerelease-packages bucket without date.""" + bucket, prefix, job_type = get_s3_config.determine_s3_config( + release_type="prerelease", + repository="ROCm/TheRock", + is_fork=False, + pkg_type="deb", + artifact_id="12345678", + rocm_version="8.1.0~pre2", + ) + self.assertEqual(bucket, "therock-prerelease-packages") + self.assertEqual(prefix, "v3/packages/deb") + self.assertEqual(job_type, "prerelease") + + def test_release_release_type(self): + """Test release uses release-packages bucket without date.""" + bucket, prefix, job_type = get_s3_config.determine_s3_config( + release_type="release", + repository="ROCm/TheRock", + is_fork=False, + pkg_type="rpm", + artifact_id="12345678", + rocm_version="8.1.0", + ) + self.assertEqual(bucket, "therock-release-packages") + self.assertEqual(prefix, "v3/packages/rpm") + self.assertEqual(job_type, "release") + + def test_ci_release_type(self): + """Test 'ci' release type falls through to CI bucket logic.""" + bucket, prefix, job_type = get_s3_config.determine_s3_config( + release_type="ci", + repository="ROCm/TheRock", + is_fork=False, + pkg_type="deb", + artifact_id="12345678", + rocm_version=None, + ) + self.assertEqual(bucket, "therock-ci-artifacts") + self.assertIn("v3/packages/deb/", prefix) + self.assertEqual(job_type, "ci") + + def test_empty_release_type(self): + """Test empty release type falls through to CI bucket logic.""" + bucket, prefix, job_type = get_s3_config.determine_s3_config( + release_type="", + repository="ROCm/TheRock", + is_fork=False, + pkg_type="deb", + artifact_id="12345678", + rocm_version=None, + ) + self.assertEqual(bucket, "therock-ci-artifacts") + self.assertIn("v3/packages/deb/", prefix) + self.assertEqual(job_type, "ci") + + +class DetermineS3ConfigRepositoryTest(unittest.TestCase): + """Tests for S3 config with different repositories.""" + + def test_fork_pr(self): + """Test fork PR uses external bucket.""" + bucket, prefix, job_type = get_s3_config.determine_s3_config( + release_type="", + repository="ROCm/TheRock", + is_fork=True, + pkg_type="rpm", + artifact_id="12345678", + rocm_version="8.1.0~dev20251203", + ) + self.assertEqual(bucket, "therock-ci-artifacts-external") + self.assertEqual(prefix, "v3/packages/rpm/20251203-12345678") + self.assertEqual(job_type, "ci") + + def test_external_repository(self): + """Test external repository uses external bucket.""" + bucket, prefix, job_type = get_s3_config.determine_s3_config( + release_type="", + repository="someone/fork", + is_fork=False, + pkg_type="deb", + artifact_id="12345678", + rocm_version="8.1.0~dev20251203", + ) + self.assertEqual(bucket, "therock-ci-artifacts-external") + self.assertEqual(prefix, "v3/packages/deb/20251203-12345678") + self.assertEqual(job_type, "ci") + + def test_default_rocm_therock(self): + """Test default ROCm/TheRock uses ci-artifacts bucket.""" + bucket, prefix, job_type = get_s3_config.determine_s3_config( + release_type="", + repository="ROCm/TheRock", + is_fork=False, + pkg_type="deb", + artifact_id="12345678", + rocm_version="8.1.0~dev20251203", + ) + self.assertEqual(bucket, "therock-ci-artifacts") + self.assertEqual(prefix, "v3/packages/deb/20251203-12345678") + self.assertEqual(job_type, "ci") + + +class DetermineS3ConfigPackageTypeTest(unittest.TestCase): + """Tests for S3 config with different package types.""" + + def test_deb_package_type(self): + """Test deb package type in prefix.""" + bucket, prefix, job_type = get_s3_config.determine_s3_config( + release_type="dev", + repository="ROCm/TheRock", + is_fork=False, + pkg_type="deb", + artifact_id="12345678", + rocm_version="8.1.0~dev20251203", + ) + self.assertIn("deb", prefix) + + def test_rpm_package_type(self): + """Test rpm package type in prefix.""" + bucket, prefix, job_type = get_s3_config.determine_s3_config( + release_type="dev", + repository="ROCm/TheRock", + is_fork=False, + pkg_type="rpm", + artifact_id="12345678", + rocm_version="8.1.0~20251203gf689a8e", + ) + self.assertIn("rpm", prefix) + + +class DetermineS3ConfigDateConsistencyTest(unittest.TestCase): + """Tests to ensure date consistency between version and S3 path.""" + + def test_date_extracted_from_deb_version(self): + """Test date in S3 path matches date in deb version.""" + version = "8.1.0~dev20251203" + bucket, prefix, job_type = get_s3_config.determine_s3_config( + release_type="dev", + repository="ROCm/TheRock", + is_fork=False, + pkg_type="deb", + artifact_id="12345678", + rocm_version=version, + ) + # Date from version should be in the prefix + self.assertIn("20251203", prefix) + + def test_date_extracted_from_rpm_version(self): + """Test date in S3 path matches date in rpm version.""" + version = "8.1.0~20251203gf689a8e" + bucket, prefix, job_type = get_s3_config.determine_s3_config( + release_type="dev", + repository="ROCm/TheRock", + is_fork=False, + pkg_type="rpm", + artifact_id="12345678", + rocm_version=version, + ) + # Date from version should be in the prefix + self.assertIn("20251203", prefix) + + def test_date_extracted_from_wheel_version(self): + """Test date in S3 path matches date in wheel version.""" + version = "7.10.0a20251021" + bucket, prefix, job_type = get_s3_config.determine_s3_config( + release_type="nightly", + repository="ROCm/TheRock", + is_fork=False, + pkg_type="deb", + artifact_id="12345678", + rocm_version=version, + ) + # Date from version should be in the prefix + self.assertIn("20251021", prefix) + + @patch("get_s3_config.datetime") + def test_fallback_to_current_date_when_no_version(self, mock_datetime): + """Test fallback to current date when version is not provided.""" + mock_now = mock_datetime.now.return_value + mock_now.strftime.return_value = "20260312" + + bucket, prefix, job_type = get_s3_config.determine_s3_config( + release_type="dev", + repository="ROCm/TheRock", + is_fork=False, + pkg_type="deb", + artifact_id="12345678", + rocm_version=None, + ) + # Should use current date + self.assertIn("20260312", prefix) + + +if __name__ == "__main__": + unittest.main() diff --git a/build_tools/packaging/linux/upload_package_repo.py b/build_tools/packaging/linux/upload_package_repo.py index 5b6b0efc8ce..ee54e78b8ed 100644 --- a/build_tools/packaging/linux/upload_package_repo.py +++ b/build_tools/packaging/linux/upload_package_repo.py @@ -603,25 +603,44 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument("--pkg-type", required=True, choices=["deb", "rpm"]) parser.add_argument("--s3-bucket", required=True) - parser.add_argument("--amdgpu-family", required=True) + parser.add_argument( + "--amdgpu-family", required=False + ) # Kept for backward compatibility, not used parser.add_argument("--artifact-id", required=True) parser.add_argument( "--job", default="dev", - choices=["dev", "nightly", "prerelease"], - help="Enable dev or nightly shared repo", + choices=["dev", "nightly", "prerelease", "ci"], + help="Job type: dev, nightly, prerelease, or ci", + ) + parser.add_argument( + "--s3-prefix", + required=False, + help="Override S3 prefix (for backward compatibility, auto-generated if not provided)", ) args = parser.parse_args() package_dir = find_package_dir() # Setup the prefix based on build type - if args.job in ["nightly", "dev"]: + if args.s3_prefix: + # Use provided prefix (new behavior for multi-arch CI) + prefix = args.s3_prefix + dedupe = True + elif args.job in ["nightly", "dev"]: + # Legacy behavior: /- prefix = f"{args.pkg_type}/{yyyymmdd()}-{args.artifact_id}" dedupe = True elif args.job == "prerelease": + # Legacy behavior: v3/packages/ prefix = f"v3/packages/{args.pkg_type}" dedupe = True + elif args.job == "ci": + # CI builds: v3/packages//- + prefix = f"v3/packages/{args.pkg_type}/{yyyymmdd()}-{args.artifact_id}" + dedupe = True + else: + raise ValueError(f"Unknown job type: {args.job}") if args.pkg_type == "deb": create_deb_repo(package_dir, args.job)