-
Notifications
You must be signed in to change notification settings - Fork 245
[release/7.11] Cherry-pick: Handle missing Artifactory during packaging #3100
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
dc319e1
b9ca153
3d591d8
c0c6619
1df4d1b
1dbe68f
fcf9965
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -27,6 +27,7 @@ | |
| from datetime import datetime, timezone | ||
| from email.utils import format_datetime | ||
| from jinja2 import Environment, FileSystemLoader, Template | ||
| from packaging_summary import * | ||
| from packaging_utils import * | ||
| from pathlib import Path | ||
| from runpath_to_rpath import * | ||
|
|
@@ -48,7 +49,8 @@ def create_deb_package(pkg_name, config: PackageConfig): | |
| pkg_name : Name of the package to be created | ||
| config: Configuration object containing package metadata | ||
|
|
||
| Returns: None | ||
| Returns: | ||
| output_list: List of packages created | ||
| """ | ||
| print_function_name() | ||
| print(f"Package Name: {pkg_name}") | ||
|
|
@@ -58,9 +60,10 @@ def create_deb_package(pkg_name, config: PackageConfig): | |
| create_nonversioned_deb_package(pkg_name, config) | ||
|
|
||
| create_versioned_deb_package(pkg_name, config) | ||
| move_packages_to_destination(pkg_name, config) | ||
| output_list = move_packages_to_destination(pkg_name, config) | ||
| # Clean debian build directory | ||
| remove_dir(Path(config.dest_dir) / config.pkg_type) | ||
| return output_list | ||
|
|
||
|
|
||
| def create_nonversioned_deb_package(pkg_name, config: PackageConfig): | ||
|
|
@@ -442,6 +445,34 @@ def package_with_dpkg_build(pkg_dir): | |
|
|
||
|
|
||
| ######################## RPM package creation #################### | ||
| def create_rpm_package(pkg_name, config: PackageConfig): | ||
| """Create an RPM package. | ||
|
|
||
| This function invokes the creation of versioned and non-versioned packages | ||
| and moves the resulting `.rpm` files to the destination directory. | ||
|
|
||
| Parameters: | ||
| pkg_name : Name of the package to be created | ||
| config: Configuration object containing package metadata | ||
|
|
||
| Returns: | ||
| output_list: List of packages created | ||
| """ | ||
| print_function_name() | ||
| print(f"Package Name: {pkg_name}") | ||
|
|
||
| # By default both versioned and non versioned packages are created. | ||
| # In case rpath is enabled need to create only versioned package. So skipping nonversioned here | ||
| if not config.enable_rpath: | ||
| create_nonversioned_rpm_package(pkg_name, config) | ||
|
|
||
| create_versioned_rpm_package(pkg_name, config) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. is here an else missing?
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @HereThereBeDragons for rpath packages as per requirement only versioned packages are needed. As a follow up patch will add a comment to make it clear.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ok. my question was here if
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. no it wont. Its a totally different set of packages. nonversioned depends on versioned packages. Here by default we need to create both versioned and nonversioned packages. But if rpath is enabled we need to create only versioned. Hence the code is bit confusing, Will make it more clear with comments
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @nunnikri so if by default we are building both it will be nice if this is added as a comment here. As we are not passing enable_rpath here by default now. |
||
| output_list = move_packages_to_destination(pkg_name, config) | ||
| # Clean rpm build directory | ||
| remove_dir(Path(config.dest_dir) / config.pkg_type) | ||
| return output_list | ||
|
|
||
|
|
||
| def create_nonversioned_rpm_package(pkg_name, config: PackageConfig): | ||
| """Create a non-versioned RPM meta package (.rpm). | ||
|
|
||
|
|
@@ -487,30 +518,6 @@ def create_versioned_rpm_package(pkg_name, config: PackageConfig): | |
| package_with_rpmbuild(specfile) | ||
|
|
||
|
|
||
| def create_rpm_package(pkg_name, config: PackageConfig): | ||
| """Create an RPM package. | ||
|
|
||
| This function invokes the creation of versioned and non-versioned packages | ||
| and moves the resulting `.rpm` files to the destination directory. | ||
|
|
||
| Parameters: | ||
| pkg_name : Name of the package to be created | ||
| config: Configuration object containing package metadata | ||
|
|
||
| Returns: None | ||
| """ | ||
| print_function_name() | ||
| print(f"Package Name: {pkg_name}") | ||
|
|
||
| if not config.enable_rpath: | ||
| create_nonversioned_rpm_package(pkg_name, config) | ||
|
|
||
| create_versioned_rpm_package(pkg_name, config) | ||
| move_packages_to_destination(pkg_name, config) | ||
| # Clean rpm build directory | ||
| remove_dir(Path(config.dest_dir) / config.pkg_type) | ||
|
|
||
|
|
||
| def generate_spec_file(pkg_name, specfile, config: PackageConfig): | ||
| """Generate an RPM spec file. | ||
|
|
||
|
|
@@ -673,20 +680,22 @@ def package_with_rpmbuild(spec_file): | |
|
|
||
|
|
||
| ######################## Begin Packaging Process################################ | ||
| def parse_input_package_list(pkg_name): | ||
| def parse_input_package_list(pkg_name, artifact_dir): | ||
| """Populate the package list from the provided input arguments. | ||
|
|
||
| Parameters: | ||
| pkg_name : List of packages to be created | ||
| artifact_dir: The path to the Artifactory directory | ||
|
|
||
| Returns: Package list | ||
| """ | ||
| print_function_name() | ||
| pkg_list = [] | ||
| skipped_list = [] | ||
| # If pkg_name is None, include all packages | ||
| if pkg_name is None: | ||
| pkg_list = get_package_list() | ||
| return pkg_list | ||
| pkg_list, skipped_list = get_package_list(artifact_dir) | ||
| return pkg_list, skipped_list | ||
|
|
||
| # Proceed if pkg_name is not None | ||
| data = read_package_json_file() | ||
|
|
@@ -705,7 +714,7 @@ def parse_input_package_list(pkg_name): | |
| break | ||
|
|
||
| print(f"pkg_list:\n {pkg_list}") | ||
| return pkg_list | ||
| return pkg_list, skipped_list | ||
|
|
||
|
|
||
| def clean_package_build_dir(config: PackageConfig): | ||
|
|
@@ -766,18 +775,52 @@ def run(args: argparse.Namespace): | |
| # Clean the packaging build directories | ||
| clean_package_build_dir(config) | ||
|
|
||
| pkg_list = parse_input_package_list(args.pkg_names) | ||
| pkg_list, skipped_list = parse_input_package_list( | ||
| args.pkg_names, config.artifacts_dir | ||
| ) | ||
| # Create deb/rpm packages | ||
| package_creators = {"deb": create_deb_package, "rpm": create_rpm_package} | ||
| for pkg_name in pkg_list: | ||
| if config.pkg_type and config.pkg_type.lower() in package_creators: | ||
| print(f"Create {config.pkg_type.upper()} package.") | ||
| package_creators[config.pkg_type.lower()](pkg_name, config) | ||
| else: | ||
| print("Create both DEB and RPM packages.") | ||
| for creator in package_creators.values(): | ||
| creator(pkg_name, config) | ||
| clean_package_build_dir(config) | ||
| valid_types = {"deb", "rpm"} | ||
| pkg_type = (config.pkg_type or "").lower() | ||
| if pkg_type not in valid_types: | ||
| raise ValueError( | ||
| f"Invalid package type: {config.pkg_type}. Must be 'deb' or 'rpm'." | ||
| ) | ||
|
|
||
| try: | ||
| built_pkglist = [] | ||
| for pkg_name in pkg_list: | ||
| print(f"Create {pkg_type} package.") | ||
| if pkg_type == "rpm": | ||
| output_list = create_rpm_package(pkg_name, config) | ||
| else: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. good practice would be this is a fail-safe to catch problems if new package types are added
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The pkg_type check is already done in line #782. Thats why the else for failsafe is not added here. But can see some more options to optimize the code. Will do as a follow up PR. |
||
| output_list = create_deb_package(pkg_name, config) | ||
|
|
||
| if output_list: | ||
| built_pkglist.extend(output_list) | ||
| print(f"Built package List: {built_pkglist}") | ||
|
|
||
| # Clean the build directories | ||
| clean_package_build_dir(config) | ||
|
|
||
| pkglist_status = PackageList( | ||
| total=pkg_list, | ||
| built=built_pkglist, | ||
| skipped=skipped_list, | ||
| ) | ||
|
|
||
| # Print build summary | ||
| print_build_summary(config, pkglist_status) | ||
| except SystemExit: | ||
| # Build aborted somewhere inside create_* functions | ||
| print("\n❌ Build aborted due to an error.\n") | ||
| pkglist_status = PackageList( | ||
| total=pkg_list, | ||
| built=built_pkglist, | ||
| skipped=skipped_list, | ||
| ) | ||
| print_build_summary(config, pkglist_status) | ||
| # Stop the program | ||
| raise | ||
|
|
||
|
|
||
| def main(argv: list[str]): | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| # Copyright Advanced Micro Devices, Inc. | ||
| # SPDX-License-Identifier: MIT | ||
|
|
||
| from dataclasses import dataclass, field | ||
| from datetime import datetime, timezone | ||
| from packaging_utils import * | ||
| from typing import List | ||
|
|
||
|
|
||
| @dataclass | ||
| class PackageList: | ||
| # All base package names that were attempted | ||
| total: List[str] | ||
| # Packages that were successfully created (versioned + non-versioned) | ||
| built: List[str] | ||
| # Base packages that were skipped | ||
| skipped: List[str] | ||
|
|
||
|
|
||
| def write_build_manifest(config: PackageConfig, pkg_list: PackageList): | ||
| """Write manifest files listing built and skipped packages. | ||
|
|
||
| Parameters: | ||
| config: Configuration object containing package metadata | ||
| pkg_list: List of all packages attempted/built/skipped | ||
|
|
||
| Returns: None | ||
| """ | ||
| print_function_name() | ||
|
|
||
| # Write successful packages manifest | ||
| manifest_file = Path(config.dest_dir) / "built_packages.txt" | ||
|
|
||
| total_basepkg = len(pkg_list.total) + len(pkg_list.skipped) | ||
| expected = 2 * len(pkg_list.total) | ||
| built = len(pkg_list.built) | ||
| failed = expected - built | ||
|
|
||
| try: | ||
| with open(manifest_file, "w", encoding="utf-8") as f: | ||
| f.write(f"# Built Packages Manifest\n") | ||
| f.write(f"# Package Type: {config.pkg_type.upper()}\n") | ||
| f.write(f"# ROCm Version: {config.rocm_version}\n") | ||
| f.write(f"# Graphics Architecture: {config.gfx_arch}\n") | ||
| f.write( | ||
| f"# Build Date: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}\n" | ||
| ) | ||
| f.write(f"# Total base packages: {total_basepkg}\n") | ||
| f.write(f"# Skipped base packages: {len(pkg_list.skipped)}\n") | ||
| f.write( | ||
| f"# Total packages attempted (versioned + non-versioned): {expected}\n" | ||
| ) | ||
| f.write(f"# Successfully built: {built}\n") | ||
| f.write(f"# Failed to build: {failed}\n") | ||
| f.write(f"\n") | ||
|
|
||
| if pkg_list.built: | ||
| f.write(f"# Created Packages:\n") | ||
| for pkg in sorted(pkg_list.built): | ||
| f.write(f"{pkg}\n") | ||
|
|
||
| if pkg_list.skipped: | ||
| f.write(f"\n# Skipped Packages:\n") | ||
| f.write( | ||
| f"# Note: Package names shown are base names from package.json\n" | ||
| ) | ||
| for pkg in sorted(pkg_list.skipped): | ||
| f.write(f"{pkg}\n") | ||
|
|
||
| print(f"✅ Built packages manifest written to: {manifest_file}") | ||
| except Exception as e: | ||
| print(f"⚠️ WARNING: Failed to write built packages manifest: {e}") | ||
|
|
||
|
|
||
| def print_build_status(config: PackageConfig, pkg_list: PackageList): | ||
| """Print a summary of the build process. | ||
|
|
||
| Parameters: | ||
| config: Configuration object containing package metadata | ||
| pkg_list: List of all packages attempted/built/skipped | ||
|
|
||
| Returns: None | ||
| """ | ||
| print("\n" + "=" * 80) | ||
| print("BUILD SUMMARY") | ||
| print("=" * 80) | ||
|
|
||
| total_basepkg = len(pkg_list.total) + len(pkg_list.skipped) | ||
| expected = 2 * len(pkg_list.total) | ||
| built = len(pkg_list.built) | ||
| failed = expected - built | ||
|
|
||
| print(f"\nTotal base packages: {total_basepkg} ") | ||
| print(f"⏭️ Skipped base packages: {len(pkg_list.skipped)}") | ||
| print(f"Total packages attempted (versioned + non-versioned): {expected}") | ||
| print(f"✅ Successfully built: {built}") | ||
| print(f"❌ Failed to build: {failed}") | ||
|
|
||
| print(f"\nCreated packages") | ||
| for pkg in sorted(pkg_list.built): | ||
| print(f" - {pkg}") | ||
|
|
||
| if pkg_list.skipped: | ||
| print(f"\n⏭️ Skipped packages") | ||
| print(f" (Base package names from package.json)") | ||
| for pkg in sorted(pkg_list.skipped): | ||
| print(f" - {pkg}") | ||
|
|
||
| print("\n" + "=" * 80) | ||
| print(f"Package type: {config.pkg_type.upper()}") | ||
| print(f"ROCm version: {config.rocm_version}") | ||
| print(f"Output directory: {config.dest_dir}") | ||
| print("=" * 80 + "\n") | ||
|
|
||
|
|
||
| def print_build_summary(config: PackageConfig, pkg_list: PackageList): | ||
| write_build_manifest(config, pkg_list) | ||
| print_build_status(config, pkg_list) |
Uh oh!
There was an error while loading. Please reload this page.