diff --git a/build_tools/packaging/linux/build_package.py b/build_tools/packaging/linux/build_package.py index 4a8dc47621a..0c2d495ec67 100755 --- a/build_tools/packaging/linux/build_package.py +++ b/build_tools/packaging/linux/build_package.py @@ -27,6 +27,7 @@ from datetime import datetime, timezone from email.utils import format_datetime from jinja2 import Environment, FileSystemLoader, Template +from packaging_summary import * from packaging_utils import * from pathlib import Path from runpath_to_rpath import * @@ -673,11 +674,12 @@ def package_with_rpmbuild(spec_file): ######################## Begin Packaging Process################################ -def parse_input_package_list(pkg_name): +def parse_input_package_list(pkg_name, artifact_dir): """Populate the package list from the provided input arguments. Parameters: pkg_name : List of packages to be created + artifact_dir: The path to the Artifactory directory Returns: Package list """ @@ -685,7 +687,7 @@ def parse_input_package_list(pkg_name): pkg_list = [] # If pkg_name is None, include all packages if pkg_name is None: - pkg_list = get_package_list() + pkg_list = get_package_list(artifact_dir) return pkg_list # Proceed if pkg_name is not None @@ -766,7 +768,7 @@ def run(args: argparse.Namespace): # Clean the packaging build directories clean_package_build_dir(config) - pkg_list = parse_input_package_list(args.pkg_names) + pkg_list = parse_input_package_list(args.pkg_names, config.artifacts_dir) # Create deb/rpm packages package_creators = {"deb": create_deb_package, "rpm": create_rpm_package} for pkg_name in pkg_list: @@ -779,6 +781,9 @@ def run(args: argparse.Namespace): creator(pkg_name, config) clean_package_build_dir(config) + # Print build summary + print_build_summary(config, pkg_list) + def main(argv: list[str]): diff --git a/build_tools/packaging/linux/packaging_summary.py b/build_tools/packaging/linux/packaging_summary.py new file mode 100644 index 00000000000..f62893ec6d1 --- /dev/null +++ b/build_tools/packaging/linux/packaging_summary.py @@ -0,0 +1,127 @@ +from datetime import datetime, timezone +from packaging_utils import * + + +def get_skipped_pkglist(pkg_list): + """Determine which packages were skipped during packaging. + + This function reads package.json and builds the full list of packages + that are eligible for packaging (i.e., not marked with 'Disablepackaging'). + It then compares that list with the list of packages that were actually + processed and returns the packages that were skipped. + + Parameters: pkg_list: A list of package names that were successfully packaged. + + Returns: A list of package names that were expected to be packaged but were skipped. + """ + + data = read_package_json_file() + original_pkglist = [ + pkginfo["Package"] for pkginfo in data if not is_packaging_disabled(pkginfo) + ] + skipped_pkglist = [item for item in original_pkglist if item not in pkg_list] + return skipped_pkglist + + +def write_build_manifest(config: PackageConfig, pkg_list): + """Write manifest files listing built and skipped packages. + + Parameters: + config: Configuration object containing package metadata + pkg_list: List of all packages attempted + + Returns: None + """ + print_function_name() + + # Write successful packages manifest + manifest_file = Path(config.dest_dir) / "built_packages.txt" + skipped_packages = get_skipped_pkglist(pkg_list) + try: + with open(manifest_file, "w", encoding="utf-8") as f: + f.write(f"# Built Packages Manifest\n") + f.write(f"# Package Type: {config.pkg_type.upper()}\n") + f.write(f"# ROCm Version: {config.rocm_version}\n") + f.write(f"# Graphics Architecture: {config.gfx_arch}\n") + f.write( + f"# Build Date: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}\n" + ) + f.write(f"# Total Attempted: {len(pkg_list) + len(skipped_packages)}\n") + f.write(f"# Successfully Built: {len(pkg_list)}\n") + f.write(f"# Skipped: {len(skipped_packages)}\n") + f.write(f"\n") + + for pkg in sorted(pkg_list): + f.write(f"{pkg}\n") + + print(f"✅ Built packages manifest written to: {manifest_file}") + except Exception as e: + print(f"⚠️ WARNING: Failed to write built packages manifest: {e}") + + # Write skipped packages manifest + if skipped_packages: + skipped_file = Path(config.dest_dir) / "skipped_packages.txt" + try: + with open(skipped_file, "w", encoding="utf-8") as f: + f.write(f"# Skipped Packages Manifest\n") + f.write(f"# Package Type: {config.pkg_type.upper()}\n") + f.write(f"# ROCm Version: {config.rocm_version}\n") + f.write(f"# Graphics Architecture: {config.gfx_arch}\n") + f.write( + f"# Build Date: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}\n" + ) + f.write(f"# Format: package_name | reason\n") + f.write( + f"# Note: Package names shown are base names from package.json\n" + ) + f.write(f"\n") + f.write(f"# Skipped Packages:\n") + for pkg in sorted(skipped_packages): + f.write(f"{pkg}\n") + + print(f"⚠️ Skipped packages manifest written to: {skipped_file}") + except Exception as e: + print(f"⚠️ WARNING: Failed to write skipped packages manifest: {e}") + + +def print_build_status(config: PackageConfig, pkg_list): + """Print a summary of the build process. + + Parameters: + config: Configuration object containing package metadata + pkg_list : List of all packages attempted + + Returns: None + """ + print("\n" + "=" * 80) + print("BUILD SUMMARY") + print("=" * 80) + + skipped_packages = get_skipped_pkglist(pkg_list) + skipped_count = len(skipped_packages) + built_count = len(pkg_list) + total_packages = built_count + skipped_count + + print(f"\nTotal packages attempted: {total_packages}") + print(f"✅ Successfully built: {built_count}") + print(f" (Showing base package names)") + for pkg in sorted(pkg_list): + print(f" - {pkg}") + + if skipped_packages: + print(f"\n⏭️ Skipped packages ({skipped_count}):") + print(f" (Showing base package names from package.json)") + for pkg in sorted(skipped_packages): + print(f" - {pkg}") + print("\nNote: Skipped packages have been excluded from dependencies") + + print("\n" + "=" * 80) + print(f"Package type: {config.pkg_type.upper()}") + print(f"ROCm version: {config.rocm_version}") + print(f"Output directory: {config.dest_dir}") + print("=" * 80 + "\n") + + +def print_build_summary(config: PackageConfig, pkg_list): + write_build_manifest(config, pkg_list) + print_build_status(config, pkg_list) diff --git a/build_tools/packaging/linux/packaging_utils.py b/build_tools/packaging/linux/packaging_utils.py index 6aeaed64a79..2f4e3624eb9 100644 --- a/build_tools/packaging/linux/packaging_utils.py +++ b/build_tools/packaging/linux/packaging_utils.py @@ -233,19 +233,58 @@ def get_package_info(pkgname): return None -def get_package_list(): - """Read package.json and return package names. +def get_package_list(artifact_dir): + """Read package.json and return a list of package names. - Packages marked as 'Disablepackaging' will be excluded from the list + Packages marked as 'Disablepackaging' are excluded. + If the entire Artifactory directory is missing, the package is excluded + unless it is a metapackage. - Parameters: None + Parameters: + artifact_dir : The path to the Artifactory directory - Returns: Package list + Returns: A list of package names. """ - + pkg_list = [] data = read_package_json_file() - pkg_list = [pkg["Package"] for pkg in data if not is_packaging_disabled(pkg)] + try: + dir_entries = os.listdir(artifact_dir) + except FileNotFoundError: + sys.exit(f"{artifact_dir}: Artifactory directory doesn not exist, Exiting") + + for pkg_info in data: + # Skip disabled packages + if is_packaging_disabled(pkg_info): + continue + + # metapackages don't need artifact lookup + if is_meta_package(pkg_info): + pkg_list.append(pkg_info["Package"]) + continue + + artifactory_list = pkg_info.get("Artifactory", []) + artifact_found = False + + for artifactory in artifactory_list: + artifact_name = artifactory.get("Artifact") + if not artifact_name: + continue + + # Look for directories starting with the artifact name + for entry in dir_entries: + path = Path(artifact_dir) / entry + + if entry.startswith(artifact_name) and path.is_dir(): + artifact_found = True + break + + if artifact_found: + break + + if artifact_found: + pkg_list.append(pkg_info["Package"]) + return pkg_list @@ -378,10 +417,17 @@ def convert_to_versiondependency(dependency_list, config: PackageConfig): local_config = copy.deepcopy(config) local_config.versioned_pkg = True - pkg_list = get_package_list() + pkg_list = get_package_list(config.artifacts_dir) + + filtered_deps = [] + # Remove amdrocm* packages that are NOT in pkg_list + for pkg in dependency_list: + if not (pkg.startswith("amdrocm") and pkg not in pkg_list): + filtered_deps.append(pkg) + updated_depends = [ f"{update_package_name(pkg,local_config)}" if pkg in pkg_list else pkg - for pkg in dependency_list + for pkg in filtered_deps ] depends = ", ".join(updated_depends) return depends @@ -403,7 +449,7 @@ def append_version_suffix(dep_string, config: PackageConfig): """ print_function_name() - pkg_list = get_package_list() + pkg_list = get_package_list(config.artifacts_dir) updated_depends = [] dep_list = [d.strip() for d in dep_string.split(",")] @@ -517,17 +563,24 @@ def filter_components_fromartifactory(pkg_name, artifacts_dir, gfx_arch): / f"{artifact_prefix}_{component}_{artifact_suffix}" ) filename = source_dir / "artifact_manifest.txt" - with open(filename, "r", encoding="utf-8") as file: - for line in file: - - match_found = ( - isinstance(artifact_subdir, str) - and (artifact_subdir.lower() + "/") in line.lower() - ) - - if match_found and line.strip(): - print("Matching line:", line.strip()) - source_path = source_dir / line.strip() - sourcedir_list.append(source_path) + if not filename.exists(): + print(f"{pkg_name} : Missing {filename}") + continue + try: + with filename.open("r", encoding="utf-8") as file: + for line in file: + + match_found = ( + isinstance(artifact_subdir, str) + and (artifact_subdir.lower() + "/") in line.lower() + ) + + if match_found and line.strip(): + print("Matching line:", line.strip()) + source_path = source_dir / line.strip() + sourcedir_list.append(source_path) + except OSError as e: + print(f"Could not read manifest {filename}: {e}") + continue return sourcedir_list