From 593f9dcd847a39cc5ead4734b06225efaa61bfe6 Mon Sep 17 00:00:00 2001 From: Nirmal Unnikrishnan Date: Mon, 19 Jan 2026 17:46:39 -0600 Subject: [PATCH] ROCM-1331 : Fixing warning error while setting up repo in debian. The Debian release file didnt had hash values like md4 sha256. The new change adds them to the release file. Also the big regenerate_repo_metadata_from_s3 fn has been split into deb/rpm smaller functions for better maintanance --- .../packaging/linux/upload_package_repo.py | 647 ++++++++++-------- 1 file changed, 369 insertions(+), 278 deletions(-) diff --git a/build_tools/packaging/linux/upload_package_repo.py b/build_tools/packaging/linux/upload_package_repo.py index c064e1b5040..1aeebd73474 100644 --- a/build_tools/packaging/linux/upload_package_repo.py +++ b/build_tools/packaging/linux/upload_package_repo.py @@ -83,327 +83,418 @@ def generate_indexes_recursive(root): generate_index_html(d) -def regenerate_repo_metadata_from_s3( - s3, bucket, prefix, pkg_type, uploaded_packages, job_type="nightly" -): - """Regenerate repository metadata efficiently using merge approach. +def regenerate_rpm_metadata_from_s3(s3, bucket, prefix, uploaded_packages): + """Regenerate RPM repository metadata using merge approach. - This uses mergerepo_c (RPM) or merges Packages files (DEB) to efficiently - update metadata without re-downloading all packages from S3. + Downloads existing repodata from S3, generates metadata for new packages, + merges them using mergerepo_c, and uploads the result back to S3. Args: s3: boto3 S3 client bucket: S3 bucket name prefix: S3 prefix (e.g., 'rpm/20251222-12345') - pkg_type: Package type ('rpm' or 'deb') - uploaded_packages: List of actually uploaded package file paths (avoids duplicates from deduplication) - job_type: Job type for Release file metadata (default: 'nightly') + uploaded_packages: List of actually uploaded .rpm file paths """ import tempfile - print(f"Updating {pkg_type.upper()} repository metadata (merge mode)...") + print(f"Updating RPM repository metadata (merge mode)...") # Create temporary directory for metadata operations with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) - if pkg_type == "rpm": - # Efficient approach: Download existing repodata and merge with new packages - old_repo_dir = temp_path / "old_repo" - new_repo_dir = temp_path / "new_repo" - merged_repo_dir = temp_path / "merged_repo" + # Efficient approach: Download existing repodata and merge with new packages + old_repo_dir = temp_path / "old_repo" + new_repo_dir = temp_path / "new_repo" + merged_repo_dir = temp_path / "merged_repo" - old_repo_dir.mkdir(parents=True, exist_ok=True) - new_repo_dir.mkdir(parents=True, exist_ok=True) - merged_repo_dir.mkdir(parents=True, exist_ok=True) + old_repo_dir.mkdir(parents=True, exist_ok=True) + new_repo_dir.mkdir(parents=True, exist_ok=True) + merged_repo_dir.mkdir(parents=True, exist_ok=True) - # Step 1: Download existing repodata from S3 (small files) - old_repodata_dir = old_repo_dir / "repodata" - old_repodata_dir.mkdir(parents=True, exist_ok=True) + # Step 1: Download existing repodata from S3 (small files) + old_repodata_dir = old_repo_dir / "repodata" + old_repodata_dir.mkdir(parents=True, exist_ok=True) - print( - f"Downloading existing repository metadata from S3: s3://{bucket}/{prefix}/x86_64/repodata/" - ) - repodata_files = [] - try: - paginator = s3.get_paginator("list_objects_v2") - for page in paginator.paginate( - Bucket=bucket, Prefix=f"{prefix}/x86_64/repodata/" - ): - if "Contents" not in page: - continue - for obj in page["Contents"]: - key = obj["Key"] - filename = Path(key).name - local_file = old_repodata_dir / filename - s3.download_file(bucket, key, str(local_file)) - repodata_files.append(filename) - print(f" Downloaded: {filename}") - if repodata_files: - print( - f"✅ Found {len(repodata_files)} existing metadata files to merge" - ) - else: - print("No existing metadata files found") - except Exception as e: - print(f"⚠️ No existing repodata found (new repo?): {e}") - - # Step 2: Generate repodata for NEW packages only (actually uploaded ones) - rpm_packages = [p for p in uploaded_packages if p.endswith(".rpm")] - if rpm_packages: + print( + f"Downloading existing repository metadata from S3: s3://{bucket}/{prefix}/x86_64/repodata/" + ) + repodata_files = [] + try: + paginator = s3.get_paginator("list_objects_v2") + for page in paginator.paginate( + Bucket=bucket, Prefix=f"{prefix}/x86_64/repodata/" + ): + if "Contents" not in page: + continue + for obj in page["Contents"]: + key = obj["Key"] + filename = Path(key).name + local_file = old_repodata_dir / filename + s3.download_file(bucket, key, str(local_file)) + repodata_files.append(filename) + print(f" Downloaded: {filename}") + if repodata_files: print( - f"Generating metadata for {len(rpm_packages)} uploaded RPM packages..." - ) - # Copy uploaded RPMs to temp dir - new_arch_dir = new_repo_dir / "x86_64" - new_arch_dir.mkdir(parents=True, exist_ok=True) - for rpm_file in rpm_packages: - shutil.copy2(rpm_file, new_arch_dir / Path(rpm_file).name) - - # Generate repodata for new packages with clean paths (no baseurl) - run_command( - "createrepo_c --no-database --simple-md-filenames .", - cwd=str(new_arch_dir), + f"✅ Found {len(repodata_files)} existing metadata files to merge" ) - print("✅ Generated metadata for uploaded packages") else: - print("No new RPM packages uploaded (all deduplicated)") - # Still need to ensure old metadata is preserved! - if repodata_files: - print("Preserving existing repodata...") - # Just re-upload the existing repodata we downloaded - for metadata_file in old_repodata_dir.iterdir(): - if metadata_file.is_file(): - s3_key = f"{prefix}/x86_64/repodata/{metadata_file.name}" - s3.upload_file(str(metadata_file), bucket, s3_key) - print(f" Uploaded: {metadata_file.name}") - print("✅ RPM repository metadata preserved") - return - - # Step 3: Merge repositories using mergerepo_c (no need to download all RPMs!) - merged_arch_dir = merged_repo_dir / "x86_64" - merged_arch_dir.mkdir(parents=True, exist_ok=True) - - if repodata_files: # If we have existing metadata - print("Merging old and new repository metadata...") - # mergerepo_c merges repodata without needing actual RPM files! - # Use --no-database, --simple-md-filenames, and --omit-baseurl to ensure clean paths - run_command( - f"mergerepo_c --no-database --simple-md-filenames --omit-baseurl " - f'--repo "{old_repo_dir}" --repo "{new_repo_dir / "x86_64"}" ' - f'--outputdir "{merged_arch_dir}"', - cwd=str(temp_path), - ) - print("✅ Merged repository metadata") - else: # First upload, no existing metadata - print("First upload - using new repository metadata") - shutil.copytree( - new_repo_dir / "x86_64" / "repodata", merged_arch_dir / "repodata" - ) + print("No existing metadata files found") + except Exception as e: + print(f"⚠️ No existing repodata found (new repo?): {e}") - # Step 4: Upload merged repodata to S3 - merged_repodata = merged_arch_dir / "repodata" - if merged_repodata.exists(): - print("Uploading merged repository metadata to S3...") - uploaded_metadata = [] - for metadata_file in merged_repodata.iterdir(): + # Step 2: Generate repodata for NEW packages only (actually uploaded ones) + rpm_packages = [p for p in uploaded_packages if p.endswith(".rpm")] + if rpm_packages: + print( + f"Generating metadata for {len(rpm_packages)} uploaded RPM packages..." + ) + # Copy uploaded RPMs to temp dir + new_arch_dir = new_repo_dir / "x86_64" + new_arch_dir.mkdir(parents=True, exist_ok=True) + for rpm_file in rpm_packages: + shutil.copy2(rpm_file, new_arch_dir / Path(rpm_file).name) + + # Generate repodata for new packages with clean paths (no baseurl) + run_command( + "createrepo_c --no-database --simple-md-filenames .", + cwd=str(new_arch_dir), + ) + print("✅ Generated metadata for uploaded packages") + else: + print("No new RPM packages uploaded (all deduplicated)") + # Still need to ensure old metadata is preserved! + if repodata_files: + print("Preserving existing repodata...") + # Just re-upload the existing repodata we downloaded + for metadata_file in old_repodata_dir.iterdir(): if metadata_file.is_file(): s3_key = f"{prefix}/x86_64/repodata/{metadata_file.name}" s3.upload_file(str(metadata_file), bucket, s3_key) - uploaded_metadata.append(metadata_file.name) print(f" Uploaded: {metadata_file.name}") - print( - f"✅ RPM repository metadata updated: {len(uploaded_metadata)} files" - ) + print("✅ RPM repository metadata preserved") + return + + # Step 3: Merge repositories using mergerepo_c (no need to download all RPMs!) + merged_arch_dir = merged_repo_dir / "x86_64" + merged_arch_dir.mkdir(parents=True, exist_ok=True) + + if repodata_files: # If we have existing metadata + print("Merging old and new repository metadata...") + # mergerepo_c merges repodata without needing actual RPM files! + # Use --no-database, --simple-md-filenames, and --omit-baseurl to ensure clean paths + run_command( + f"mergerepo_c --no-database --simple-md-filenames --omit-baseurl " + f'--repo "{old_repo_dir}" --repo "{new_repo_dir / "x86_64"}" ' + f'--outputdir "{merged_arch_dir}"', + cwd=str(temp_path), + ) + print("✅ Merged repository metadata") + else: # First upload, no existing metadata + print("First upload - using new repository metadata") + shutil.copytree( + new_repo_dir / "x86_64" / "repodata", merged_arch_dir / "repodata" + ) - elif pkg_type == "deb": - # Efficient approach: Merge existing Packages file with new packages - dists_dir = temp_path / "dists" / "stable" / "main" / "binary-amd64" - dists_dir.mkdir(parents=True, exist_ok=True) + # Step 4: Upload merged repodata to S3 + merged_repodata = merged_arch_dir / "repodata" + if merged_repodata.exists(): + print("Uploading merged repository metadata to S3...") + uploaded_metadata = [] + for metadata_file in merged_repodata.iterdir(): + if metadata_file.is_file(): + s3_key = f"{prefix}/x86_64/repodata/{metadata_file.name}" + s3.upload_file(str(metadata_file), bucket, s3_key) + uploaded_metadata.append(metadata_file.name) + print(f" Uploaded: {metadata_file.name}") + print(f"✅ RPM repository metadata updated: {len(uploaded_metadata)} files") - pool_dir = temp_path / "pool" / "main" - pool_dir.mkdir(parents=True, exist_ok=True) - # Step 1: Download existing Packages file from S3 (small file) - existing_packages = dists_dir / "Packages.old" - packages_s3_key = f"{prefix}/dists/stable/main/binary-amd64/Packages" - try: - print( - f"Downloading existing Packages file from S3: s3://{bucket}/{packages_s3_key}" - ) - s3.download_file(bucket, packages_s3_key, str(existing_packages)) - # Count existing packages - with open(existing_packages, "r") as f: - content = f.read() - pkg_count = content.count("\nPackage: ") - print(f"✅ Downloaded existing Packages file ({pkg_count} packages)") - except Exception as e: - print(f"⚠️ No existing Packages file found (new repo?): {e}") - existing_packages = None - - # Step 2: Generate Packages entries for NEW packages only (actually uploaded ones) - deb_packages = [p for p in uploaded_packages if p.endswith(".deb")] - if deb_packages: - print( - f"Generating Packages entries for {len(deb_packages)} uploaded DEB packages..." - ) - # Copy uploaded DEBs to temp dir - for deb_file in deb_packages: - shutil.copy2(deb_file, pool_dir / Path(deb_file).name) - - # Generate Packages entries for uploaded packages - new_packages = dists_dir / "Packages.new" - run_command( - f'dpkg-scanpackages -m pool/main /dev/null > "{new_packages}"', - cwd=str(temp_path), - ) - print("✅ Generated Packages entries for uploaded packages") - else: - print("No new DEB packages uploaded (all deduplicated)") - # Still need to ensure old metadata is preserved! - if existing_packages and existing_packages.exists(): - import datetime - - print("Preserving existing Packages file...") - shutil.copy2(existing_packages, dists_dir / "Packages") - run_command("gzip -9c Packages > Packages.gz", cwd=str(dists_dir)) - - # Generate Release file - release_dir = temp_path / "dists" / "stable" - release_dir.mkdir(parents=True, exist_ok=True) - release_file = release_dir / "Release" - - with open(release_file, "w") as f: - f.write( - f"""Origin: AMD ROCm +def generate_release_file_with_checksums(release_file, job_type, dists_dir): + """Generate a Debian Release file with MD5Sum, SHA1, and SHA256 checksums. + + Args: + release_file: Path to the Release file to create + job_type: Job type for metadata (nightly/dev/release) + dists_dir: Directory containing Packages files (main/binary-amd64/) + """ + import hashlib + import datetime + + # Files to hash (relative paths from dists/stable/) + files_to_hash = [ + (dists_dir / "Packages", "main/binary-amd64/Packages"), + (dists_dir / "Packages.gz", "main/binary-amd64/Packages.gz"), + ] + + # Calculate all hashes + md5_entries = [] + sha1_entries = [] + sha256_entries = [] + + for file_path, rel_path in files_to_hash: + if not file_path.exists(): + continue + + # Get file size + file_size = file_path.stat().st_size + + # Calculate hashes + md5_hash = hashlib.md5() + sha1_hash = hashlib.sha1() + sha256_hash = hashlib.sha256() + + with open(file_path, "rb") as f: + while True: + data = f.read(65536) # Read in 64KB chunks + if not data: + break + md5_hash.update(data) + sha1_hash.update(data) + sha256_hash.update(data) + + # Store entries (space-aligned format) + md5_entries.append(f" {md5_hash.hexdigest()} {file_size:16d} {rel_path}") + sha1_entries.append(f" {sha1_hash.hexdigest()} {file_size:16d} {rel_path}") + sha256_entries.append(f" {sha256_hash.hexdigest()} {file_size:16d} {rel_path}") + + # Write Release file + with open(release_file, "w") as f: + # Header fields + f.write( + f"""Origin: AMD ROCm Label: ROCm {job_type} Packages Suite: stable Codename: stable Architectures: amd64 Components: main +Description: ROCm APT Repository Date: {datetime.datetime.utcnow():%a, %d %b %Y %H:%M:%S UTC} """ - ) + ) + + # MD5Sum section + if md5_entries: + f.write("MD5Sum:\n") + f.write("\n".join(md5_entries)) + f.write("\n") + + # SHA1 section + if sha1_entries: + f.write("SHA1:\n") + f.write("\n".join(sha1_entries)) + f.write("\n") + + # SHA256 section + if sha256_entries: + f.write("SHA256:\n") + f.write("\n".join(sha256_entries)) + f.write("\n") + + print(f"✅ Release file generated with checksums: MD5, SHA1, SHA256") + + +def upload_deb_metadata_to_s3(s3, bucket, prefix, dists_dir, release_file): + """Helper function to upload Debian metadata files to S3. + + Args: + s3: boto3 S3 client + bucket: S3 bucket name + prefix: S3 prefix + dists_dir: Directory containing Packages files + release_file: Path to Release file + """ + packages_file = dists_dir / "Packages" + packages_gz = dists_dir / "Packages.gz" + + uploaded_count = 0 + if packages_file.exists(): + s3_key = f"{prefix}/dists/stable/main/binary-amd64/Packages" + s3.upload_file(str(packages_file), bucket, s3_key) + print(f" Uploaded: Packages") + uploaded_count += 1 + + if packages_gz.exists(): + s3_key = f"{prefix}/dists/stable/main/binary-amd64/Packages.gz" + s3.upload_file(str(packages_gz), bucket, s3_key) + print(f" Uploaded: Packages.gz") + uploaded_count += 1 - packages_file = dists_dir / "Packages" - packages_gz = dists_dir / "Packages.gz" + if release_file.exists(): + s3_key = f"{prefix}/dists/stable/Release" + s3.upload_file(str(release_file), bucket, s3_key) + print(f" Uploaded: Release") + uploaded_count += 1 - if packages_file.exists(): - s3_key = f"{prefix}/dists/stable/main/binary-amd64/Packages" - s3.upload_file(str(packages_file), bucket, s3_key) - print(f" Uploaded: Packages") + print(f"✅ DEB repository metadata updated: {uploaded_count} files") - if packages_gz.exists(): - s3_key = f"{prefix}/dists/stable/main/binary-amd64/Packages.gz" - s3.upload_file(str(packages_gz), bucket, s3_key) - print(f" Uploaded: Packages.gz") - if release_file.exists(): - s3_key = f"{prefix}/dists/stable/Release" - s3.upload_file(str(release_file), bucket, s3_key) - print(f" Uploaded: Release") - return +def regenerate_deb_metadata_from_s3( + s3, bucket, prefix, uploaded_packages, job_type="nightly" +): + """Regenerate Debian repository metadata efficiently with proper checksums. + + Uses dpkg-scanpackages for efficiency (no package downloads), but generates + proper Release file with MD5Sum, SHA1, and SHA256 checksums. + + Args: + s3: boto3 S3 client + bucket: S3 bucket name + prefix: S3 prefix (e.g., 'deb/20251222-12345') + uploaded_packages: List of actually uploaded .deb file paths + job_type: Job type for Release file metadata (default: 'nightly') + """ + import tempfile + + print(f"Updating DEB repository metadata (merge mode with checksums)...") + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Setup directories + dists_dir = temp_path / "dists" / "stable" / "main" / "binary-amd64" + dists_dir.mkdir(parents=True, exist_ok=True) - # Step 3: Merge old and new Packages files (with deduplication by filename) - merged_packages = dists_dir / "Packages" + pool_dir = temp_path / "pool" / "main" + pool_dir.mkdir(parents=True, exist_ok=True) + # Step 1: Download existing Packages file from S3 (SMALL FILE - efficient!) + existing_packages = dists_dir / "Packages.old" + packages_s3_key = f"{prefix}/dists/stable/main/binary-amd64/Packages" + try: + print( + f"Downloading existing Packages file from S3: s3://{bucket}/{packages_s3_key}" + ) + s3.download_file(bucket, packages_s3_key, str(existing_packages)) + with open(existing_packages, "r") as f: + content = f.read() + pkg_count = content.count("\nPackage: ") + print(f"✅ Downloaded existing Packages file ({pkg_count} packages)") + except Exception as e: + print(f"⚠️ No existing Packages file found (new repo?): {e}") + existing_packages = None + + # Step 2: Generate Packages entries for NEW packages only + deb_packages = [p for p in uploaded_packages if p.endswith(".deb")] + if deb_packages: + print( + f"Generating Packages entries for {len(deb_packages)} uploaded DEB packages..." + ) + # Copy uploaded DEBs to temp dir + for deb_file in deb_packages: + shutil.copy2(deb_file, pool_dir / Path(deb_file).name) + + # Generate Packages entries for uploaded packages + new_packages = dists_dir / "Packages.new" + run_command( + f'dpkg-scanpackages -m pool/main /dev/null > "{new_packages}"', + cwd=str(temp_path), + ) + print("✅ Generated Packages entries for uploaded packages") + else: + print("No new DEB packages uploaded (all deduplicated)") if existing_packages and existing_packages.exists(): - print("Merging old and new Packages files...") - - def parse_packages_file(filepath): - """Parse Packages file into dict keyed by Filename""" - packages = {} - with open(filepath, "r") as f: - current_entry = [] - current_filename = None - - for line in f: - if line.strip() == "": # Blank line = end of entry - if current_entry and current_filename: - packages[current_filename] = ( - "\n".join(current_entry) + "\n" - ) - current_entry = [] - current_filename = None - else: - current_entry.append(line.rstrip()) - if line.startswith("Filename:"): - current_filename = line.split(":", 1)[1].strip() - - # Handle last entry (no trailing blank line) - if current_entry and current_filename: - packages[current_filename] = "\n".join(current_entry) + "\n" - - return packages - - # Parse both files - old_packages = parse_packages_file(existing_packages) - new_packages_dict = parse_packages_file(new_packages) - - print(f" Old metadata: {len(old_packages)} packages") - print(f" New metadata: {len(new_packages_dict)} packages") - - # Merge: new packages override old ones with same filename - merged = old_packages.copy() - merged.update(new_packages_dict) # New overwrites old - - # Write merged Packages file - with open(merged_packages, "w") as outfile: - for filename in sorted(merged.keys()): - outfile.write(merged[filename]) - outfile.write("\n") # Blank line separator - - print(f"✅ Merged Packages files: {len(merged)} total packages") - else: # First upload, no existing Packages file - print("First upload - using new Packages file") - shutil.copy2(new_packages, merged_packages) - - # Compress Packages file - run_command("gzip -9c Packages > Packages.gz", cwd=str(dists_dir)) - - # Step 4: Generate Release file - import datetime - - release_dir = temp_path / "dists" / "stable" - release_dir.mkdir(parents=True, exist_ok=True) - release_file = release_dir / "Release" - - with open(release_file, "w") as f: - f.write( - f"""Origin: AMD ROCm -Label: ROCm {job_type} Packages -Suite: stable -Codename: stable -Architectures: amd64 -Components: main -Date: {datetime.datetime.utcnow():%a, %d %b %Y %H:%M:%S UTC} -""" - ) + print("Preserving existing metadata...") + shutil.copy2(existing_packages, dists_dir / "Packages") + run_command("gzip -9c Packages > Packages.gz", cwd=str(dists_dir)) + + # Generate Release file with checksums + release_dir = temp_path / "dists" / "stable" + release_dir.mkdir(parents=True, exist_ok=True) + release_file = release_dir / "Release" + + generate_release_file_with_checksums(release_file, job_type, dists_dir) + + # Upload preserved files + upload_deb_metadata_to_s3(s3, bucket, prefix, dists_dir, release_file) + return + + # Step 3: Merge old and new Packages files + merged_packages = dists_dir / "Packages" + + if existing_packages and existing_packages.exists(): + print("Merging old and new Packages files...") + + def parse_packages_file(filepath): + """Parse Packages file into dict keyed by Filename""" + packages = {} + with open(filepath, "r") as f: + current_entry = [] + current_filename = None + + for line in f: + if line.strip() == "": + if current_entry and current_filename: + packages[current_filename] = ( + "\n".join(current_entry) + "\n" + ) + current_entry = [] + current_filename = None + else: + current_entry.append(line.rstrip()) + if line.startswith("Filename:"): + current_filename = line.split(":", 1)[1].strip() + + if current_entry and current_filename: + packages[current_filename] = "\n".join(current_entry) + "\n" + + return packages + + old_packages = parse_packages_file(existing_packages) + new_packages_dict = parse_packages_file(new_packages) + + print(f" Old metadata: {len(old_packages)} packages") + print(f" New metadata: {len(new_packages_dict)} packages") + + merged = old_packages.copy() + merged.update(new_packages_dict) + + with open(merged_packages, "w") as outfile: + for filename in sorted(merged.keys()): + outfile.write(merged[filename]) + outfile.write("\n") + + print(f"✅ Merged Packages files: {len(merged)} total packages") + else: + print("First upload - using new Packages file") + shutil.copy2(new_packages, merged_packages) + + # Compress Packages file + run_command("gzip -9c Packages > Packages.gz", cwd=str(dists_dir)) + + # Step 4: Generate Release file with checksums + release_dir = temp_path / "dists" / "stable" + release_dir.mkdir(parents=True, exist_ok=True) + release_file = release_dir / "Release" - # Step 5: Upload merged Packages files and Release to S3 - packages_file = dists_dir / "Packages" - packages_gz = dists_dir / "Packages.gz" - - uploaded_count = 0 - if packages_file.exists(): - s3_key = f"{prefix}/dists/stable/main/binary-amd64/Packages" - s3.upload_file(str(packages_file), bucket, s3_key) - print(f" Uploaded: Packages to s3://{bucket}/{s3_key}") - uploaded_count += 1 - - if packages_gz.exists(): - s3_key = f"{prefix}/dists/stable/main/binary-amd64/Packages.gz" - s3.upload_file(str(packages_gz), bucket, s3_key) - print(f" Uploaded: Packages.gz to s3://{bucket}/{s3_key}") - uploaded_count += 1 - - if release_file.exists(): - s3_key = f"{prefix}/dists/stable/Release" - s3.upload_file(str(release_file), bucket, s3_key) - print(f" Uploaded: Release to s3://{bucket}/{s3_key}") - uploaded_count += 1 - - print(f"✅ DEB repository metadata updated: {uploaded_count} files") + generate_release_file_with_checksums(release_file, job_type, dists_dir) + + # Step 5: Upload merged files to S3 + upload_deb_metadata_to_s3(s3, bucket, prefix, dists_dir, release_file) + + +def regenerate_repo_metadata_from_s3( + s3, bucket, prefix, pkg_type, uploaded_packages, job_type="nightly" +): + """Regenerate repository metadata efficiently using merge approach. + + This uses mergerepo_c (RPM) or merges Packages files (DEB) to efficiently + update metadata without re-downloading all packages from S3. + + Args: + s3: boto3 S3 client + bucket: S3 bucket name + prefix: S3 prefix (e.g., 'rpm/20251222-12345') + pkg_type: Package type ('rpm' or 'deb') + uploaded_packages: List of actually uploaded package file paths (avoids duplicates from deduplication) + job_type: Job type for Release file metadata (default: 'nightly') + """ + if pkg_type == "rpm": + regenerate_rpm_metadata_from_s3(s3, bucket, prefix, uploaded_packages) + elif pkg_type == "deb": + regenerate_deb_metadata_from_s3(s3, bucket, prefix, uploaded_packages, job_type) + else: + raise ValueError(f"Unsupported package type: {pkg_type}") def generate_top_index_from_s3(s3, bucket, prefix):