diff --git a/.github/workflows/multi_arch_build_native_linux_packages.yml b/.github/workflows/multi_arch_build_native_linux_packages.yml index de5aa469375..83870e273b1 100644 --- a/.github/workflows/multi_arch_build_native_linux_packages.yml +++ b/.github/workflows/multi_arch_build_native_linux_packages.yml @@ -61,6 +61,11 @@ on: built_packages.txt exist under the packages directory. type: boolean default: true + enable_repo_setup_package: + description: "Build, verify and upload the amd-repo-setup package" + type: boolean + required: false + default: false outputs: package_repository_url: description: "Public repository URL for package installation" @@ -72,6 +77,75 @@ permissions: run-name: Build ${{ inputs.native_package_type }} packages (${{ inputs.rocm_version }}, ${{ inputs.dist_amdgpu_families }}${{ inputs.release_type && format(', {0}', inputs.release_type) || '' }}) jobs: + # Resolve the parameters used to build and upload the amd-repo-setup package: + # the OS-profile build matrix for this package type, each profile's container + # image, and the repo.amd.com stream the build line configures, with its + # tier, packages base URL, signing-key URL and build sub-folder. + # + # The build line and the stream are different vocabularies: prerelease + # configures rc. release_type stays the workflow input because it also selects + # the artifact bucket; the stream is derived from it. + resolve_repo_setup_params: + name: Resolve amd-repo-setup parameters + # nightly and prerelease are the lines that upload the package. release + # configures a stream too, but has no artifacts bucket to upload into. + # Sanitizer variants skip: the resolved repository URL serves neither ASAN + # packages (promoted under packages-asan) nor TSAN ones (not promoted). + if: inputs.enable_repo_setup_package && contains(fromJSON('["nightly", "prerelease"]'), inputs.release_type) && !contains(inputs.build_variant, 'asan') && !contains(inputs.build_variant, 'tsan') + runs-on: ${{ github.repository_owner == 'ROCm' && inputs.build_runs_on || 'ubuntu-26.04' }} + container: + image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:15241d0e87bf81fe0986006ab832a36ea53a2cda0f9a31b4a5dc7826bc2396b6 + # Resolves parameters only; it needs no cloud credentials. + permissions: + contents: read + env: + NATIVE_PACKAGE_TYPE: ${{ inputs.native_package_type }} + RELEASE_TYPE: ${{ inputs.release_type }} + ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" + outputs: + os_profiles: ${{ steps.derive.outputs.os_profiles }} + images: ${{ steps.derive.outputs.images }} + stream: ${{ steps.derive.outputs.stream }} + tier: ${{ steps.derive.outputs.tier }} + repo_base_url: ${{ steps.derive.outputs.repo_base_url }} + gpg_key_url: ${{ steps.derive.outputs.gpg_key_url }} + repo_sub_folder: ${{ steps.derive.outputs.repo_sub_folder }} + steps: + - name: "Quartz - started - build amd-repo-setup" + continue-on-error: true + if: ${{ vars.NOTIFY_QUARTZ_ENABLED == 'true' }} + uses: ROCm/Quartz/.github/actions/notify_quartz@f386a9756620938616af0b4d5d04b24ae6e0353f # notify_quartz/v1.1.1 + with: + run_phase: started + reporting_workflow: multi_arch_build_native_linux_packages.yml + workflow_inputs: ${{ toJSON(inputs) }} + gh_app_client_id: ${{ secrets.GH_APP_HAULY_CID }} + gh_app_private_key: ${{ secrets.GH_APP_HAULY_PRIVATE_KEY }} + + - name: Checking out repository + timeout-minutes: 15 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref || '' }} + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + + - name: Install Python requirements + run: pip install jinja2 + + - name: Derive amd-repo-setup parameters + id: derive + run: | + python ./build_tools/packaging/linux/get_url_repo_params.py get-repo-setup-params \ + --pkg-type "${NATIVE_PACKAGE_TYPE}" \ + --release-type "${RELEASE_TYPE}" \ + --run-id "${ARTIFACT_RUN_ID}" + build_native_packages: permissions: contents: read @@ -218,3 +292,133 @@ jobs: workflow_step_outputs: ${{ toJSON(steps) }} gh_app_client_id: ${{ secrets.GH_APP_HAULY_CID }} gh_app_private_key: ${{ secrets.GH_APP_HAULY_PRIVATE_KEY }} + + # Build the amd-repo-setup package for each OS profile of this package type, + # check its contents and upload it to this run's artifacts bucket, under + # amd-repo-setup//. + stage_repo_setup_package: + name: Stage amd-repo-setup (${{ matrix.os_profile }}) + needs: [resolve_repo_setup_params] + if: inputs.enable_repo_setup_package && needs.resolve_repo_setup_params.outputs.repo_base_url != '' + strategy: + fail-fast: false + matrix: + # resolve_repo_setup_params does not run on the lines that do not upload, and a + # skipped job's outputs are empty strings, which fromJSON cannot parse. + # Fall back to an empty matrix so this job skips cleanly. + os_profile: ${{ fromJSON(needs.resolve_repo_setup_params.outputs.os_profiles || '[]') }} + runs-on: ubuntu-26.04 + permissions: + contents: read + id-token: write # Authenticate to AWS to upload the amd-repo-setup package. + container: + # get_url_repo_params.py manages image references for each OS profile. + # zizmor cannot inspect the dynamically selected image. + image: ${{ fromJSON(needs.resolve_repo_setup_params.outputs.images || '{}')[matrix.os_profile] }} # zizmor: ignore[unpinned-images] + env: + ROCM_VERSION: ${{ inputs.rocm_version }} + NATIVE_PACKAGE_TYPE: ${{ inputs.native_package_type }} + RELEASE_TYPE: ${{ inputs.release_type }} + ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}" + OS_PROFILE: ${{ matrix.os_profile }} + STREAM: ${{ needs.resolve_repo_setup_params.outputs.stream }} + TIER: ${{ needs.resolve_repo_setup_params.outputs.tier }} + REPO_BASE_URL: ${{ needs.resolve_repo_setup_params.outputs.repo_base_url }} + GPG_KEY_URL: ${{ needs.resolve_repo_setup_params.outputs.gpg_key_url }} + REPO_SUB_FOLDER: ${{ needs.resolve_repo_setup_params.outputs.repo_sub_folder }} + steps: + - name: Checkout + timeout-minutes: 15 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.ref || '' }} + + - name: Install system prerequisites + run: | + bash build_tools/packaging/linux/setup_repo_build_deps.sh \ + --os-profile "${OS_PROFILE}" + + - name: Setup Python + run: | + bash build_tools/packaging/linux/setup_python_cmd.sh \ + --os-profile "${OS_PROFILE}" \ + --install-runtime \ + --output-format github + + - name: Install Python dependencies + run: | + $PYTHON_CMD -m venv /tmp/repo-venv + /tmp/repo-venv/bin/python -m pip install --upgrade pip + /tmp/repo-venv/bin/python -m pip install jinja2 boto3 + echo "PYTHON_CMD=/tmp/repo-venv/bin/python" >> "$GITHUB_ENV" + # configure_aws_artifacts_credentials runs bare `python`, which these + # per-distro images do not provide. A venv always does, so putting its + # bin/ on PATH lets the credentials action run here and the package + # upload from the job that built it. + echo "/tmp/repo-venv/bin" >> "$GITHUB_PATH" + + - name: Build amd-repo-setup + run: | + # The build sub-folder is empty except on a per-build stream, which is + # the only kind the builder accepts one for. The key URL is empty on an + # unsigned stream, where the builder needs none. + $PYTHON_CMD build_tools/packaging/linux/build_repo_setup_package.py \ + --os-profile "${OS_PROFILE}" \ + --stream "${STREAM}" \ + --repo-base-url "${REPO_BASE_URL}" \ + --gpg-key-url "${GPG_KEY_URL}" \ + --repo-sub-folder "${REPO_SUB_FOLDER}" \ + --rocm-version "${ROCM_VERSION}" \ + --dest-dir repo-setup-package-out \ + --verify-repo-url + + # Check the package before it is uploaded: the repository file for this + # stream (and, on a signed stream, the key) at the expected paths, nothing + # it must not ship, and everything owned by root. + - name: Inspect amd-repo-setup + run: | + $PYTHON_CMD build_tools/packaging/linux/inspect_repo_setup_package.py \ + --package-dir repo-setup-package-out \ + --os-profile "${OS_PROFILE}" \ + --stream "${STREAM}" + + # Configured after the build so the key fetch, the repository check and + # the package build run without artifacts-bucket credentials. + - name: Configure AWS credentials for artifact uploads + uses: ./.github/actions/configure_aws_artifacts_credentials + with: + release_type: ${{ inputs.release_type }} + + - name: Upload amd-repo-setup + id: upload-repo-setup-package + run: | + $PYTHON_CMD build_tools/packaging/linux/upload_repo_setup_package.py \ + --dest-dir repo-setup-package-out \ + --tier "${TIER}" \ + --run-id "${ARTIFACT_RUN_ID}" \ + --os-profile "${OS_PROFILE}" \ + --pkg-type "${NATIVE_PACKAGE_TYPE}" + + # Report the amd-repo-setup jobs' result to Quartz. resolve_repo_setup_params reports + # their start, so this runs only when that job ran. + notify_quartz_completed: + name: "Quartz - completed - build amd-repo-setup" + if: ${{ always() && vars.NOTIFY_QUARTZ_ENABLED == 'true' && needs.resolve_repo_setup_params.result != 'skipped' }} + needs: [resolve_repo_setup_params, stage_repo_setup_package] + runs-on: ubuntu-26.04 + timeout-minutes: 10 + continue-on-error: true + permissions: + contents: read + steps: + - name: "Quartz - completed - build amd-repo-setup" + uses: ROCm/Quartz/.github/actions/notify_quartz@f386a9756620938616af0b4d5d04b24ae6e0353f # notify_quartz/v1.1.1 + with: + run_phase: completed + reporting_workflow: multi_arch_build_native_linux_packages.yml + workflow_inputs: ${{ toJSON(inputs) }} + workflow_captured_outputs: ${{ toJSON(needs) }} + gh_app_client_id: ${{ secrets.GH_APP_HAULY_CID }} + gh_app_private_key: ${{ secrets.GH_APP_HAULY_PRIVATE_KEY }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cdd8b42ade2..307c2d21449 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,7 +42,7 @@ repos: rev: v1.5.5 hooks: - id: forbid-tabs - exclude: ".gitmodules|patches/.*|docs/development/git_chores.md|build_tools/packaging/linux/template/debian_rules.j2|third-party/sysdeps/linux/libdrm/patch_source.sh" + exclude: ".gitmodules|patches/.*|docs/development/git_chores.md|build_tools/packaging/linux/template/debian_rules.j2|build_tools/packaging/linux/template/repo/deb/rules.j2|third-party/sysdeps/linux/libdrm/patch_source.sh" - repo: https://github.com/rhysd/actionlint rev: v1.7.10 diff --git a/build_tools/.gitignore b/build_tools/.gitignore deleted file mode 100644 index f606d5e0b04..00000000000 --- a/build_tools/.gitignore +++ /dev/null @@ -1 +0,0 @@ -repo diff --git a/build_tools/packaging/linux/build_repo_setup_package.py b/build_tools/packaging/linux/build_repo_setup_package.py new file mode 100644 index 00000000000..f9af1ec73d9 --- /dev/null +++ b/build_tools/packaging/linux/build_repo_setup_package.py @@ -0,0 +1,1114 @@ +#!/usr/bin/env python3 + +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +r"""Build the ``amd-repo-setup`` package that configures a system package +manager (apt/dnf/zypper) to install AMD ROCm from the public ROCm repositories. + +One ``amd-repo-setup`` package is generated per target OS profile and RFC0012 +stream. It ships a single repository definition and, for signed streams, the AMD +signing key, so that after installing the package a user can install ROCm with +their native package manager. The signing key is fetched at build time and +embedded in the package; it is never stored in the source tree. + +Package naming +-------------- + +Every OS profile builds a *different* package, because each embeds its own +repository URL. The profile therefore has to reach the package name, or two of +them would be indistinguishable. Each ecosystem does that its own way: use the +distro token the ecosystem already defines where there is one, and the +``--os-profile`` name where there is not. + + deb the profile goes in the version: 10.0.0-1~ubuntu2404 + rpm the profile goes in the Release: 10.0.0-1.stable.el8 + +``-1`` is ``--version-suffix``, this package's own build number (default 1), +bumped to re-ship the repository configuration at an unchanged ROCm version. + +``--rocm-version`` is used as given, for every stream. Pass the version of the +ROCm packages in the repository the package configures, as +compute_rocm_package_version.py computes it for the stream's release type; CI +passes exactly that. The package then sorts the way those ROCm packages do: +below stable at the same ROCm version. rc is spelled differently per format +(``~preN`` on deb, ``~rcN`` on rpm). + +Worked examples with the default ``--version-suffix``. A build_id stream +additionally takes ``--repo-sub-folder`` (nightly here uses 20260716-12345); a +flat stream rejects it: + + --os-profile --stream --rocm-version resulting package file + ------------ -------- --------------- ------------------------------------------------------- + ubuntu2404 stable 10.0.0 amd-repo-setup_10.0.0-1~ubuntu2404_all.deb + ubuntu2404 rc 10.0.0~pre1 amd-repo-setup_10.0.0~pre1-1~ubuntu2404_all.deb + ubuntu2404 nightly 10.0.0~20260716 amd-repo-setup_10.0.0~20260716-1~ubuntu2404_all.deb + rhel8 stable 10.0.0 amd-repo-setup-10.0.0-1.stable.el8.noarch.rpm + rhel8 rc 10.0.0~rc1 amd-repo-setup-10.0.0~rc1-1.rc.el8.noarch.rpm + rhel8 nightly 10.0.0~20260716 amd-repo-setup-10.0.0~20260716-1.nightly.el8.noarch.rpm + rhel10 stable 10.0.0 amd-repo-setup-10.0.0-1.stable.el10.noarch.rpm + sles16 stable 10.0.0 amd-repo-setup-10.0.0-1.stable.sles16.noarch.rpm + +``rc`` configures the release-candidate repository for the next GA, and nightly +builds a later ROCm version still, so moving from either back to stable is +usually a downgrade. rc's repo file is ``amdrocm-stablerc`` -- see STREAM_IDS. + +``` +python build_repo_setup_package.py \ + --os-profile ubuntu2404 \ + --stream stable \ + --repo-base-url https://stable.repo.amd.com/rocm/core/packages \ + --rocm-version 10.0.0 \ + --dest-dir ./output +``` + +The same build for a per-build stream, which pins the repository to one build +folder and ships no signing key (those repositories are unsigned): + +``` +python build_repo_setup_package.py \ + --os-profile rhel8 \ + --stream nightly \ + --repo-base-url https://nightly.repo.amd.com/rocm/core/packages \ + --repo-sub-folder 20260716-12345 \ + --rocm-version 10.0.0~20260716 \ + --dest-dir ./output +``` +""" + +import argparse +import json +import re +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request + +from datetime import datetime, timezone +from email.utils import format_datetime +from pathlib import Path, PurePosixPath + +from jinja2 import Environment, FileSystemLoader, select_autoescape + +SCRIPT_DIR = Path(__file__).resolve().parent + +# The package name. The repository identifier used for the repo file and its +# section header is derived per stream by ``repo_id()``. A single repository is +# configured per package. +PACKAGE_NAME = "amd-repo-setup" +REPO_ID_PREFIX = "amdrocm" +REPO_NAME = "AMD ROCm" + +# The deb suite the repositories publish under (``dists//``). Used to +# locate the index when verifying a repo URL. It is written literally in +# template/repo/deb/amdrocm.sources.j2; a test asserts the two agree. +# +# One suite for every stream, matching what the servers actually publish rather +# than naming the suite after the stream. Every stream serves ``dists/stable/`` +# and declares ``Suite: stable`` / ``Codename: stable`` inside its own Release +# file; a per-stream suite such as ``dists/stablerc/`` returns 404. Pointing apt +# at a stream-named suite would therefore produce a package that installs +# cleanly and then fails on the user's first update, so this follows the server. +DEB_SUITE = "stable" + +MAINTAINER = "ROCm Dev Support " + +# Where each package installs the signing key, and which the repo files +# reference. deb consumes a dearmored (binary) keyring via Signed-By; rpm +# references the (armored) key via gpgkey=file://. +# +# Both are named for this package (amdrocm) rather than the generic "rocm", so +# that no other package or hand-written setup step can claim the same path. +# dpkg and rpm both refuse to unpack two packages owning one path, so a generic +# name risks making this package uninstallable alongside something unrelated. +# This is precautionary: amdgpu-install 31.50 keeps its key elsewhere +# (/etc/apt/keyrings/rocm.gpg, /etc/amdgpu-install/rocm.gpg.key), so it is not +# a collision we have observed. The overlap that does exist with that package +# is the repository definition, and Conflicts handles it -- see +# LEGACY_INSTALLER_PACKAGE. +# These are installed names only; the key itself is fetched from the URL +# given by --gpg-key-url. +# +# /etc/pki/rpm-gpg is the RHEL-family convention. SUSE has no filesystem +# convention for third-party repo keys -- its trust store is the rpm database, +# and gpgkey= is only an import source -- so the same path is used there. No +# SUSE package provides /etc/pki/rpm-gpg, so nothing can collide with it. +DEB_KEYRING_PATH = "/usr/share/keyrings/amdrocm.gpg" +RPM_GPG_KEY_PATH = "/etc/pki/rpm-gpg/RPM-GPG-KEY-amdrocm" + +# The legacy installer this package must not be co-installed with. It ships its +# own enabled ROCm repository as a packaged file -- /etc/apt/sources.list.d/ +# rocm.list on deb, [rocm] in /etc/yum.repos.d/rocm.repo on rpm -- both pointing +# at repo.radeon.com. Its repo files use different names than ours, so nothing +# stops the two being installed together; the result is a system with two ROCm +# repositories configured. It also grants itself precedence (an apt pin at +# priority 600 against a 500 default, and priority=50 against dnf's 99 default), +# so it wins package resolution rather than reporting a conflict. +# RFC0012's "Repository Package" section names this rule and defers its +# definition to a separate repository-package RFC. +# +# What the declaration actually achieves differs by package manager, so do not +# describe it as closing the overlap outright: +# +# dnf/zypper refuse the pair with an explicit conflict error, and erasing +# that package also removes its %config repo files. Clean. +# apt satisfies the conflict by removing the package to "rc" state. +# Its repo file and priority pin are conffiles, so both survive. +# The packages are no longer co-installed, but the repository +# overlap remains until the user purges rather than removes. +# +# The Debian remedy is therefore "apt purge", which the documentation states. +LEGACY_INSTALLER_PACKAGE = "amdgpu-install" + +GPG_FETCH_TIMEOUT_SEC = 60 +# Retry the key fetch so a transient network/CDN blip does not fail a build. +GPG_FETCH_ATTEMPTS = 3 +GPG_FETCH_BACKOFF_SEC = 3 +# Upper bound on the fetched signing key. A real armored key is a few KB; this +# bounds memory use if a repository host returns an unexpectedly large body. +MAX_GPG_KEY_BYTES = 1 << 20 # 1 MiB + +# Bounds for the optional repository reachability check (--verify-repo-url). +REPO_VERIFY_TIMEOUT_SEC = 30 +REPO_VERIFY_ATTEMPTS = 3 +REPO_VERIFY_BACKOFF_SEC = 3 +# HTTP statuses worth retrying, in this check and in the signing-key fetch: +# request timeout and rate limiting. Any other 4xx means nothing is served at +# that URL, which retrying cannot change. +RETRYABLE_HTTP_STATUS = frozenset({408, 429}) + +# Primary-key fingerprint of the AMD ROCm signing key. The key is fetched over +# the network at build time and embedded as the package's trust anchor, so its +# identity is pinned here and verified after fetch. Rotating the repository +# signing key requires updating this value. +EXPECTED_KEY_FINGERPRINT = "D0F004A0025A1145C7807FCD0701EAC4D5E02107" + +# Accepted input shapes. rocm_version is rendered into the package version +# fields and changelog, and repo_sub_folder into the repository URL, so they are +# constrained to characters that cannot alter the surrounding syntax. +# repo_base_url is concatenated into the repo files as well. Use \Z (not $) so a +# trailing newline is not accepted. +# +# No "-": an rpm Version cannot contain one, since rpm uses it to separate the +# version from the release (see rpm_version_release). A pre-release marker +# belongs in the "~rc1" form, which both rpm and dpkg sort before the plain +# version. +# +# The sub-folder must have the shape of a published build folder, +# -; anything else names a folder that does not exist. +_VERSION_RE = re.compile(r"^[0-9][0-9A-Za-z.+~]*\Z") +_SUB_FOLDER_RE = re.compile(r"^[0-9]{8}-[0-9A-Za-z._]+\Z") + +# --version-suffix is the package's own build number, bumped to re-ship the +# repository configuration at an unchanged ROCm version. It is stricter than +# _VERSION_RE in two ways: +# +# no "-" deb_version joins it to the upstream version with the single "-" +# that separates a debian revision, and rpm forbids one in a Release. +# no "~" deb_version appends "~" after it, and "~" sorts before +# everything, so a "~" inside the suffix would reorder the two. +_VERSION_SUFFIX_RE = re.compile(r"^[0-9][0-9A-Za-z.+]*\Z") + +# The RFC0012 streams, each served from .repo.amd.com. +# +# A stream has one of two shapes, and everything else follows from the shape: +# +# flat one repository per distro, holding every retained version. +# Signed, so the package ships the key and enables gpgcheck. +# build_id the same tree with a segment naming a single +# build. Unsigned, so the package ships no key and trusts the repo. +# +# Signedness is not an independent axis: the flat streams are the ones that +# publish InRelease/Release.gpg and repomd.xml.asc, and the build_id streams +# publish none of them. Branch on the shape, never on the stream name. +_FLAT = "flat" +_BUILD_ID = "build_id" + +STREAM_SHAPES = { + "stable": _FLAT, + "rc": _FLAT, + "nightly": _BUILD_ID, +} +STREAMS = tuple(STREAM_SHAPES) + +# Adding a stream is an entry here, any alternate names it carries (see +# STREAM_IDS), and its golden tests. A build_id stream that publishes several +# builds a day, as the dev repository does, would also need the run ID as +# --version-suffix: its version carries only the date, so two of its packages +# would otherwise share one. + +# Alternate names a stream is known by, where they differ from the stream id. +# Streams absent here, or missing a key, use the stream id unchanged. +# +# "file_id" the installed repo file. RFC0012 names rc's tier package +# amdrocm-repo-stablerc, so the file is amdrocm-stablerc while +# the subdomain stays rc. Matching it now keeps the eventual +# per-tier split from also having to rename an installed file. +STREAM_IDS = { + "rc": {"file_id": "stablerc"}, +} + + +def _stream_id(stream: str, key: str) -> str: + """Return a stream's alternate name for ``key``, or the stream name.""" + return STREAM_IDS.get(stream, {}).get(key, stream) + + +# "dist_tag" is appended to the rpm Release so two profiles cannot produce an +# identically named package. Stated per profile rather than taken from rpm's +# %{?dist}, which reads the *build host* rather than the target (a rhel8 build +# in a RHEL 9 container would stamp ".el9") and is empty on SUSE, so every SUSE +# profile would share one Release -- invisible while sles16 is the only one. +# +# ".el8"/".el10" reproduce what %{?dist} yields there, so RHEL names do not +# change. ".sles16" is a deliberate departure from SUSE's own convention, which +# leads the Release with a numeric code (160000.2.2 on SLE 16.0): that is a +# prefix where RHEL's is a suffix, so no single placement satisfies both. +# +# deb profiles carry no dist_tag -- deb_version() puts the profile in the +# version -- so read it with .get(); build_context() runs for both types. +OS_PROFILES = { + "ubuntu2404": { + "family": "debian", + "pkg_type": "deb", + "description": "Ubuntu 24.04", + }, + "rhel8": { + "family": "rhel", + "pkg_type": "rpm", + "rpm_repo_dir": "/etc/yum.repos.d", + "rpm_refresh_cmd": "sudo dnf makecache", + "dist_tag": ".el8", + "description": "RHEL/Rocky/OL 8", + }, + "rhel10": { + "family": "rhel", + "pkg_type": "rpm", + "rpm_repo_dir": "/etc/yum.repos.d", + "rpm_refresh_cmd": "sudo dnf makecache", + "dist_tag": ".el10", + "description": "RHEL/Rocky/OL 10", + }, + "sles16": { + "family": "sles", + "pkg_type": "rpm", + "rpm_repo_dir": "/etc/zypp/repos.d", + "rpm_refresh_cmd": "sudo zypper refresh", + "dist_tag": ".sles16", + "description": "SLES 16", + }, +} + + +# --- URL / signing derivation ------------------------------------------------ +# +# Every stream is served per distro under a common base: +# +# flat // (deb) +# //x86_64/ (rpm) +# build_id /// (deb) +# ///x86_64/ (rpm) +# +# where is https://.repo.amd.com/rocm/core/packages. +# +# The per-format trees (/deb/, /rpm/) are also served, and hold the +# same builds, so a package pointing at either one works today. They are not +# used here: the published install instructions configure the per-distro tree, +# and a stream that ever drops the per-format alias would break silently in the +# field rather than in CI. A dual-served layout cannot tell a right mapping from +# a wrong one. +# +# The signing key is not under at all -- it sits beside core/, two +# levels up -- so its URL is passed in whole rather than derived here. That +# also keeps the key's location a property of the publisher rather than an +# assumption baked into this tool. + + +def repo_id(stream: str) -> str: + """Return the repo-file stem and section id for a stream. + + ``amdrocm-stable``, ``amdrocm-stablerc`` etc. The stem drives the installed + filename in both families -- ``debian/install`` maps ``{repo_id}.sources`` + and the spec ``%files`` lists ``{repo_id}.repo`` -- so naming the stream + here is what keeps two streams' repo files from overwriting one another on + disk. + + The stem is not always the stream name: rc's file is ``amdrocm-stablerc`` + while its subdomain stays ``rc``. See STREAM_IDS. + """ + stream_shape(stream) # reject unknown streams before they reach a filename + return f"{REPO_ID_PREFIX}-{_stream_id(stream, 'file_id')}" + + +def stream_shape(stream: str) -> str: + """Return the layout shape for a stream. + + Raises: + ValueError: If the stream is unknown. + """ + try: + return STREAM_SHAPES[stream] + except KeyError as e: + raise ValueError( + f"stream={stream!r} is invalid, expected one of {STREAMS}" + ) from e + + +def is_signed(stream: str) -> bool: + """Whether this stream's public repository is signed. + + Signedness follows the shape: flat streams publish ``InRelease`` / + ``Release.gpg`` (deb) and ``repomd.xml.asc`` (rpm); build_id streams publish + none of them and are consumed with ``trusted=yes`` / ``gpgcheck=0``. + """ + return stream_shape(stream) == _FLAT + + +def repo_baseurl( + repo_base_url: str, + pkg_type: str, + os_profile: str, + stream: str, + repo_sub_folder: str, +) -> str: + """Return the repository baseurl baked into the repo file. + + deb returns the repo root, beneath which apt resolves ``dists/stable/main/``; + rpm returns the directory containing ``repodata/``. ``repo_sub_folder`` is + the ``YYYYMMDD-runid`` segment naming a single build, required by the + build_id shape and rejected for the flat one. + + Raises: + ValueError: If the stream is unknown, or the sub-folder does not match + the stream's shape. + """ + base = repo_base_url.rstrip("/") + shape = stream_shape(stream) + # Silently dropping a missing sub-folder would emit a flat URL for a + # build_id stream -- indistinguishable from a correct stable URL, and + # serving nothing, so the package would install and fail on first refresh. + if shape == _BUILD_ID and not repo_sub_folder: + raise ValueError(f"stream={stream!r} requires a build sub-folder") + if shape != _BUILD_ID and repo_sub_folder: + raise ValueError(f"stream={stream!r} takes no build sub-folder") + parts = [base, os_profile] + if shape == _BUILD_ID: + parts.append(repo_sub_folder) + if pkg_type != "deb": + parts.append("x86_64") + return "/".join(parts) + "/" + + +def repo_metadata_url(baseurl: str, pkg_type: str) -> str: + """Return a repository index file that must exist under ``baseurl``. + + apt reads ``dists//Release``; dnf and zypper read + ``repodata/repomd.xml``. Fetching one proves the baseurl serves a repository. + """ + if pkg_type == "deb": + return f"{baseurl}dists/{DEB_SUITE}/Release" + return f"{baseurl}repodata/repomd.xml" + + +def verify_repo_url(baseurl: str, pkg_type: str) -> None: + """Fail the build if ``baseurl`` does not serve a package repository. + + The baseurl is baked into the package and is only exercised the first time a + user refreshes their package manager, so an unreachable one would otherwise + ship undetected. Transient failures are retried; a repository that is simply + not there fails immediately. + + The scheme is not re-checked here: this response is neither trusted nor + retained, and the signing key (which is) enforces https separately. + """ + url = repo_metadata_url(baseurl, pkg_type) + last_error: Exception | None = None + for attempt in range(1, REPO_VERIFY_ATTEMPTS + 1): + try: + print(f"Verifying repository (attempt {attempt}): {url}") + with urllib.request.urlopen(url, timeout=REPO_VERIFY_TIMEOUT_SEC) as resp: + status = getattr(resp, "status", None) or resp.getcode() + if status == 200: + print(f"Repository verified: {baseurl}") + return + # urlopen already raised for the error codes, so anything else here + # is a URL that answers but does not serve a repository index. + # Retrying cannot change that, so fail now rather than sleeping + # through the remaining attempts. + raise RuntimeError( + f"repository is not reachable at {baseurl} " + f"(checked {url}): unexpected HTTP status {status}" + ) + except urllib.error.HTTPError as e: + # A missing or forbidden index is a wrong URL, not a blip. Some + # object stores answer 403 rather than 404 for absent keys, so treat + # both as final; only rate limiting and timeouts are worth retrying. + if e.code not in RETRYABLE_HTTP_STATUS: + raise RuntimeError( + f"repository is not reachable at {baseurl} " + f"(checked {url}): HTTP {e.code}" + ) from e + last_error = e + except (urllib.error.URLError, TimeoutError, ConnectionError) as e: + last_error = e + if attempt < REPO_VERIFY_ATTEMPTS: + time.sleep(REPO_VERIFY_BACKOFF_SEC * attempt) + raise RuntimeError( + f"repository is not reachable at {baseurl} (checked {url}): {last_error}" + ) + + +def _require_https(url: str, what: str) -> None: + """Reject a non-https URL used to fetch trusted key material.""" + if urllib.parse.urlsplit(url).scheme != "https": + raise ValueError(f"{what} must use https to protect the key in transit: {url}") + + +def _valid_repo_base_url(url: str) -> bool: + """Whether a repo base URL is a well-formed http(s) URL with no control chars. + + The value is concatenated into the generated repo files, so whitespace or + control characters (e.g. a newline) that could inject extra directives are + rejected here. + """ + if any(c.isspace() or ord(c) < 0x20 for c in url): + return False + parts = urllib.parse.urlsplit(url) + return parts.scheme in ("http", "https") and bool(parts.netloc) + + +def rpm_version_release( + stream: str, + rocm_version: str, + version_suffix: str, + dist_tag: str, +) -> tuple[str, str]: + """Return (Version, Release) for the rpm package. + + Version is ``--rocm-version`` as given (see deb_version). Release is + ``.``, e.g. ``1.stable.el8``. Each part + exists to keep two packages from sharing a name: + + - ``stream`` -- without it, installing one stream's package over another at + the same version is a silent no-op: the package manager reports success + and leaves the old repository configured. + - ``dist_tag`` -- one package is built per OS profile and their contents + differ, so the profile must reach the name too (see OS_PROFILES). + - ``version_suffix`` -- this package's own build number, so the repository + configuration can be re-shipped at an unchanged ROCm version. + """ + return rocm_version, f"{version_suffix}.{stream}{dist_tag}" + + +# An rpm %changelog date has to be English whatever LC_TIME the build machine +# happens to use. Both strftime("%a %b") and calendar.day_abbr read the process +# locale, so the names are pinned here instead. This mirrors what the standard +# library does for the deb changelog: email.utils.format_datetime carries its +# own English tables for the same reason. +_CHANGELOG_DAY_ABBR = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun") +_CHANGELOG_MONTH_ABBR = ( + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec", +) + + +def rpm_changelog_date(when: datetime) -> str: + """Return ``when`` formatted as an rpm %changelog date, always in English.""" + return ( + f"{_CHANGELOG_DAY_ABBR[when.weekday()]} " + f"{_CHANGELOG_MONTH_ABBR[when.month - 1]} " + f"{when.day:02d} {when.year}" + ) + + +def deb_version(rocm_version: str, version_suffix: str, os_profile: str) -> str: + """Return the deb package version, e.g. ``10.0.0-1~ubuntu2404``. + + The value is ``-``. The upstream part is + ``--rocm-version`` as given, for every stream. CI passes the version that + compute_rocm_package_version.py computes for the stream's release type, + which is also the version of the ROCm packages in that stream's repository + (``10.1.0``, ``10.1.0~pre3``, ``10.2.0~20261013``), so this package sorts + the way they do. + + The revision carries ``~``: + + - ``version_suffix`` is this package's own build number, so the repository + configuration can be re-shipped at an unchanged ROCm version. + - ``os_profile`` is part of the version because one package is built per + profile and their contents differ -- each embeds its own repository URL. + Without it every debian-family profile would produce an identically named + package holding different files. + + Profile names rather than Debian codenames: codenames are not + alphabetically ordered, so ``10.0.0-1~bullseye1`` sorts *above* + ``10.0.0-1~bookworm1`` and an in-place Debian 11 -> 12 upgrade would see the + newer package as a downgrade. ``ubuntu2404``/``debian12`` are monotonic, and + match both ``--os-profile`` and the ``repo//`` publish path. + + Note that ``~`` sorts before everything, including nothing at all, so + ``10.0.0-1~ubuntu2404`` ranks *below* a bare ``10.0.0-1``. Every package + built here carries a profile, so that comparison never arises between two of + ours -- but it is why the separator must stay ``~`` and not become ``.``. + """ + return f"{rocm_version}-{version_suffix}~{os_profile}" + + +# --- signing key ------------------------------------------------------------- + + +def _fetch_signing_key(url: str) -> bytes: + """Fetch the armored signing key over https, size-bounded, with retries. + + Network errors and the HTTP statuses in ``RETRYABLE_HTTP_STATUS`` are + retried. Any other HTTP status, a scheme downgrade or an over-size body + fails immediately, as in ``verify_repo_url``. + """ + # The fetched key is embedded and trusted, so it must travel over https. + _require_https(url, "signing key URL") + last_error: Exception | None = None + for attempt in range(1, GPG_FETCH_ATTEMPTS + 1): + try: + print(f"Fetching signing key (attempt {attempt}): {url}") + with urllib.request.urlopen(url, timeout=GPG_FETCH_TIMEOUT_SEC) as resp: + # urlopen follows redirects; re-check the scheme actually served + # so an https URL cannot be silently downgraded to http. + _require_https(resp.geturl(), "signing key URL (after redirects)") + # Read one byte past the cap to detect an over-size body. + key = resp.read(MAX_GPG_KEY_BYTES + 1) + if len(key) > MAX_GPG_KEY_BYTES: + raise ValueError( + f"signing key exceeds {MAX_GPG_KEY_BYTES} bytes: {url}" + ) + return key + except urllib.error.HTTPError as e: + # HTTPError is a URLError, so it must be caught first. A missing or + # forbidden key is a wrong URL, not a blip: fail now rather than + # sleeping through the remaining attempts. + if e.code not in RETRYABLE_HTTP_STATUS: + raise RuntimeError( + f"failed to fetch signing key: HTTP {e.code}: {url}" + ) from e + last_error = e + except (urllib.error.URLError, TimeoutError, ConnectionError) as e: + last_error = e + if attempt < GPG_FETCH_ATTEMPTS: + time.sleep(GPG_FETCH_BACKOFF_SEC * attempt) + raise RuntimeError( + f"failed to fetch signing key after {GPG_FETCH_ATTEMPTS} attempts: {url}" + ) from last_error + + +def _run_gpg(argv: list[str], **kwargs) -> subprocess.CompletedProcess: + """Run gpg, reporting a missing binary by name instead of a traceback. + + A signed stream reaches gpg in two places: the deb path always, to dearmor + the keyring, and either path when the key is fetched, to check its + fingerprint. + An rpm build given --gpg-key-file does neither, so it needs no gpg at all. + + The callers capture gpg's output, and ``CalledProcessError`` leaves it out + of its message, so a failure is re-raised with gpg's own error text. + """ + try: + return subprocess.run(argv, check=True, **kwargs) + except FileNotFoundError as e: + raise RuntimeError( + "gpg is required to build a signed stream: it dearmors the deb " + "keyring, and checks the fingerprint of a fetched key. Install " + "gnupg/gnupg2. Note that --gpg-key-file skips the fetch, not the deb " + "keyring step, so a signed deb build needs gpg either way" + ) from e + except subprocess.CalledProcessError as e: + stderr = e.stderr + if isinstance(stderr, bytes): + stderr = stderr.decode(errors="replace") + detail = (stderr or "").strip() or "no error output" + raise RuntimeError(f"gpg exited with status {e.returncode}: {detail}") from e + + +def _primary_fingerprints(listing: str) -> list[str]: + """Return the primary-key fingerprints in ``gpg --with-colons`` output. + + Each ``fpr`` record belongs to the ``pub`` or ``sub`` record before it; only + those following a ``pub`` identify a primary key. + """ + primaries = [] + owner = None + for line in listing.splitlines(): + record = line.split(":") + if record[0] in ("pub", "sub"): + owner = record[0] + elif record[0] == "fpr" and owner == "pub": + primaries.append(record[9]) + return primaries + + +def _extract_pinned_key(armored: bytes) -> bytes: + """Return the pinned AMD ROCm key, armored, from a fetched key file. + + The file must hold exactly one primary key, the pinned one. Checking only + the first key would let a file with other keys appended pass, and every + key in the embedded file is trusted on the user's machine. The key is + returned as gpg re-exports it, which keeps the primary key and its valid + subkeys (the packages are signed by a subkey) and drops anything else. + ``import-export`` writes no keyring, so no gpg daemon is started. + """ + with tempfile.TemporaryDirectory() as home: + key_path = Path(home) / "key.asc" + key_path.write_bytes(armored) + listing = _run_gpg( + ["gpg", "--homedir", home, "--with-colons", "--show-keys", str(key_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ).stdout.decode(errors="replace") + primaries = _primary_fingerprints(listing) + if primaries != [EXPECTED_KEY_FINGERPRINT]: + raise ValueError( + f"fetched signing key must be exactly the pinned key " + f"{EXPECTED_KEY_FINGERPRINT}; found {primaries or 'no key'}" + ) + return _run_gpg( + [ + "gpg", + "--homedir", + home, + "--batch", + "--armor", + "--import-options", + "import-export", + "--import", + str(key_path), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ).stdout + + +def load_signing_key(args: argparse.Namespace) -> bytes: + """Return the armored signing key bytes for a signed stream. + + Read from ``--gpg-key-file`` when given (offline/test builds), otherwise + fetched from ``--gpg-key-url`` at build time. Never written into the + source tree. + + Two guards apply to the fetch only: the https requirement + (``_require_https``, including after redirects) and the pinned key (the + file must hold only that key, and only it is embedded). They exist to + detect a tampered or misconfigured repository, which is a remote input. A key passed with ``--gpg-key-file`` is a local file the + caller has already chosen, so it is embedded as provided and neither guard + runs; callers supplying their own key are responsible for it being the + right one. + + ``--gpg-key-url`` is still checked for http(s) shape by ``parse_args`` + whenever it is non-empty, including alongside ``--gpg-key-file``. That is a + separate, earlier check -- it rejects a malformed value rather than + vouching for the key. + """ + if args.gpg_key_file: + return args.gpg_key_file.read_bytes() + return _extract_pinned_key(_fetch_signing_key(args.gpg_key_url)) + + +def dearmor_key(armored: bytes) -> bytes: + """Convert an armored key to a binary keyring (for deb Signed-By).""" + result = _run_gpg( + ["gpg", "--dearmor"], + input=armored, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return result.stdout + + +# --- templating -------------------------------------------------------------- + + +def get_jinja_env() -> Environment: + # keep_trailing_newline so rendered config files end with a newline (POSIX + # text-file hygiene; Jinja strips the final newline by default). + # + # autoescape matches deb_package.py and rpm_package.py: escaping is enabled + # for markup extensions only. Every template here renders a package-manager + # config file, not markup, and escaping one would corrupt it -- an "&" in a + # repository URL would become "&". The values interpolated into these + # files are constrained by the input patterns above, which is the control + # that actually applies to this syntax. + return Environment( + loader=FileSystemLoader(str(SCRIPT_DIR)), + autoescape=select_autoescape( + enabled_extensions=("html", "htm", "xml"), + default_for_string=True, + default=False, + ), + keep_trailing_newline=True, + trim_blocks=True, + lstrip_blocks=True, + ) + + +def _render( + env: Environment, template_path: str, output_path: Path, context: dict +) -> None: + # Explicit encoding: write_text() would otherwise follow the build + # machine's locale, and this module is deliberately locale-independent + # elsewhere (see rpm_changelog_date). + output_path.write_text( + env.get_template(template_path).render(context), encoding="utf-8" + ) + + +def build_context(args: argparse.Namespace, profile: dict) -> dict: + """Template fields shared by the repo file, spec, and deb metadata.""" + signed = is_signed(args.stream) + return { + "pkg_name": PACKAGE_NAME, + "repo_id": repo_id(args.stream), + "name": REPO_NAME, + "baseurl": repo_baseurl( + args.repo_base_url, + profile["pkg_type"], + args.os_profile, + args.stream, + args.repo_sub_folder, + ), + "signed": signed, + "rocm_version": args.rocm_version, + "maintainer": MAINTAINER, + "description_short": "AMD ROCm repository configuration", + "description_long": ( + "Configures the system package manager to install AMD ROCm packages." + ), + "deb_keyring_path": DEB_KEYRING_PATH, + # Split out so debian/install maps the staged file to the same place + # Signed-By points at, without repeating the name. + # + # PurePosixPath, not Path: these describe the layout of the package + # being built, not of the machine building it. Path follows the local + # flavour, so on Windows the parent would render as + # "\usr\share\keyrings" and apt would never find the keyring. + "deb_keyring_file": PurePosixPath(DEB_KEYRING_PATH).name, + "deb_keyring_dir": str(PurePosixPath(DEB_KEYRING_PATH).parent), + "rpm_gpg_key_path": RPM_GPG_KEY_PATH, + # Same reason as deb: the spec stages and installs by basename, so + # keep one source of truth for the name. + "rpm_gpg_key_file": PurePosixPath(RPM_GPG_KEY_PATH).name, + "rpm_repo_dir": profile.get("rpm_repo_dir", ""), + # .get(), not a subscript: this function runs before main() dispatches + # on pkg_type, so it is called for deb profiles too -- and those carry + # no dist_tag. + # + # Consumed by build_rpm_package, which hands it to rpm_version_release. + # The spec template renders the finished Release string and must not + # append the tag a second time. + "dist_tag": profile.get("dist_tag", ""), + # Same .get() reasoning as dist_tag: deb profiles reach here too. + "rpm_refresh_cmd": profile.get("rpm_refresh_cmd", ""), + # Declared for every stream: amdgpu-install configures a ROCm + # repository whichever stream this package points at. + "legacy_installer_package": LEGACY_INSTALLER_PACKAGE, + } + + +# --- rpm build --------------------------------------------------------------- + + +def build_rpm_package( + args: argparse.Namespace, profile: dict, context: dict, dest_dir: Path +) -> None: + env = get_jinja_env() + build_dir = dest_dir / "rpm" / PACKAGE_NAME + if build_dir.exists(): + shutil.rmtree(build_dir) + for subdir in ("SOURCES", "SPECS", "BUILD", "RPMS", "SRPMS"): + (build_dir / subdir).mkdir(parents=True, exist_ok=True) + sources_dir = build_dir / "SOURCES" + + _render( + env, + "template/repo/rpm/amdrocm.repo.j2", + sources_dir / f"{context['repo_id']}.repo", + context, + ) + + if context["signed"]: + # rpm consumes the armored key as-is (gpgkey=file://). + (sources_dir / PurePosixPath(RPM_GPG_KEY_PATH).name).write_bytes( + load_signing_key(args) + ) + + version, release = rpm_version_release( + args.stream, + args.rocm_version, + args.version_suffix, + context["dist_tag"], + ) + spec_context = { + **context, + "version": version, + "release": release, + "changelog_date": rpm_changelog_date(datetime.now(timezone.utc)), + } + spec_path = build_dir / "SPECS" / f"{PACKAGE_NAME}.spec" + _render(env, f"template/repo/rpm/{PACKAGE_NAME}.spec.j2", spec_path, spec_context) + + print(f"Building rpm package: {PACKAGE_NAME} for {profile['description']}") + subprocess.run( + ["rpmbuild", "--define", f"_topdir {build_dir}", "-bb", str(spec_path)], + check=True, + ) + for rpm_file in (build_dir / "RPMS" / "noarch").glob("*.rpm"): + target = dest_dir / rpm_file.name + shutil.move(str(rpm_file), str(target)) + print(f"Package created: {target}") + + # Leave only the built packages behind. The rpmbuild tree is scratch, and + # CI publishes this directory, so anything left here ships with the + # package. Failing to clean up must not fail a build that succeeded. + shutil.rmtree(build_dir.parent, ignore_errors=True) + + +# --- deb build --------------------------------------------------------------- + + +def build_deb_package( + args: argparse.Namespace, profile: dict, context: dict, dest_dir: Path +) -> None: + env = get_jinja_env() + package_dir = dest_dir / "deb" / PACKAGE_NAME + deb_dir = package_dir / "debian" + if package_dir.exists(): + shutil.rmtree(package_dir) + deb_dir.mkdir(parents=True, exist_ok=True) + + deb_context = { + **context, + "deb_version": deb_version( + args.rocm_version, + args.version_suffix, + args.os_profile, + ), + "date": format_datetime(datetime.now(timezone.utc)), + } + + # Repository definition (deb822) and, for a signed stream, the dearmored key, + # placed at the package root and mapped into the filesystem by debian/install. + _render( + env, + "template/repo/deb/amdrocm.sources.j2", + package_dir / f"{deb_context['repo_id']}.sources", + deb_context, + ) + if context["signed"]: + (package_dir / PurePosixPath(DEB_KEYRING_PATH).name).write_bytes( + dearmor_key(load_signing_key(args)) + ) + + for name in ("control", "changelog", "rules", "postinst", "install"): + _render( + env, + f"template/repo/deb/{name}.j2", + deb_dir / name, + deb_context, + ) + (deb_dir / "rules").chmod(0o755) + (deb_dir / "postinst").chmod(0o755) + # Native source format: this package has no upstream outside this + # repository, so there is no separate tarball for a non-native format to + # reference. + # + # A native package conventionally carries no debian revision, and this one + # does (see deb_version). That convention is enforced when dpkg-source + # builds a source package, which a binary-only build never does: + # "dpkg-buildpackage -b" below runs only dpkg-source's --before-build and + # --after-build hooks and produces the .deb straight from debian/. + (deb_dir / "source").mkdir(exist_ok=True) + (deb_dir / "source" / "format").write_text("3.0 (native)\n", encoding="utf-8") + + print(f"Building deb package: {PACKAGE_NAME} for {profile['description']}") + subprocess.run( + ["dpkg-buildpackage", "-uc", "-us", "-b"], + cwd=package_dir, + check=True, + ) + # dpkg-buildpackage writes the .deb to the parent of the package directory. + for deb_file in package_dir.parent.glob("*.deb"): + target = dest_dir / deb_file.name + shutil.move(str(deb_file), str(target)) + print(f"Package created: {target}") + + # As for rpm: drop the build tree, and with it the .buildinfo and .changes + # dpkg-buildpackage leaves alongside the package. + shutil.rmtree(package_dir.parent, ignore_errors=True) + + +# --- CLI --------------------------------------------------------------------- + + +def list_profiles(pkg_type: str) -> list[str]: + """Return the OS profiles that build the given package type.""" + return [name for name, p in OS_PROFILES.items() if p["pkg_type"] == pkg_type] + + +def run_list_profiles(argv: list[str]) -> None: + """Print the OS profiles for a package type as a JSON array (CI matrix).""" + p = argparse.ArgumentParser( + prog="build_repo_setup_package.py list-profiles", + description="Print the OS profiles for a package type as a JSON array.", + ) + p.add_argument("--pkg-type", required=True, choices=["deb", "rpm"]) + ns = p.parse_args(argv) + print(json.dumps(list_profiles(ns.pkg_type))) + + +def parse_args(argv=None) -> argparse.Namespace: + p = argparse.ArgumentParser( + description="Build the amd-repo-setup package that configures ROCm repositories.", + ) + p.add_argument( + "--os-profile", + required=True, + choices=sorted(OS_PROFILES.keys()), + help="Target distro profile", + ) + p.add_argument( + "--stream", + required=True, + choices=STREAMS, + help="RFC0012 repository stream to configure", + ) + p.add_argument( + "--repo-base-url", + required=True, + help=( + "Stream repository base URL, e.g. " + "https://stable.repo.amd.com/rocm/core/packages" + ), + ) + p.add_argument( + "--repo-sub-folder", + default="", + help="Build sub-folder for a per-build stream (YYYYMMDD-)", + ) + p.add_argument( + "--gpg-key-file", + type=Path, + default=None, + help=( + "Signing key file to embed (defaults to fetching --gpg-key-url). The " + "pinned-fingerprint check applies to the fetched key only; a key " + "given here is embedded as provided" + ), + ) + p.add_argument( + "--rocm-version", + required=True, + help=( + "Package version, used as given: the version of the ROCm packages " + "in the repository, as compute_rocm_package_version.py computes it " + "for the stream's release type (e.g. 10.0.0; 10.1.0~pre3 on deb " + "or 10.1.0~rc3 on rpm; 10.2.0~20261013)" + ), + ) + p.add_argument( + "--version-suffix", + default="1", + help=( + "This package's own build number, separate from --rocm-version " + "(default: 1). Bump it to re-ship the repository configuration " + "at an unchanged ROCm version: without it there would be no " + "higher version to publish, and the package manager would treat " + "the corrected package as already installed" + ), + ) + p.add_argument( + "--dest-dir", + type=Path, + required=True, + help="Output directory for built packages", + ) + p.add_argument( + "--gpg-key-url", + default="", + help=( + "URL of the armored signing key to embed, e.g. https://" + "stable.repo.amd.com/rocm/gpg/packages.gpg. Given whole rather " + "than derived: the key is not under --repo-base-url (packages sit " + "at /core/packages/ and the key beside core/), and where it " + "sits is the publisher's choice, not this tool's. The fetch is " + "https-only and the file must hold only the pinned key. " + "Required for a signed stream unless --gpg-key-file is given" + ), + ) + p.add_argument( + "--verify-repo-url", + action="store_true", + help=( + "Fail the build unless the configured repository is reachable. " + "No-op for a build_id stream, whose build folder is published by " + "the same run that builds the package" + ), + ) + args = p.parse_args(argv) + if is_signed(args.stream) and not args.gpg_key_file and not args.gpg_key_url: + p.error( + f"--gpg-key-url or --gpg-key-file is required for --stream {args.stream}" + ) + if args.gpg_key_url and not _valid_repo_base_url(args.gpg_key_url): + p.error(f"--gpg-key-url is not a valid http(s) URL: {args.gpg_key_url!r}") + build_id_shape = stream_shape(args.stream) == _BUILD_ID + if build_id_shape and not args.repo_sub_folder: + p.error(f"--repo-sub-folder is required for --stream {args.stream}") + if args.repo_sub_folder and not build_id_shape: + p.error(f"--repo-sub-folder is not valid for --stream {args.stream}") + # Validate values that are rendered into repo files, version fields, and + # maintainer metadata so they cannot alter the surrounding syntax. + if not _VERSION_RE.match(args.rocm_version): + p.error(f"--rocm-version has an unexpected format: {args.rocm_version!r}") + if not _VERSION_SUFFIX_RE.match(args.version_suffix): + p.error(f"--version-suffix has an unexpected format: {args.version_suffix!r}") + if not _valid_repo_base_url(args.repo_base_url): + p.error(f"--repo-base-url is not a valid http(s) URL: {args.repo_base_url!r}") + if args.repo_sub_folder and not _SUB_FOLDER_RE.match(args.repo_sub_folder): + p.error(f"--repo-sub-folder has an unexpected format: {args.repo_sub_folder!r}") + # rpmbuild requires an absolute _topdir; normalize so a relative --dest-dir works. + args.dest_dir = args.dest_dir.resolve() + return args + + +def main(argv=None) -> None: + if argv is None: + argv = sys.argv[1:] + if argv and argv[0] == "list-profiles": + run_list_profiles(argv[1:]) + return + args = parse_args(argv) + profile = OS_PROFILES[args.os_profile] + args.dest_dir.mkdir(parents=True, exist_ok=True) + context = build_context(args, profile) + # A build_id repository is published by the same run that builds this + # package, so its build folder does not exist yet and cannot be checked. + if args.verify_repo_url and stream_shape(args.stream) != _BUILD_ID: + verify_repo_url(context["baseurl"], profile["pkg_type"]) + if profile["pkg_type"] == "deb": + build_deb_package(args, profile, context, args.dest_dir) + else: + build_rpm_package(args, profile, context, args.dest_dir) + + +if __name__ == "__main__": + main() diff --git a/build_tools/packaging/linux/get_url_repo_params.py b/build_tools/packaging/linux/get_url_repo_params.py index 99e89bb16d1..f212da996f8 100644 --- a/build_tools/packaging/linux/get_url_repo_params.py +++ b/build_tools/packaging/linux/get_url_repo_params.py @@ -24,7 +24,7 @@ regardless of release_type (if omitted, install runs without GPG verification). Subcommands: get-base-url, get-gpg-url, get-repo-sub-folder, get-repo-url, extract-gfx-arch, -get-container-image. Run with -h for examples. +get-container-image, get-repo-setup-params. Run with -h for examples. Maintenance (when packaging or install URLs change): @@ -36,9 +36,11 @@ """ import argparse +import json import os import re import sys +from datetime import date, datetime, timezone from pathlib import Path from urllib.parse import urlparse @@ -498,6 +500,178 @@ def cmd_repo_url(args: argparse.Namespace) -> int: return 0 +# --- public repository stream (RFC0012) --- + +# Which repo.amd.com stream a build line's amd-repo-setup package configures. +# +# Not s3_buckets.get_release_stream(), which picks the bucket a build publishes +# into and raises on "release", the line that configures stable. A line absent +# here has no public stream, and callers skip the package for it. +# +# Only the stream name and where it is served are recorded here; its shape, +# signing and repo id belong to build_repo_setup_package. The key is at +# /gpg/packages.gpg, outside the packages base, so its URL is built here +# and passed to the builder whole. +_STREAM_ROOTS = { + "release": { + "stream": "stable", + "root": "https://stable.repo.amd.com/rocm", + }, + "prerelease": { + "stream": "rc", + "root": "https://rc.repo.amd.com/rocm", + }, + "nightly": { + "stream": "nightly", + "root": "https://nightly.repo.amd.com/rocm", + }, +} + + +def get_public_repo_release_types() -> tuple[str, ...]: + """Return the build lines that configure a public stream, in table order.""" + return tuple(_STREAM_ROOTS) + + +def get_public_repo_stream(release_type: str) -> str: + """Return the stream a build line's package configures (empty if none). + + Note that the build line and the stream are not the same vocabulary and do + not always share a name: the ``release`` line configures the ``stable`` + stream and ``prerelease`` configures ``rc``. Passed to the builder as + ``--stream``. + """ + entry = _STREAM_ROOTS.get(release_type.strip().lower()) + return entry["stream"] if entry else "" + + +def get_public_repo_stream_root(release_type: str) -> str: + """Return the stream root for a build line (empty if it has no stream).""" + entry = _STREAM_ROOTS.get(release_type.strip().lower()) + return entry["root"] if entry else "" + + +def get_public_repo_base_url(release_type: str) -> str: + """Return the packages base URL for a release line (empty if none).""" + root = get_public_repo_stream_root(release_type) + return f"{root}/core/packages" if root else "" + + +def get_public_repo_gpg_key_url(release_type: str, *, signed: bool) -> str: + """Return the full signing-key URL for a release line (empty if none). + + Passed to the builder as ``--gpg-key-url`` and fetched verbatim. Only a + signed stream publishes a key, so an unsigned one yields an empty value + even though it has a base URL. ``signed`` comes from + ``build_repo_setup_package.is_signed(stream)``, which owns that rule. + """ + root = get_public_repo_stream_root(release_type) + if not root or not signed: + return "" + return f"{root}/gpg/packages.gpg" + + +# --- nightly sub-folder --- + + +# A workflow run id is always numeric. It is interpolated into a value written +# to $GITHUB_OUTPUT, and gha_set_output writes multi-line values with a heredoc +# whose delimiter it does not verify, so an unconstrained value could terminate +# the heredoc early and inject further step outputs. \Z (not $) so a trailing +# newline is not accepted. +_RUN_ID_RE = re.compile(r"^[0-9]+\Z") + + +def nightly_sub_folder(run_id: str, *, today: date | None = None) -> str: + """Return the dated nightly sub-folder ``YYYYMMDD-``. + + ``today`` is injectable for deterministic tests; it defaults to the current + UTC date. + + Raises: + ValueError: If ``run_id`` is not a run of digits. + """ + if not _RUN_ID_RE.match(run_id): + raise ValueError(f"run_id must be numeric, got {run_id!r}") + day = today or datetime.now(timezone.utc).date() + return f"{day:%Y%m%d}-{run_id}" + + +# --- container image map --- + + +def get_container_image_map(os_profiles: list[str]) -> dict[str, str]: + """Map each OS profile to its container image (for a build matrix).""" + return {profile: get_container_image(profile) for profile in os_profiles} + + +# --- amd-repo-setup parameters --- + +# These mirror build_repo_setup_package's private _FLAT / _BUILD_ID. They are compared +# against its public stream_shape(), and a test asserts the two agree. +_SHAPE_FLAT = "flat" +_SHAPE_BUILD_ID = "build_id" + + +def cmd_repo_setup_params(args: argparse.Namespace) -> int: + """Emit every parameter the amd-repo-setup build matrix needs, in one output. + + Outputs: ``os_profiles`` and ``images`` (JSON, the matrix), then the stream + the build line configures and what follows from it -- ``stream``, ``tier``, + ``repo_base_url``, ``gpg_key_url``, ``repo_sub_folder``. A line with no + public stream gets all five empty, which callers read as "no package for + this line"; the matrix is still emitted. + """ + # Deferred: build_repo_setup_package imports jinja2 at module level, and this + # script's other subcommands run in a workflow job that installs no Python + # packages. Only this subcommand needs the builder. + from build_repo_setup_package import ( + REPO_ID_PREFIX, + is_signed, + list_profiles, + repo_id, + stream_shape, + ) + + profiles = list_profiles(args.pkg_type) + outputs = { + "os_profiles": json.dumps(profiles), + "images": json.dumps(get_container_image_map(profiles)), + "stream": "", + "tier": "", + "repo_base_url": "", + "gpg_key_url": "", + "repo_sub_folder": "", + } + stream = get_public_repo_stream(args.release_type) + if stream: + shape = stream_shape(stream) + if shape not in (_SHAPE_FLAT, _SHAPE_BUILD_ID): + print( + f"Error: stream {stream!r} has unknown shape {shape!r}", file=sys.stderr + ) + return 1 + outputs["stream"] = stream + # Names the stream's package file (amd-repo-setup-.): the repo + # file stem without its prefix, so rc is "stablerc". + outputs["tier"] = repo_id(stream).removeprefix(f"{REPO_ID_PREFIX}-") + outputs["repo_base_url"] = get_public_repo_base_url(args.release_type) + outputs["gpg_key_url"] = get_public_repo_gpg_key_url( + args.release_type, signed=is_signed(stream) + ) + # A build_id stream serves each build from its own dated folder; a flat + # one serves one tree at the root, and the builder rejects a sub-folder + # there. The run id is only validated where it is used. + if shape == _SHAPE_BUILD_ID: + try: + outputs["repo_sub_folder"] = nightly_sub_folder(args.run_id) + except ValueError as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + gha_set_output(outputs) + return 0 + + # --- extract-gfx-arch --- @@ -776,6 +950,39 @@ def main(argv: list[str] | None = None) -> int: ) p_img.set_defaults(func=cmd_container_image) + # get-repo-setup-params: every amd-repo-setup build parameter in one output + p_rp = subparsers.add_parser( + "get-repo-setup-params", + help=( + "Print os_profiles=, images=, stream=, tier=, repo_base_url=, " + "gpg_key_url= and repo_sub_folder= for an amd-repo-setup build. The last " + "five are empty for a line with no public repository (e.g. ci/dev)." + ), + ) + p_rp.add_argument( + "--pkg-type", + type=str, + required=True, + choices=["deb", "rpm"], + help="Package type whose OS profiles form the build matrix", + ) + p_rp.add_argument( + "--release-type", + type=str, + required=True, + help="Release line (e.g. prerelease, release, nightly)", + ) + p_rp.add_argument( + "--run-id", + type=str, + required=True, + help=( + "Workflow run id, embedded in the dated build folder of a per-build " + "stream. Ignored (and not validated) for other streams." + ), + ) + p_rp.set_defaults(func=cmd_repo_setup_params) + args = parser.parse_args(argv) return args.func(args) diff --git a/build_tools/packaging/linux/inspect_repo_setup_package.py b/build_tools/packaging/linux/inspect_repo_setup_package.py new file mode 100644 index 00000000000..fd8499098a3 --- /dev/null +++ b/build_tools/packaging/linux/inspect_repo_setup_package.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +# Copyright Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +r"""Verify the payload of a built amd-repo-setup package before it is uploaded. + +This is a build-time check, not an install test. Given the directory a build +wrote its package to, it lists the archive and asserts that it contains the +files the package is supposed to install, at the right paths, owned by root, +and none of the files it must not ship. + +What a package installs depends on its stream as well as its OS profile: the +installed repository filename carries the stream, and the two layout shapes +install different files: + + flat (stable, rc) the repository file and the signing key + build_id (nightly) the repository file only + +Checking a package against the stream it was built for therefore also confirms +that it configures that stream: a package built for another stream installs a +differently named repository file, which is reported as missing. The name +itself (``rc`` installs ``amdrocm-stablerc``, not ``amdrocm-rc``) is pinned by +the unit tests. + +The key paths were renamed away from generic ``rocm`` names as a precaution; +this checks that the package ships the renamed paths and not the previous ones. + +Listing a package needs ``dpkg-deb`` for a deb and ``rpm`` for an rpm, which +building one already requires. + +Usage: + python build_tools/packaging/linux/inspect_repo_setup_package.py \ + --package-dir repo-setup-package-out \ + --os-profile ubuntu2404 \ + --stream nightly +""" + +import argparse +import subprocess +import sys +from pathlib import Path, PurePosixPath + +_THIS_DIR = Path(__file__).resolve().parent + +if str(_THIS_DIR) not in sys.path: + sys.path.insert(0, str(_THIS_DIR)) + +from build_repo_setup_package import ( + DEB_KEYRING_PATH, + OS_PROFILES, + RPM_GPG_KEY_PATH, + STREAMS, + repo_id, + stream_shape, +) + +# The layout shapes this inspector knows how to check. A flat stream is signed +# and ships its key; a per-build stream is unsigned and ships none. Streams come +# from the builder's own table, so one added there is accepted here without +# another edit -- but one whose shape is not handled here would be checked +# against the wrong files, so an unknown shape raises. +# +# These mirror build_repo_setup_package's private _FLAT / _BUILD_ID. They are compared +# against the public stream_shape() return value, and a test asserts the two +# spellings still agree. +_SHAPE_FLAT = "flat" +_SHAPE_BUILD_ID = "build_id" + +# Everything the package installs is owned by root, whatever user builds it. +_EXPECTED_OWNER = "root/root" + +# Key paths used before the rename. A generic rocm name is one another package +# could also claim, and neither dpkg nor rpm will unpack two packages that own +# the same file; see build_repo_setup_package's DEB_KEYRING_PATH for why the rename is +# precautionary. Asserted absent from every package. +_SUPERSEDED_KEY_PATHS = ( + PurePosixPath("/usr/share/keyrings/rocm.gpg"), + PurePosixPath("/etc/pki/rpm-gpg/RPM-GPG-KEY-rocm"), +) + + +def repo_file_path(os_profile: str, stream: str) -> PurePosixPath: + """Return the repository configuration file this profile installs. + + The filename carries the stream, so it cannot be derived from the profile + alone. + """ + profile = OS_PROFILES[os_profile] + stem = repo_id(stream) + if profile["pkg_type"] == "deb": + return PurePosixPath(f"/etc/apt/sources.list.d/{stem}.sources") + return PurePosixPath(profile["rpm_repo_dir"]) / f"{stem}.repo" + + +def key_path(os_profile: str) -> PurePosixPath: + """Return the signing key path this profile installs when signed.""" + if OS_PROFILES[os_profile]["pkg_type"] == "deb": + return PurePosixPath(DEB_KEYRING_PATH) + return PurePosixPath(RPM_GPG_KEY_PATH) + + +def shape_of(stream: str) -> str: + """Return a stream's layout shape, rejecting one this inspector cannot check.""" + shape = stream_shape(stream) + if shape not in (_SHAPE_FLAT, _SHAPE_BUILD_ID): + raise RuntimeError( + f"stream {stream!r} has shape {shape!r}, which this inspector does " + f"not know how to check" + ) + return shape + + +def is_signed_stream(stream: str) -> bool: + """Whether a stream's package ships a signing key. Follows the shape.""" + return shape_of(stream) == _SHAPE_FLAT + + +def expected_paths(os_profile: str, stream: str) -> set[PurePosixPath]: + """Return the paths a build of this profile and stream must install.""" + paths = {repo_file_path(os_profile, stream)} + if is_signed_stream(stream): + paths.add(key_path(os_profile)) + return paths + + +def forbidden_paths(os_profile: str, stream: str) -> set[PurePosixPath]: + """Return the paths a build of this profile and stream must not install.""" + paths = set(_SUPERSEDED_KEY_PATHS) + if not is_signed_stream(stream): + # An unsigned stream ships no key at all: the repository it configures is + # unsigned, so a key here would be inert at best and misleading at worst. + paths.add(key_path(os_profile)) + return paths + + +def parse_deb_contents(text: str) -> list[tuple[PurePosixPath, str]]: + """Parse ``dpkg-deb -c`` output into ``(path, owner)`` pairs. + + Entries are relative to the package root and are reported with a leading + ``./``; they are normalised to absolute paths so both package types compare + against the same values. Directory entries are included, since they carry + ownership too. + """ + entries = [] + for line in text.splitlines(): + line = line.strip() + if not line: + continue + # /