From 5f768dbb1eae42459d62f92b73d62afee01bd6f4 Mon Sep 17 00:00:00 2001 From: Marius Brehler Date: Fri, 18 Sep 2026 03:23:33 +0200 Subject: [PATCH 1/2] Group target tarballs by package owner Individual gfx1250 and gfx1250-strict selections share the gfx1250 archive name while retaining only the requested payloads. Combined selections collect both targets before compression, avoiding separate strict archives or overwritten outputs. Family archive names remain unchanged, and test archives follow the same ownership grouping. The tarball workflow checks out rocm-systems so the ownership adapter can use shared metadata when #8325 lands. Refs #8278 Test plan: - python -m pytest -q -p no:cacheprovider build_tools/packaging/archives/tests build_tools/tests/sdk_targets_test.py Test results: - 15 tests passed, covering individual and combined target selections, both input orders, and test and multiarch archive naming. - Real artifact tarball assembly remains pending CI validation. --- .../workflows/multi_arch_build_tarballs.yml | 6 +++ .../packaging/archives/build_tarballs.py | 17 +++++--- .../archives/tests/build_tarballs_test.py | 41 +++++++++++++++++++ 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/.github/workflows/multi_arch_build_tarballs.yml b/.github/workflows/multi_arch_build_tarballs.yml index a4e03d4cfbb..a61b49461d7 100644 --- a/.github/workflows/multi_arch_build_tarballs.yml +++ b/.github/workflows/multi_arch_build_tarballs.yml @@ -117,12 +117,18 @@ jobs: gh_app_client_id: ${{ secrets.GH_APP_HAULY_CID }} gh_app_private_key: ${{ secrets.GH_APP_HAULY_PRIVATE_KEY }} + - name: Install Git + run: apt-get update && apt-get install -y --no-install-recommends git + - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.repository || github.repository }} ref: ${{ inputs.ref || github.ref_name }} + - name: Checkout shared package ownership metadata + run: git submodule update --init --depth 1 rocm-systems + - name: Install Python requirements run: pip install -r requirements.txt diff --git a/build_tools/packaging/archives/build_tarballs.py b/build_tools/packaging/archives/build_tarballs.py index 990c56ef61b..afacaba6971 100644 --- a/build_tools/packaging/archives/build_tarballs.py +++ b/build_tools/packaging/archives/build_tarballs.py @@ -4,7 +4,7 @@ """Fetch multi-arch build artifacts and package them into per-family tarballs. -For each GPU family in --dist-amdgpu-families, this script: +For each package-owner group in --dist-amdgpu-families, this script: 1. Fetches artifacts (generic + family-specific) using artifact_manager.py 2. Flattens them into a single install-prefix-like layout 3. Compresses the result into a tarball @@ -17,6 +17,10 @@ A shared download cache avoids re-downloading generic (host) artifacts when processing multiple families. +Individual targets sharing a package owner are collected into one tarball +named for that owner. Only the requested targets are fetched; family selections +retain their family names. + By default, generated tarballs exclude test artifacts and fftw3. Pass ``--include-test-tarballs`` to also generate full tarballs, named with a ``-tests`` suffix, that include test artifacts. @@ -41,7 +45,7 @@ and use its run ID. Use ``--platform`` to select which platform's artifacts to fetch (defaults to the current system). -Expected output: one .tar.gz per family in ``--output-dir``, named +Expected output: one .tar.gz per family or shared target owner in ``--output-dir``, named ``therock-dist-{platform}-{family}-{version}.tar.gz``. If KPACK_SPLIT_ARTIFACTS is enabled in the build, also a ``therock-dist-{platform}-multiarch-{version}.tar.gz``. @@ -66,6 +70,9 @@ from zlib_ng import gzip_ng_threaded _BUILD_TOOLS_DIR = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(_BUILD_TOOLS_DIR)) + +from _therock_utils.sdk_targets import group_package_targets DEFAULT_EXCLUDED_ARTIFACTS: list[str] = ["fftw3"] DEFAULT_EXCLUDED_COMPONENTS: list[str] = ["test"] @@ -389,11 +396,11 @@ def main(argv: list[str] | None = None) -> None: # family and multiarch staging trees. family_dirs = [] compress_tasks: list[CompressionTask] = [] - for family in families: + for family, members in group_package_targets(families).items(): flatten_dir = work_dir / family fetch_and_flatten( run_id=args.run_id, - amdgpu_families=[family], + amdgpu_families=members, platform=args.platform, output_dir=flatten_dir, download_cache_dir=download_cache_dir, @@ -417,7 +424,7 @@ def main(argv: list[str] | None = None) -> None: tests_dir = work_dir / "tests" / family fetch_and_flatten( run_id=args.run_id, - amdgpu_families=[family], + amdgpu_families=members, platform=args.platform, output_dir=tests_dir, download_cache_dir=download_cache_dir, diff --git a/build_tools/packaging/archives/tests/build_tarballs_test.py b/build_tools/packaging/archives/tests/build_tarballs_test.py index c129cf72553..0c746f43d02 100644 --- a/build_tools/packaging/archives/tests/build_tarballs_test.py +++ b/build_tools/packaging/archives/tests/build_tarballs_test.py @@ -169,6 +169,47 @@ def _run_main_with_mocks( main(argv) return MainMocks(fetch_mock, compress_mock, kpack_mock) + def test_shared_owner_preserves_selected_targets(self) -> None: + for selected in ( + ["gfx1250"], + ["gfx1250-strict"], + ["gfx1250", "gfx1250-strict"], + ["gfx1250-strict", "gfx1250"], + ): + with self.subTest( + selected=selected + ), tempfile.TemporaryDirectory() as tmpdir: + families = [*selected, "gfx1151"] + fetch, compress, _ = self._run_main_with_mocks( + [ + "--run-id=123", + f"--dist-amdgpu-families={';'.join(families)}", + "--platform=linux", + "--package-version=7.13.0", + f"--output-dir={tmpdir}", + "--include-test-tarballs", + ], + kpack_split=True, + ) + self.assertEqual( + [call.kwargs["amdgpu_families"] for call in fetch.call_args_list], + [selected, selected, ["gfx1151"], ["gfx1151"], families, families], + ) + self.assertCountEqual( + [ + call.kwargs["tarball_path"].name + for call in compress.call_args_list + ], + [ + "therock-dist-linux-gfx1250-7.13.0.tar.gz", + "therock-dist-linux-gfx1250-tests-7.13.0.tar.gz", + "therock-dist-linux-gfx1151-7.13.0.tar.gz", + "therock-dist-linux-gfx1151-tests-7.13.0.tar.gz", + "therock-dist-linux-multiarch-7.13.0.tar.gz", + "therock-dist-linux-multiarch-tests-7.13.0.tar.gz", + ], + ) + def test_default_builds_tarballs_without_tests_only(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: output_dir = Path(tmpdir) / "tarballs" From a8febe04490d98c3fbf7ee2e3f1af640f3901960 Mon Sep 17 00:00:00 2001 From: Marius Brehler Date: Fri, 18 Sep 2026 19:56:55 +0200 Subject: [PATCH 2/2] Add TODO note --- .github/workflows/multi_arch_build_tarballs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/multi_arch_build_tarballs.yml b/.github/workflows/multi_arch_build_tarballs.yml index a61b49461d7..8ef3c9166dd 100644 --- a/.github/workflows/multi_arch_build_tarballs.yml +++ b/.github/workflows/multi_arch_build_tarballs.yml @@ -126,6 +126,7 @@ jobs: repository: ${{ inputs.repository || github.repository }} ref: ${{ inputs.ref || github.ref_name }} + # TODO(8354): pip install rocm-bootstrap instead - name: Checkout shared package ownership metadata run: git submodule update --init --depth 1 rocm-systems