diff --git a/.github/workflows/multi_arch_build_tarballs.yml b/.github/workflows/multi_arch_build_tarballs.yml index a4e03d4cfbb..8ef3c9166dd 100644 --- a/.github/workflows/multi_arch_build_tarballs.yml +++ b/.github/workflows/multi_arch_build_tarballs.yml @@ -117,12 +117,19 @@ jobs: gh_app_client_id: ${{ secrets.GH_APP_HAULY_CID }} gh_app_private_key: ${{ secrets.GH_APP_HAULY_PRIVATE_KEY }} + - name: Install Git + run: apt-get update && apt-get install -y --no-install-recommends git + - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.repository || github.repository }} ref: ${{ inputs.ref || github.ref_name }} + # TODO(8354): pip install rocm-bootstrap instead + - name: Checkout shared package ownership metadata + run: git submodule update --init --depth 1 rocm-systems + - name: Install Python requirements run: pip install -r requirements.txt diff --git a/build_tools/packaging/archives/build_tarballs.py b/build_tools/packaging/archives/build_tarballs.py index 990c56ef61b..afacaba6971 100644 --- a/build_tools/packaging/archives/build_tarballs.py +++ b/build_tools/packaging/archives/build_tarballs.py @@ -4,7 +4,7 @@ """Fetch multi-arch build artifacts and package them into per-family tarballs. -For each GPU family in --dist-amdgpu-families, this script: +For each package-owner group in --dist-amdgpu-families, this script: 1. Fetches artifacts (generic + family-specific) using artifact_manager.py 2. Flattens them into a single install-prefix-like layout 3. Compresses the result into a tarball @@ -17,6 +17,10 @@ A shared download cache avoids re-downloading generic (host) artifacts when processing multiple families. +Individual targets sharing a package owner are collected into one tarball +named for that owner. Only the requested targets are fetched; family selections +retain their family names. + By default, generated tarballs exclude test artifacts and fftw3. Pass ``--include-test-tarballs`` to also generate full tarballs, named with a ``-tests`` suffix, that include test artifacts. @@ -41,7 +45,7 @@ and use its run ID. Use ``--platform`` to select which platform's artifacts to fetch (defaults to the current system). -Expected output: one .tar.gz per family in ``--output-dir``, named +Expected output: one .tar.gz per family or shared target owner in ``--output-dir``, named ``therock-dist-{platform}-{family}-{version}.tar.gz``. If KPACK_SPLIT_ARTIFACTS is enabled in the build, also a ``therock-dist-{platform}-multiarch-{version}.tar.gz``. @@ -66,6 +70,9 @@ from zlib_ng import gzip_ng_threaded _BUILD_TOOLS_DIR = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(_BUILD_TOOLS_DIR)) + +from _therock_utils.sdk_targets import group_package_targets DEFAULT_EXCLUDED_ARTIFACTS: list[str] = ["fftw3"] DEFAULT_EXCLUDED_COMPONENTS: list[str] = ["test"] @@ -389,11 +396,11 @@ def main(argv: list[str] | None = None) -> None: # family and multiarch staging trees. family_dirs = [] compress_tasks: list[CompressionTask] = [] - for family in families: + for family, members in group_package_targets(families).items(): flatten_dir = work_dir / family fetch_and_flatten( run_id=args.run_id, - amdgpu_families=[family], + amdgpu_families=members, platform=args.platform, output_dir=flatten_dir, download_cache_dir=download_cache_dir, @@ -417,7 +424,7 @@ def main(argv: list[str] | None = None) -> None: tests_dir = work_dir / "tests" / family fetch_and_flatten( run_id=args.run_id, - amdgpu_families=[family], + amdgpu_families=members, platform=args.platform, output_dir=tests_dir, download_cache_dir=download_cache_dir, diff --git a/build_tools/packaging/archives/tests/build_tarballs_test.py b/build_tools/packaging/archives/tests/build_tarballs_test.py index c129cf72553..0c746f43d02 100644 --- a/build_tools/packaging/archives/tests/build_tarballs_test.py +++ b/build_tools/packaging/archives/tests/build_tarballs_test.py @@ -169,6 +169,47 @@ def _run_main_with_mocks( main(argv) return MainMocks(fetch_mock, compress_mock, kpack_mock) + def test_shared_owner_preserves_selected_targets(self) -> None: + for selected in ( + ["gfx1250"], + ["gfx1250-strict"], + ["gfx1250", "gfx1250-strict"], + ["gfx1250-strict", "gfx1250"], + ): + with self.subTest( + selected=selected + ), tempfile.TemporaryDirectory() as tmpdir: + families = [*selected, "gfx1151"] + fetch, compress, _ = self._run_main_with_mocks( + [ + "--run-id=123", + f"--dist-amdgpu-families={';'.join(families)}", + "--platform=linux", + "--package-version=7.13.0", + f"--output-dir={tmpdir}", + "--include-test-tarballs", + ], + kpack_split=True, + ) + self.assertEqual( + [call.kwargs["amdgpu_families"] for call in fetch.call_args_list], + [selected, selected, ["gfx1151"], ["gfx1151"], families, families], + ) + self.assertCountEqual( + [ + call.kwargs["tarball_path"].name + for call in compress.call_args_list + ], + [ + "therock-dist-linux-gfx1250-7.13.0.tar.gz", + "therock-dist-linux-gfx1250-tests-7.13.0.tar.gz", + "therock-dist-linux-gfx1151-7.13.0.tar.gz", + "therock-dist-linux-gfx1151-tests-7.13.0.tar.gz", + "therock-dist-linux-multiarch-7.13.0.tar.gz", + "therock-dist-linux-multiarch-tests-7.13.0.tar.gz", + ], + ) + def test_default_builds_tarballs_without_tests_only(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: output_dir = Path(tmpdir) / "tarballs"