From 5f94f4de57648213ce8919533aef063a5509d0c8 Mon Sep 17 00:00:00 2001 From: Harrison King Saturley-Hall Date: Mon, 14 Sep 2026 17:12:54 -0400 Subject: [PATCH 1/3] fix(packaging): repair Linux wheels for manylinux 2.28 Signed-off-by: Harrison King Saturley-Hall --- .github/workflows/ci.yml | 23 +++-- .github/workflows/fpe-support-matrix.yml | 5 +- python/aisimulate/docker/Dockerfile | 26 ++++-- scripts/build_manylinux_wheel.py | 102 +++++++++++++++++++++++ scripts/build_release_artifacts.py | 77 +++++++---------- tests/test_ci_workflow_contracts.py | 59 +++++++++++++ 6 files changed, 231 insertions(+), 61 deletions(-) create mode 100644 scripts/build_manylinux_wheel.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c71077ea5..714dbc8d1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -569,11 +569,17 @@ jobs: strategy: fail-fast: false matrix: - arch: [amd64, arm64] + include: + - arch: amd64 + # quay.io/pypa/manylinux_2_28_x86_64:2026.08.24-1 + container_image: quay.io/pypa/manylinux_2_28_x86_64@sha256:4dc41da7df20400310c80d162a2fe2d2c2f3d9734d8dec20f6b9843711618deb + - arch: arm64 + # quay.io/pypa/manylinux_2_28_aarch64:2026.08.24-1 + container_image: quay.io/pypa/manylinux_2_28_aarch64@sha256:29a6ceb78de5b00494e6e7456a72782a4cb54238de102e7491d15fe47789b4d1 runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 timeout-minutes: 10 container: - image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} + image: ${{ matrix.container_image }} env: HOME: /__w/aisimulate/aisimulate USER: runner @@ -587,8 +593,7 @@ jobs: python-version: "3.12" - run: python -m pip install 'maturin>=1.12,<2' - name: Build the application test wheel once - run: python -m maturin build --release --out ../../application-test-dist - working-directory: python/aisimulate + run: python scripts/build_manylinux_wheel.py --output-dir application-test-dist - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: application-test-wheel-${{ matrix.arch }} @@ -798,10 +803,16 @@ jobs: strategy: fail-fast: false matrix: - arch: [amd64, arm64] + include: + - arch: amd64 + # quay.io/pypa/manylinux_2_28_x86_64:2026.08.24-1 + container_image: quay.io/pypa/manylinux_2_28_x86_64@sha256:4dc41da7df20400310c80d162a2fe2d2c2f3d9734d8dec20f6b9843711618deb + - arch: arm64 + # quay.io/pypa/manylinux_2_28_aarch64:2026.08.24-1 + container_image: quay.io/pypa/manylinux_2_28_aarch64@sha256:29a6ceb78de5b00494e6e7456a72782a4cb54238de102e7491d15fe47789b4d1 runs-on: prod-aisimulate-default-${{ matrix.arch }}-v1 container: - image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} + image: ${{ matrix.container_image }} env: HOME: /__w/aisimulate/aisimulate USER: runner diff --git a/.github/workflows/fpe-support-matrix.yml b/.github/workflows/fpe-support-matrix.yml index bd7dfabcbd..12af589a10 100644 --- a/.github/workflows/fpe-support-matrix.yml +++ b/.github/workflows/fpe-support-matrix.yml @@ -29,7 +29,8 @@ jobs: runs-on: prod-aisimulate-default-amd64-v1 timeout-minutes: 30 container: - image: ${{ vars.CI_JOB_CONTAINER_IMAGE }} + # quay.io/pypa/manylinux_2_28_x86_64:2026.08.24-1 + image: quay.io/pypa/manylinux_2_28_x86_64@sha256:4dc41da7df20400310c80d162a2fe2d2c2f3d9734d8dec20f6b9843711618deb env: HOME: /__w/aisimulate/aisimulate USER: runner @@ -52,7 +53,7 @@ jobs: if: inputs.wheel_artifact == '' run: | python -m pip install 'maturin>=1.12,<2' - (cd python/aisimulate && python -m maturin build --release --out ../../fpe-wheel) + python scripts/build_manylinux_wheel.py --output-dir fpe-wheel - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 if: inputs.wheel_artifact != '' with: diff --git a/python/aisimulate/docker/Dockerfile b/python/aisimulate/docker/Dockerfile index 5e528d70a9..630b2f8d7b 100644 --- a/python/aisimulate/docker/Dockerfile +++ b/python/aisimulate/docker/Dockerfile @@ -22,7 +22,7 @@ ENV MPLCONFIGDIR=/tmp/matplotlib # One `aisimulate` wheel owns the application, compatibility namespaces, # SDK/data payload, Replay runtime, and unified Rust extension. It is built in # the PyPA manylinux_2_28 image so the native PyO3 `.so` links glibc 2.28 and -# maturin's auditwheel stamps the portable +# standalone auditwheel repairs and stamps the portable # `manylinux_2_28_` tag. We add `uv` and a rustup toolchain so cargo can # fetch crates over TLS. # @@ -70,12 +70,26 @@ RUN set -e; \ echo "Run 'git lfs install && git lfs pull' in the source tree before building." >&2; \ exit 1; \ fi -# `uv build` runs maturin's PEP517 backend, which defaults to `--compatibility -# off`; MATURIN_PEP517_ARGS overrides it to stamp the manylinux_2_28 tag (maturin -# appends it last, so it wins). `python` isn't on PATH here -> verify via UV_PYTHON. +# Match Dynamo's release-wheel boundary: Maturin emits an unaudited wheel, +# then the standalone auditwheel tool repairs and retags that exact wheel. +# Compiling in this manylinux_2_28 image ensures repair never tries to claim a +# policy older than the glibc symbols in the native extension. RUN cd /workspace/python/aisimulate && \ - MATURIN_PEP517_ARGS="--compatibility manylinux_2_28" \ - uv build --wheel --out-dir /workspace/dist && \ + mkdir -p /workspace/unrepaired-dist && \ + MATURIN_PEP517_ARGS="--auditwheel skip" \ + uv build --wheel --out-dir /workspace/unrepaired-dist && \ + set -- /workspace/unrepaired-dist/aisimulate-*.whl && \ + [ "$#" -eq 1 ] && [ -f "$1" ] && \ + case "$(uname -m)" in \ + x86_64) manylinux_policy=manylinux_2_28_x86_64 ;; \ + aarch64|arm64) manylinux_policy=manylinux_2_28_aarch64 ;; \ + *) echo "unsupported wheel architecture: $(uname -m)" >&2; exit 1 ;; \ + esac && \ + auditwheel repair \ + --plat "${manylinux_policy}" \ + --wheel-dir /workspace/dist \ + "$1" && \ + auditwheel show /workspace/dist/aisimulate-*.whl && \ "${UV_PYTHON}" tools/verify_release_wheels.py /workspace/dist # Exercise the one-wheel boundary outside the checkout, including all diff --git a/scripts/build_manylinux_wheel.py b/scripts/build_manylinux_wheel.py new file mode 100644 index 0000000000..32598ffc76 --- /dev/null +++ b/scripts/build_manylinux_wheel.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build one Linux wheel and repair it for the manylinux 2.28 policy.""" + +from __future__ import annotations + +import argparse +import platform +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +PYTHON_PROJECT = ROOT / "python" / "aisimulate" +PLATFORM_BY_MACHINE = { + "aarch64": "manylinux_2_28_aarch64", + "arm64": "manylinux_2_28_aarch64", + "amd64": "manylinux_2_28_x86_64", + "x86_64": "manylinux_2_28_x86_64", +} + + +def manylinux_platform(machine: str | None = None) -> str: + """Return the required manylinux policy for the native architecture.""" + machine = (machine or platform.machine()).lower() + try: + return PLATFORM_BY_MACHINE[machine] + except KeyError as error: + supported = ", ".join(sorted(PLATFORM_BY_MACHINE)) + raise SystemExit(f"unsupported wheel architecture {machine!r}; expected one of: {supported}") from error + + +def _run(*command: str, cwd: Path = ROOT) -> None: + subprocess.run(command, cwd=cwd, check=True) + + +def _single_wheel(directory: Path, description: str) -> Path: + wheels = sorted(directory.glob("aisimulate-*.whl")) + if len(wheels) != 1: + names = [path.name for path in wheels] + raise SystemExit(f"expected exactly one {description}, found {names}") + return wheels[0] + + +def build(output: Path) -> Path: + """Build an unrepaired wheel, then make auditwheel produce the final wheel.""" + if not sys.platform.startswith("linux"): + raise SystemExit("the manylinux wheel builder must run on Linux") + if shutil.which("auditwheel") is None: + raise SystemExit("auditwheel is required; run inside the pinned manylinux image") + + policy = manylinux_platform() + output.mkdir(parents=True, exist_ok=True) + if any(output.iterdir()): + raise SystemExit(f"output directory must be empty: {output}") + + with tempfile.TemporaryDirectory(prefix="aisimulate-unrepaired-wheel-") as temp: + raw_output = Path(temp) + _run( + sys.executable, + "-m", + "maturin", + "build", + "--release", + "--auditwheel", + "skip", + "--out", + str(raw_output), + cwd=PYTHON_PROJECT, + ) + raw_wheel = _single_wheel(raw_output, "unrepaired AISimulate wheel") + _run( + "auditwheel", + "repair", + "--plat", + policy, + "--wheel-dir", + str(output), + str(raw_wheel), + ) + + repaired_wheel = _single_wheel(output, "repaired AISimulate wheel") + if policy not in repaired_wheel.name: + raise SystemExit(f"repaired wheel does not carry required tag {policy}: {repaired_wheel.name}") + _run("auditwheel", "show", str(repaired_wheel)) + print(f"built {repaired_wheel}") + return repaired_wheel + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output-dir", type=Path, default=ROOT / "dist") + args = parser.parse_args() + build(args.output_dir.resolve()) + + +if __name__ == "__main__": + main() diff --git a/scripts/build_release_artifacts.py b/scripts/build_release_artifacts.py index 3bfbdd5bb9..3aa538dc24 100755 --- a/scripts/build_release_artifacts.py +++ b/scripts/build_release_artifacts.py @@ -44,13 +44,10 @@ def _is_source_manifest(path: Path) -> bool: def check_manifests() -> tuple[str, str]: """Validate the manifest set; return (wheel version, crate version).""" pyprojects = { - path: str(_toml(path)["project"]["name"]) - for path in ROOT.rglob("pyproject.toml") - if _is_source_manifest(path) + path: str(_toml(path)["project"]["name"]) for path in ROOT.rglob("pyproject.toml") if _is_source_manifest(path) } assert pyprojects == EXPECTED_PYTHON_PROJECTS, ( - "publishable Python manifest set changed:\n" - f"expected={EXPECTED_PYTHON_PROJECTS}\nactual={pyprojects}" + f"publishable Python manifest set changed:\nexpected={EXPECTED_PYTHON_PROJECTS}\nactual={pyprojects}" ) publishable_crates: dict[Path, str] = {} @@ -71,32 +68,20 @@ def check_manifests() -> tuple[str, str]: py_version = str(app["version"]) crate_version = str(crate["version"]) dev_suffix = py_version.removeprefix(VERSION) - assert py_version.startswith(VERSION) and ( - dev_suffix == "" or DEV_SUFFIX_RE.fullmatch(dev_suffix) - ), f"wheel version must be {VERSION} or {VERSION}.devYYYYMMDD, got {py_version}" - expected_crate_version = ( - f"{VERSION}-dev.{dev_suffix[len('.dev'):]}" if dev_suffix else VERSION + assert py_version.startswith(VERSION) and (dev_suffix == "" or DEV_SUFFIX_RE.fullmatch(dev_suffix)), ( + f"wheel version must be {VERSION} or {VERSION}.devYYYYMMDD, got {py_version}" ) + expected_crate_version = f"{VERSION}-dev.{dev_suffix[len('.dev') :]}" if dev_suffix else VERSION assert crate_version == expected_crate_version, ( - f"crate version {crate_version} does not match wheel version " - f"{py_version} (expected {expected_crate_version})" + f"crate version {crate_version} does not match wheel version {py_version} (expected {expected_crate_version})" ) optional_dependencies = app.get("optional-dependencies", {}) dependencies = [ *app["dependencies"], - *( - dependency - for group in optional_dependencies.values() - for dependency in group - ), + *(dependency for group in optional_dependencies.values() for dependency in group), ] - assert not any( - str(dep).lower().startswith(("aisimulate-core", "aiconfigurator")) - for dep in dependencies - ) - assert not any( - str(dep).lower().startswith(("dynamo", "ai-dynamo")) for dep in dependencies - ) + assert not any(str(dep).lower().startswith(("aisimulate-core", "aiconfigurator")) for dep in dependencies) + assert not any(str(dep).lower().startswith(("dynamo", "ai-dynamo")) for dep in dependencies) assert app["scripts"] == { "aiconfigurator": "aiconfigurator.main:main", "aisimulate": "aisimulate.main:main", @@ -117,16 +102,24 @@ def build(output: Path, py_version: str, crate_version: str) -> None: if any(output.iterdir()): raise SystemExit(f"output directory must be empty: {output}") - _run( - sys.executable, - "-m", - "maturin", - "build", - "--release", - "--out", - str(output), - cwd=ROOT / "python" / "aisimulate", - ) + if sys.platform.startswith("linux"): + _run( + sys.executable, + str(ROOT / "scripts" / "build_manylinux_wheel.py"), + "--output-dir", + str(output), + ) + else: + _run( + sys.executable, + "-m", + "maturin", + "build", + "--release", + "--out", + str(output), + cwd=ROOT / "python" / "aisimulate", + ) with tempfile.TemporaryDirectory(prefix="aisimulate-crate-") as temp: target = Path(temp) / "target" @@ -151,25 +144,15 @@ def build(output: Path, py_version: str, crate_version: str) -> None: def verify_output(output: Path, py_version: str, crate_version: str) -> None: names = sorted(path.name for path in output.iterdir() if path.is_file()) expected_crate = f"aisimulate-core-{crate_version}.crate" - app_wheels = [ - name - for name in names - if name.startswith(f"aisimulate-{py_version}-") and name.endswith(".whl") - ] + app_wheels = [name for name in names if name.startswith(f"aisimulate-{py_version}-") and name.endswith(".whl")] assert len(names) == 2, f"expected exactly two artifacts, got {names}" assert len(app_wheels) == 1, f"missing or duplicate aisimulate wheel: {names}" assert expected_crate in names, f"missing {expected_crate}: {names}" wheel_path = output / app_wheels[0] with zipfile.ZipFile(wheel_path) as wheel: for legal_file in LEGAL_FILES: - matches = [ - name - for name in wheel.namelist() - if name.endswith(f".dist-info/licenses/{legal_file}") - ] - assert len(matches) == 1, ( - f"expected one packaged {legal_file}, found {matches}" - ) + matches = [name for name in wheel.namelist() if name.endswith(f".dist-info/licenses/{legal_file}")] + assert len(matches) == 1, f"expected one packaged {legal_file}, found {matches}" assert wheel.read(matches[0]) == (ROOT / legal_file).read_bytes(), ( f"packaged {legal_file} differs from the root original" ) diff --git a/tests/test_ci_workflow_contracts.py b/tests/test_ci_workflow_contracts.py index 4cd93342d3..ff712aa76a 100644 --- a/tests/test_ci_workflow_contracts.py +++ b/tests/test_ci_workflow_contracts.py @@ -17,6 +17,7 @@ import pytest import yaml +from scripts.build_manylinux_wheel import manylinux_platform from scripts.check_application_test_inventory import Inventory, assignment from scripts.select_full_ci import COMPONENTS, select_components @@ -380,6 +381,64 @@ def test_platform_wheel_build_and_verifiers_cover_collector_payload() -> None: assert 'importlib.import_module("collector.fpm_forward")' in installed_verifier +def test_linux_release_wheels_are_repaired_for_manylinux_2_28() -> None: + expected_images = { + "amd64": "quay.io/pypa/manylinux_2_28_x86_64@sha256:", + "arm64": "quay.io/pypa/manylinux_2_28_aarch64@sha256:", + } + full_ci = _workflow("ci.yml")["jobs"] + for job_name in ("application-test-wheel", "release-artifact-contract"): + job = full_ci[job_name] + assert job["container"]["image"] == "${{ matrix.container_image }}" + images = {entry["arch"]: entry["container_image"] for entry in job["strategy"]["matrix"]["include"]} + assert images.keys() == expected_images.keys() + for arch, prefix in expected_images.items(): + assert images[arch].startswith(prefix) + assert len(images[arch].removeprefix(prefix)) == 64 + + application_commands = _run_commands(full_ci["application-test-wheel"]) + assert "scripts/build_manylinux_wheel.py" in application_commands + assert "maturin build" not in application_commands + + fpe_prepare = _workflow("fpe-support-matrix.yml")["jobs"]["prepare-wheel"] + assert fpe_prepare["container"]["image"].startswith(expected_images["amd64"]) + assert "scripts/build_manylinux_wheel.py" in _run_commands(fpe_prepare) + + release_builder = (REPOSITORY_ROOT / "scripts" / "build_release_artifacts.py").read_text() + assert 'sys.platform.startswith("linux")' in release_builder + assert '"build_manylinux_wheel.py"' in release_builder + + manylinux_builder = (REPOSITORY_ROOT / "scripts" / "build_manylinux_wheel.py").read_text() + assert '"--auditwheel",\n "skip"' in manylinux_builder + assert '"auditwheel",\n "repair"' in manylinux_builder + assert '"--plat",\n policy' in manylinux_builder + assert '_run("auditwheel", "show", str(repaired_wheel))' in manylinux_builder + + dockerfile = (REPOSITORY_ROOT / "python" / "aisimulate" / "docker" / "Dockerfile").read_text() + assert 'MATURIN_PEP517_ARGS="--auditwheel skip"' in dockerfile + assert "auditwheel repair" in dockerfile + assert "auditwheel show /workspace/dist/aisimulate-*.whl" in dockerfile + assert "--compatibility manylinux_2_28" not in dockerfile + + +@pytest.mark.parametrize( + ("machine", "expected"), + [ + ("x86_64", "manylinux_2_28_x86_64"), + ("amd64", "manylinux_2_28_x86_64"), + ("aarch64", "manylinux_2_28_aarch64"), + ("arm64", "manylinux_2_28_aarch64"), + ], +) +def test_manylinux_platform_maps_native_architectures(machine: str, expected: str) -> None: + assert manylinux_platform(machine) == expected + + +def test_manylinux_platform_rejects_unknown_architecture() -> None: + with pytest.raises(SystemExit, match="unsupported wheel architecture"): + manylinux_platform("riscv64") + + def test_prediction_gate_owns_report_dependencies_and_pre_harness_fallback() -> None: jobs = _workflow("prediction-regression-gate.yml")["jobs"] report_commands = _run_commands(jobs["report"]) From c57caa6a286170bc5b7bb64568717be537ede14e Mon Sep 17 00:00:00 2001 From: "Jason Zhou (Engrg-Hardware 1)" Date: Mon, 14 Sep 2026 14:34:59 -0700 Subject: [PATCH 2/3] fix(ci): use bundled Python in manylinux jobs Signed-off-by: Jason Zhou (Engrg-Hardware 1) --- .../setup-manylinux-python-rust/action.yml | 33 ++++++++ .github/workflows/ci.yml | 8 +- .github/workflows/fpe-support-matrix.yml | 4 +- tests/test_ci_workflow_contracts.py | 77 +++++++++++++++++-- 4 files changed, 108 insertions(+), 14 deletions(-) create mode 100644 .github/actions/setup-manylinux-python-rust/action.yml diff --git a/.github/actions/setup-manylinux-python-rust/action.yml b/.github/actions/setup-manylinux-python-rust/action.yml new file mode 100644 index 0000000000..58dbb5f07a --- /dev/null +++ b/.github/actions/setup-manylinux-python-rust/action.yml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Set up manylinux Python and Rust +description: Select bundled CPython 3.12 and prepare Rust in the pinned manylinux images + +runs: + using: composite + steps: + - name: Select bundled Python and verify image tools + shell: bash + env: + MANYLINUX_PYTHON_ROOT: /opt/python/cp312-cp312 + run: | + set -euo pipefail + "${MANYLINUX_PYTHON_ROOT}/bin/python" -c \ + 'import sys; assert sys.version_info[:2] == (3, 12), sys.version' + for tool in cc c++ make auditwheel patchelf; do + command -v "${tool}" >/dev/null || { + echo "::error::manylinux image is missing required tool: ${tool}" + exit 1 + } + done + echo "${MANYLINUX_PYTHON_ROOT}/bin" >> "${GITHUB_PATH}" + { + echo "PYO3_PYTHON=${MANYLINUX_PYTHON_ROOT}/bin/python" + echo "LIBRARY_PATH=${MANYLINUX_PYTHON_ROOT}/lib" + # HOME is the checkout; keep toolchain manifests outside the source + # tree inspected by build_release_artifacts.py and cargo package. + echo "CARGO_HOME=${RUNNER_TEMP}/cargo" + echo "RUSTUP_HOME=${RUNNER_TEMP}/rustup" + } >> "${GITHUB_ENV}" + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 714dbc8d1f..ff2016d626 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -588,9 +588,7 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - uses: ./.github/actions/setup-python-rust - with: - python-version: "3.12" + - uses: ./.github/actions/setup-manylinux-python-rust - run: python -m pip install 'maturin>=1.12,<2' - name: Build the application test wheel once run: python scripts/build_manylinux_wheel.py --output-dir application-test-dist @@ -820,9 +818,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: ./.github/actions/setup-python-rust - with: - python-version: "3.12" + - uses: ./.github/actions/setup-manylinux-python-rust - run: python -m pip install build maturin - run: python scripts/build_release_artifacts.py --check-only - run: python scripts/build_release_artifacts.py --output-dir dist diff --git a/.github/workflows/fpe-support-matrix.yml b/.github/workflows/fpe-support-matrix.yml index 12af589a10..db8fe8ab6a 100644 --- a/.github/workflows/fpe-support-matrix.yml +++ b/.github/workflows/fpe-support-matrix.yml @@ -45,10 +45,8 @@ jobs: env: EXPECTED_SHA: ${{ inputs.expected_sha }} run: test -n "${EXPECTED_SHA}" && test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}" - - uses: ./.github/actions/setup-python-rust + - uses: ./.github/actions/setup-manylinux-python-rust if: inputs.wheel_artifact == '' - with: - python-version: "3.12" - name: Build one wheel for standalone qualification if: inputs.wheel_artifact == '' run: | diff --git a/tests/test_ci_workflow_contracts.py b/tests/test_ci_workflow_contracts.py index ff712aa76a..322f88d172 100644 --- a/tests/test_ci_workflow_contracts.py +++ b/tests/test_ci_workflow_contracts.py @@ -11,6 +11,7 @@ import re import shlex import subprocess +import sys from pathlib import Path from types import SimpleNamespace @@ -761,17 +762,83 @@ def test_shared_python_rust_setup_is_used_by_same_revision_jobs() -> None: "rust", "rust-feature-modes", "public-api-rust", - "application-test-wheel", "python-compatibility", "engine-golden-regression", - "release-artifact-contract", ): assert any(step.get("uses") == action_path for step in full_ci[job_name]["steps"]) assert any(step.get("uses") == action_path for step in _workflow("collector-check.yml")["jobs"]["check"]["steps"]) - assert any( - step.get("uses") == action_path - for step in _workflow("fpe-support-matrix.yml")["jobs"]["prepare-wheel"]["steps"] + + +def test_manylinux_jobs_select_bundled_python_without_setup_python() -> None: + action_path = "./.github/actions/setup-manylinux-python-rust" + for workflow_name, job_name in ( + ("ci.yml", "application-test-wheel"), + ("ci.yml", "release-artifact-contract"), + ("fpe-support-matrix.yml", "prepare-wheel"), + ): + steps = _workflow(workflow_name)["jobs"][job_name]["steps"] + assert any(step.get("uses") == action_path for step in steps) + assert not any( + step.get("uses", "").startswith(("actions/setup-python@", "./.github/actions/setup-python-rust")) + for step in steps + ) + action = yaml.safe_load((ACTION_ROOT / "setup-manylinux-python-rust" / "action.yml").read_text()) + assert action["runs"]["steps"][0]["env"]["MANYLINUX_PYTHON_ROOT"] == "/opt/python/cp312-cp312" + assert not any(step.get("uses", "").startswith("actions/setup-python@") for step in action["runs"]["steps"]) + + +@pytest.mark.parametrize("missing", [None, "python", "cc", "c++", "make", "auditwheel", "patchelf"]) +def test_manylinux_bootstrap_exports_working_python_or_fails_before_build(tmp_path: Path, missing: str | None) -> None: + action = yaml.safe_load((ACTION_ROOT / "setup-manylinux-python-rust" / "action.yml").read_text()) + bootstrap = action["runs"]["steps"][0]["run"] + python_root = tmp_path / "bundled-python" + binaries = python_root / "bin" + binaries.mkdir(parents=True) + if missing != "python": + (binaries / "python").symlink_to(sys.executable) + tools = tmp_path / "tools" + tools.mkdir() + for name in ("cc", "c++", "make", "auditwheel", "patchelf"): + if name != missing: + (tools / name).symlink_to("/usr/bin/true") + env_file = tmp_path / "github-env" + path_file = tmp_path / "github-path" + env_file.touch() + path_file.touch() + runner_temp = tmp_path / "runner-temp" + result = subprocess.run( + ["/bin/bash", "-c", bootstrap], + env={ + **os.environ, + "PATH": str(tools), + "MANYLINUX_PYTHON_ROOT": str(python_root), + "RUNNER_TEMP": str(runner_temp), + "GITHUB_ENV": str(env_file), + "GITHUB_PATH": str(path_file), + }, + capture_output=True, + text=True, + timeout=10, + ) + if missing: + assert result.returncode != 0 + assert env_file.read_text() == path_file.read_text() == "" + return + assert result.returncode == 0, result.stdout + result.stderr + exported = dict(line.split("=", 1) for line in env_file.read_text().splitlines()) + assert exported == { + "PYO3_PYTHON": str(binaries / "python"), + "LIBRARY_PATH": str(python_root / "lib"), + "CARGO_HOME": str(runner_temp / "cargo"), + "RUSTUP_HOME": str(runner_temp / "rustup"), + } + assert path_file.read_text().splitlines() == [str(binaries)] + subprocess.run( + ["python", "-c", "import sys; assert sys.version_info[:2] == (3, 12)"], + env={**os.environ, **exported, "PATH": path_file.read_text().strip()}, + check=True, + timeout=10, ) From 3e82de87e3eba987fb4e68ebef90c87e23b439ff Mon Sep 17 00:00:00 2001 From: "Jason Zhou (Engrg-Hardware 1)" Date: Mon, 14 Sep 2026 14:48:37 -0700 Subject: [PATCH 3/3] test(packaging): verify manylinux repair behavior and pin Rust Signed-off-by: Jason Zhou (Engrg-Hardware 1) --- .../setup-manylinux-python-rust/action.yml | 7 ++ tests/test_ci_workflow_contracts.py | 99 +++++++++++++++++-- 2 files changed, 98 insertions(+), 8 deletions(-) diff --git a/.github/actions/setup-manylinux-python-rust/action.yml b/.github/actions/setup-manylinux-python-rust/action.yml index 58dbb5f07a..287bb69478 100644 --- a/.github/actions/setup-manylinux-python-rust/action.yml +++ b/.github/actions/setup-manylinux-python-rust/action.yml @@ -21,6 +21,10 @@ runs: exit 1 } done + # Versions are pinned by the container digest. Exercise and record + # the tools before exporting an environment for the wheel build. + auditwheel --version + patchelf --version echo "${MANYLINUX_PYTHON_ROOT}/bin" >> "${GITHUB_PATH}" { echo "PYO3_PYTHON=${MANYLINUX_PYTHON_ROOT}/bin/python" @@ -31,3 +35,6 @@ runs: echo "RUSTUP_HOME=${RUNNER_TEMP}/rustup" } >> "${GITHUB_ENV}" - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + with: + # Match the existing nightly release builder. + toolchain: "1.98.0" diff --git a/tests/test_ci_workflow_contracts.py b/tests/test_ci_workflow_contracts.py index 322f88d172..1662a20d8f 100644 --- a/tests/test_ci_workflow_contracts.py +++ b/tests/test_ci_workflow_contracts.py @@ -18,6 +18,7 @@ import pytest import yaml +from scripts import build_manylinux_wheel as manylinux_builder from scripts.build_manylinux_wheel import manylinux_platform from scripts.check_application_test_inventory import Inventory, assignment from scripts.select_full_ci import COMPONENTS, select_components @@ -409,12 +410,6 @@ def test_linux_release_wheels_are_repaired_for_manylinux_2_28() -> None: assert 'sys.platform.startswith("linux")' in release_builder assert '"build_manylinux_wheel.py"' in release_builder - manylinux_builder = (REPOSITORY_ROOT / "scripts" / "build_manylinux_wheel.py").read_text() - assert '"--auditwheel",\n "skip"' in manylinux_builder - assert '"auditwheel",\n "repair"' in manylinux_builder - assert '"--plat",\n policy' in manylinux_builder - assert '_run("auditwheel", "show", str(repaired_wheel))' in manylinux_builder - dockerfile = (REPOSITORY_ROOT / "python" / "aisimulate" / "docker" / "Dockerfile").read_text() assert 'MATURIN_PEP517_ARGS="--auditwheel skip"' in dockerfile assert "auditwheel repair" in dockerfile @@ -440,6 +435,79 @@ def test_manylinux_platform_rejects_unknown_architecture() -> None: manylinux_platform("riscv64") +def _manylinux_build_case( + tmp_path, monkeypatch, *, machine="x86_64", raw_count=1, repaired_count=1, repaired_tag=None, fail_repair=False +): + monkeypatch.setattr(manylinux_builder.sys, "platform", "linux") + monkeypatch.setattr(manylinux_builder.platform, "machine", lambda: machine) + monkeypatch.setattr(manylinux_builder.shutil, "which", lambda _: "/usr/bin/auditwheel") + calls = [] + output = tmp_path / "dist" + + def run(*command, cwd=REPOSITORY_ROOT): + calls.append((command, cwd)) + if command[:3] == (sys.executable, "-m", "maturin"): + raw_output = Path(command[command.index("--out") + 1]) + for index in range(raw_count): + (raw_output / f"aisimulate-0.12.{index}-cp311-abi3-linux_{machine}.whl").touch() + elif command[:2] == ("auditwheel", "repair"): + if fail_repair: + raise subprocess.CalledProcessError(1, command) + tag = repaired_tag or f"manylinux_2_28_{machine}" + for index in range(repaired_count): + (output / f"aisimulate-0.12.{index}-cp311-abi3-{tag}.whl").touch() + else: + assert command[:2] == ("auditwheel", "show") + + monkeypatch.setattr(manylinux_builder, "_run", run) + return output, calls + + +@pytest.mark.parametrize("machine", ["x86_64", "aarch64"]) +def test_manylinux_build_repairs_the_exact_raw_wheel_then_audits_it(tmp_path, monkeypatch, machine): + output, calls = _manylinux_build_case(tmp_path, monkeypatch, machine=machine) + repaired = manylinux_builder.build(output) + raw_output = calls[0][0][-1] + raw_wheel = str(Path(raw_output) / f"aisimulate-0.12.0-cp311-abi3-linux_{machine}.whl") + assert repaired == output / f"aisimulate-0.12.0-cp311-abi3-manylinux_2_28_{machine}.whl" + assert list(output.iterdir()) == [repaired] + assert calls == [ + ( + (sys.executable, "-m", "maturin", "build", "--release", "--auditwheel", "skip", "--out", raw_output), + manylinux_builder.PYTHON_PROJECT, + ), + ( + ("auditwheel", "repair", "--plat", f"manylinux_2_28_{machine}", "--wheel-dir", str(output), raw_wheel), + REPOSITORY_ROOT, + ), + (("auditwheel", "show", str(repaired)), REPOSITORY_ROOT), + ] + + +@pytest.mark.parametrize( + ("options", "message"), + [ + ({"raw_count": 0}, "exactly one unrepaired"), + ({"raw_count": 2}, "exactly one unrepaired"), + ({"repaired_count": 0}, "exactly one repaired"), + ({"repaired_count": 2}, "exactly one repaired"), + ({"repaired_tag": "manylinux_2_17_x86_64"}, "required tag manylinux_2_28_x86_64"), + ], +) +def test_manylinux_build_rejects_missing_duplicate_or_wrong_policy_wheels(tmp_path, monkeypatch, options, message): + output, calls = _manylinux_build_case(tmp_path, monkeypatch, **options) + with pytest.raises(SystemExit, match=message): + manylinux_builder.build(output) + assert not any(command[:2] == ("auditwheel", "show") for command, _ in calls) + + +def test_manylinux_build_propagates_repair_failure(tmp_path, monkeypatch): + output, calls = _manylinux_build_case(tmp_path, monkeypatch, fail_repair=True) + with pytest.raises(subprocess.CalledProcessError): + manylinux_builder.build(output) + assert not any(command[:2] == ("auditwheel", "show") for command, _ in calls) + + def test_prediction_gate_owns_report_dependencies_and_pre_harness_fallback() -> None: jobs = _workflow("prediction-regression-gate.yml")["jobs"] report_commands = _run_commands(jobs["report"]) @@ -786,9 +854,13 @@ def test_manylinux_jobs_select_bundled_python_without_setup_python() -> None: action = yaml.safe_load((ACTION_ROOT / "setup-manylinux-python-rust" / "action.yml").read_text()) assert action["runs"]["steps"][0]["env"]["MANYLINUX_PYTHON_ROOT"] == "/opt/python/cp312-cp312" assert not any(step.get("uses", "").startswith("actions/setup-python@") for step in action["runs"]["steps"]) + rust = next(step for step in action["runs"]["steps"] if step.get("uses", "").startswith("dtolnay/rust-toolchain@")) + assert rust["with"]["toolchain"] == _workflow("nightly-ci.yml")["jobs"]["build-artifacts"]["env"]["RUST_TOOLCHAIN"] -@pytest.mark.parametrize("missing", [None, "python", "cc", "c++", "make", "auditwheel", "patchelf"]) +@pytest.mark.parametrize( + "missing", [None, "python", "cc", "c++", "make", "auditwheel", "patchelf", "auditwheel-broken", "patchelf-broken"] +) def test_manylinux_bootstrap_exports_working_python_or_fails_before_build(tmp_path: Path, missing: str | None) -> None: action = yaml.safe_load((ACTION_ROOT / "setup-manylinux-python-rust" / "action.yml").read_text()) bootstrap = action["runs"]["steps"][0]["run"] @@ -799,9 +871,18 @@ def test_manylinux_bootstrap_exports_working_python_or_fails_before_build(tmp_pa (binaries / "python").symlink_to(sys.executable) tools = tmp_path / "tools" tools.mkdir() + tool_log = tmp_path / "tool-log" for name in ("cc", "c++", "make", "auditwheel", "patchelf"): if name != missing: - (tools / name).symlink_to("/usr/bin/true") + binary = tools / name + if name in ("auditwheel", "patchelf"): + binary.write_text( + f'#!/bin/bash\necho "{name} $*" >> "$AUDIT_TOOL_LOG"\n' + + ("exit 1\n" if missing == f"{name}-broken" else 'test "$1" = --version\n') + ) + binary.chmod(0o755) + else: + binary.symlink_to("/usr/bin/true") env_file = tmp_path / "github-env" path_file = tmp_path / "github-path" env_file.touch() @@ -816,6 +897,7 @@ def test_manylinux_bootstrap_exports_working_python_or_fails_before_build(tmp_pa "RUNNER_TEMP": str(runner_temp), "GITHUB_ENV": str(env_file), "GITHUB_PATH": str(path_file), + "AUDIT_TOOL_LOG": str(tool_log), }, capture_output=True, text=True, @@ -826,6 +908,7 @@ def test_manylinux_bootstrap_exports_working_python_or_fails_before_build(tmp_pa assert env_file.read_text() == path_file.read_text() == "" return assert result.returncode == 0, result.stdout + result.stderr + assert tool_log.read_text().splitlines() == ["auditwheel --version", "patchelf --version"] exported = dict(line.split("=", 1) for line in env_file.read_text().splitlines()) assert exported == { "PYO3_PYTHON": str(binaries / "python"),