From fc79dd6af391c040612e035c52e8d9a50494a495 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 4 Dec 2025 13:15:44 +0100 Subject: [PATCH 1/3] build-image: add support for (optional) --arch to build This is useful to do basic testing for architectures like s390x or ppc64le that are not available easily (or even for a quick local aarch64 test). --- cmd/build/main.go | 14 ++++++++++---- test/scripts/build-image | 6 +++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/cmd/build/main.go b/cmd/build/main.go index da9adf536c..7edeab085a 100644 --- a/cmd/build/main.go +++ b/cmd/build/main.go @@ -26,11 +26,12 @@ func u(s string) string { func run() error { // common args - var outputDir, osbuildStore, rpmCacheRoot, repositories string + var outputDir, osbuildStore, rpmCacheRoot, repositories, archName string flag.StringVar(&outputDir, "output", ".", "artifact output directory") flag.StringVar(&osbuildStore, "store", ".osbuild", "osbuild store for intermediate pipeline trees") flag.StringVar(&rpmCacheRoot, "rpmmd", "/tmp/rpmmd", "rpm metadata cache directory") flag.StringVar(&repositories, "repositories", "test/data/repositories", "path to repository file or directory") + flag.StringVar(&archName, "arch", "", "target architecture") // osbuild checkpoint arg var checkpoints cmdutil.MultiValue @@ -64,8 +65,10 @@ func run() error { return fmt.Errorf("invalid or unsupported distribution: %q", distroName) } - archName := arch.Current().String() - arch, err := distribution.GetArch(archName) + if archName == "" { + archName = arch.Current().String() + } + archi, err := distribution.GetArch(archName) if err != nil { return fmt.Errorf("invalid arch name %q for distro %q: %w", archName, distroName, err) } @@ -76,7 +79,7 @@ func run() error { return fmt.Errorf("failed to create target directory: %w", err) } - imgType, err := arch.GetImageType(imgTypeName) + imgType, err := archi.GetImageType(imgTypeName) if err != nil { return fmt.Errorf("invalid image type %q for distro %q and arch %q: %w", imgTypeName, distroName, archName, err) } @@ -109,6 +112,9 @@ func run() error { OverrideRepos: overrideRepos, CustomSeed: &seedArg, } + if archName != arch.Current().String() { + manifestOpts.UseBootstrapContainer = true + } // add RHSM fact to detect changes config.Options.Facts = &facts.ImageOptions{ APIType: facts.TEST_APITYPE, diff --git a/test/scripts/build-image b/test/scripts/build-image index 97b3096fc0..6b937cc607 100755 --- a/test/scripts/build-image +++ b/test/scripts/build-image @@ -12,6 +12,7 @@ def main(): parser.add_argument("distro", type=str, default=None, help="distro for the image to boot test") parser.add_argument("image_type", type=str, default=None, help="type of the image to boot test") parser.add_argument("config", type=str, help="config used to build the image") + parser.add_argument("--arch", default="", type=str, help="target arch of the image") args = parser.parse_args() distro = args.distro @@ -30,6 +31,10 @@ def main(): cmd = ["sudo", "-E", "./bin/build", "--output", "./build", "--distro", distro, "--type", image_type, "--config", config_path] + arch = os.uname().machine + if args.arch: + arch = args.arch + cmd += ["--arch", args.arch] testlib.runcmd_nc(cmd, extra_env=testlib.rng_seed_env()) print("✅ Build finished!!") @@ -37,7 +42,6 @@ def main(): # Build artifacts are owned by root. Make them world accessible. testlib.runcmd(["sudo", "chmod", "a+rwX", "-R", "./build"]) - arch = os.uname().machine build_dir = os.path.join("build", testlib.gen_build_name(distro, arch, image_type, config_name)) manifest_path = os.path.join(build_dir, "manifest.json") with open(manifest_path, "r", encoding="utf-8") as manifest_fp: From 4f1df3b2bc1445b42e19444efed314971c6c1e41 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 4 Dec 2025 13:05:23 +0100 Subject: [PATCH 2/3] vmtest: add cross arch boot support for s390x/ppc64le This adds basic support to boot ppc64le/s390x machine. This allows to do: ``` $ ./test/scripts/build-image --arch ppc64le centos-10 qcow2 cfgpath $ ./test/scripts/boot-image ./build/centos_10-ppc64le-qcow2-empty ``` to easily boot test machines that are not easily available otherwise. --- test/scripts/boot-image | 10 ++++------ vmtest/vm.py | 31 ++++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/test/scripts/boot-image b/test/scripts/boot-image index 59e9946710..862d025818 100755 --- a/test/scripts/boot-image +++ b/test/scripts/boot-image @@ -134,14 +134,12 @@ class CannotRunQemuTest(Exception): self.skip_reason = skip_reason -def ensure_can_run_qemu_test(arch, image_path, config_file): +def ensure_can_run_qemu_test(image_path, config_file): """ Check if the given image_path, config_file is capable of running a qemu based test. Will return a bool and a skip_reason. """ # keep in sync with imagetestlib.py:CAN_BOOT_TEST - if arch not in ["x86_64"]: - raise CannotRunQemuTest(f"no qemu boot test support for {arch} yet") manifest_path = pathlib.Path(image_path).parent / "../manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf8")) # Note that this needs adjustment when we switch to librepo @@ -193,7 +191,7 @@ def qemu_cmd_scp_and_run(vm, cmd, privkey_path): def boot_qemu(arch, image_path, config_file, keep_booted=False): - ensure_can_run_qemu_test(arch, image_path, config_file) + ensure_can_run_qemu_test(image_path, config_file) cmd = [BASE_TEST_SCRIPT, config_file] with contextlib.ExitStack() as cm: uncompressed_image_path = cm.enter_context(ensure_uncompressed(image_path)) @@ -214,7 +212,7 @@ def boot_qemu(arch, image_path, config_file, keep_booted=False): def boot_qemu_iso_no_unattended_support(arch, installer_iso_path, config_file): - ensure_can_run_qemu_test(arch, installer_iso_path, config_file) + ensure_can_run_qemu_test(installer_iso_path, config_file) # this is for ISOs that have no "unattneded" support in their blueprint, # manually create one and modify the ISO rootpw = "".join( @@ -273,7 +271,7 @@ def boot_qemu_iso_no_unattended_support(arch, installer_iso_path, config_file): def boot_qemu_iso(arch, installer_iso_path, config_file): - ensure_can_run_qemu_test(arch, installer_iso_path, config_file) + ensure_can_run_qemu_test(installer_iso_path, config_file) # We can only test the unattended-iso as the other configs require # interactive setup of the installer which we do not support in this # test-runner. diff --git a/vmtest/vm.py b/vmtest/vm.py index 5eb35fdccd..fcae780dda 100644 --- a/vmtest/vm.py +++ b/vmtest/vm.py @@ -151,8 +151,7 @@ def find_ovmf(): class QEMU(VM): - - def __init__(self, img, arch="", snapshot=True, cdrom=None, extra_args=None, memory="2000"): + def __init__(self, img, arch="", snapshot=True, cdrom=None, extra_args=None, memory="2048"): super().__init__() self._img = pathlib.Path(img) self._tmpdir = tempfile.mkdtemp(prefix="vmtest-", suffix=f"-{self._img.name}") @@ -172,6 +171,11 @@ def __del__(self): shutil.rmtree(self._tmpdir) def _gen_qemu_cmdline(self, snapshot, use_ovmf): + virtio_scsi_hd = [ + "-device", "virtio-scsi-pci,id=scsi", + "-device", "scsi-hd,drive=disk0", + ] + virtio_net_device = "virtio-net-pci" if self._arch in ("arm64", "aarch64"): qemu_cmdline = [ "qemu-system-aarch64", @@ -179,16 +183,31 @@ def _gen_qemu_cmdline(self, snapshot, use_ovmf): "-cpu", "cortex-a57", "-smp", "2", "-bios", "/usr/share/AAVMF/AAVMF_CODE.fd", - ] + ] + virtio_scsi_hd elif self._arch in ("amd64", "x86_64"): qemu_cmdline = [ "qemu-system-x86_64", "-M", "accel=kvm", # get "illegal instruction" inside the VM otherwise "-cpu", "host", - ] + ] + virtio_scsi_hd if use_ovmf: qemu_cmdline.extend(["-bios", find_ovmf()]) + elif self._arch in ("ppc64le", "ppc64"): + qemu_cmdline = [ + "qemu-system-ppc64", + "-machine", "pseries", + "-smp", "2", + ] + virtio_scsi_hd + elif self._arch == "s390x": + qemu_cmdline = [ + "qemu-system-s390x", + "-machine", "s390-ccw-virtio", + "-smp", "2", + # sepcial disk setup + "-device", "virtio-blk,drive=disk0,bootindex=1", + ] + virtio_net_device = "virtio-net-ccw" else: raise ValueError(f"unsupported architecture {self._arch}") @@ -197,9 +216,11 @@ def _gen_qemu_cmdline(self, snapshot, use_ovmf): "-m", self._memory, "-serial", "stdio", "-monitor", "none", + "-device", f"{virtio_net_device},netdev=net.0,id=net.0", "-netdev", f"user,id=net.0,hostfwd=tcp::{self._ssh_port}-:22", - "-device", "e1000,netdev=net.0", "-qmp", f"unix:{self._qmp_socket},server,nowait", + # boot + "-drive", f"file={self._img},if=none,id=disk0,format=qcow2", ] if not os.environ.get("OSBUILD_TEST_QEMU_GUI"): qemu_cmdline.append("-nographic") From 63263ad2b15231cb6a868d38f11c784cef6db105 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 10 Dec 2025 12:26:10 +0100 Subject: [PATCH 3/3] test: run cross arch smoke test on ppc64le, s390x This commit adds a minimal smoke test for centos-10/qcow2 that will create a qcow2 and boot it. It demos how we could do more cross arch testing and keeps the qemu code tested. We need to decide if we want to do more tests like this, its a trade-off. OTOH its nice to have (some) assurance that our images boot (we did break s390/ppc64 partition tables in the past by accident). OTOH its a bit of a pain when something fails to figure out of it is failing because of some qemu incompatibilites or because there is a real issue in the image. Having this at least for local testing/validation is probably useful though. Note that the s390x test needs qemu-user >= 7.2-rc2 so this will not run currently (I did run it on my local machine and it works but the kdump service takes > 5min to startup). --- .github/workflows/tests.yml | 38 +++++++++++++++++++++++++++++ test/test_cross_arch_integration.py | 32 ++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 test/test_cross_arch_integration.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8ad8006abc..4a5f703c7b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -278,3 +278,41 @@ jobs: # yq will catch issues that yamllint will not, like duplicate anchros run: | find . "(" -iname "*.yaml" -or -iname "*.yml" ")" -exec yq . {} \+ > /dev/null + + cross-arch-integration-test: + strategy: + matrix: + arch: + - ppc64le + - s390x + fail-fast: false # if one fails, keep the other(s) running + name: "Cross-arch qemu based image build/boot smoke test" + runs-on: ubuntu-24.04 + env: + # workaround for expired cert at source of indirect dependency + # (go.opencensus.io/trace) + GOPROXY: "https://proxy.golang.org|direct" + GOFLAGS: "-buildvcs=false" + container: + image: registry.fedoraproject.org/fedora:latest + options: "--privileged" + steps: + - name: Install build and test dependencies + run: dnf -y install python3-pytest podman go btrfs-progs-devel device-mapper-devel gpgme-devel python3-pip qemu-system-s390x-core qemu-system-ppc64 qemu-user-static osbuild osbuild-depsolve-dnf cloud-utils + - name: Manually start qemu-user-static + run: | + # mostly for debugging + mount | grep binfmt_misc || true + ls /proc/sys/fs/binfmt_misc/ + sudo mount -t binfmt_misc none /proc/sys/fs/binfmt_misc + ls /proc/sys/fs/binfmt_misc/ + sudo SYSTEMD_LOG_LEVEL=debug /usr/lib/systemd/systemd-binfmt + ls /proc/sys/fs/binfmt_misc/ + sudo SYSTEMD_LOG_LEVEL=debug /usr/lib/systemd/systemd-binfmt /usr/lib/binfmt.d/qemu-*.conf + ls /proc/sys/fs/binfmt_misc/ + - name: Check out code into the Go module directory + uses: actions/checkout@v6 + - name: Cross arch integration test + run: | + pip install . + sudo -E pytest -rs -k ${{ matrix.arch }} -s -v ./test/test_cross_arch_integration.py diff --git a/test/test_cross_arch_integration.py b/test/test_cross_arch_integration.py new file mode 100644 index 0000000000..4551981016 --- /dev/null +++ b/test/test_cross_arch_integration.py @@ -0,0 +1,32 @@ +import os +import platform +import re +import shutil +import subprocess + +import pytest +import scripts.imgtestlib as testlib + +if os.getuid() != 0: + pytest.skip(reason="need root to build the images", allow_module_level=True) + + +@pytest.mark.images_integration +@pytest.mark.parametrize("arch", ["ppc64le", "s390x"]) +def test_build_boot_cross_arch_smoke(arch): + if arch == "s390x": + output = subprocess.check_output(["qemu-s390x-static", "--version"], text=True) + m = re.match(r'(?m).*version ([0-9]+)\.([0-9]+)\.([0-9]+)', output) + major, minor, patch = m.group(1, 2, 3) + if not (int(major) >= 10 and int(minor) >= 1 and int(patch) >= 91): + pytest.skip("need qemu-user >= 10.2 to run s390x builds") + + # very minimal as this just a smoke test + distro = "centos-10" + image_type = "qcow2" + config_name = "empty" + subprocess.check_call( + ["./test/scripts/build-image", f"--arch={arch}", distro, image_type, f"test/configs/{config_name}.json"]) + build_dir = os.path.join("build", testlib.gen_build_name(distro, arch, image_type, config_name)) + subprocess.check_call( + ["./test/scripts/boot-image", build_dir])