test: cross arch build/boot smoke test for ppc64le,s390x

.github/workflows/tests.yml

@@ -278,3 +278,41 @@ jobs:
         # yq will catch issues that yamllint will not, like duplicate anchros
         run: |
           find . "(" -iname "*.yaml" -or -iname "*.yml" ")" -exec yq . {} \+ > /dev/null
+
+  cross-arch-integration-test:
+    strategy:
+      matrix:
+        arch:
+          - ppc64le
+          - s390x
+      fail-fast: false  # if one fails, keep the other(s) running
+    name: "Cross-arch qemu based image build/boot smoke test"
+    runs-on: ubuntu-24.04
+    env:
+      # workaround for expired cert at source of indirect dependency
+      # (go.opencensus.io/trace)
+      GOPROXY: "https://proxy.golang.org|direct"
+      GOFLAGS: "-buildvcs=false"
+    container:
+      image: registry.fedoraproject.org/fedora:latest
+      options: "--privileged"
+    steps:
+      - name: Install build and test dependencies
+        run: dnf -y install python3-pytest podman go btrfs-progs-devel device-mapper-devel gpgme-devel python3-pip qemu-system-s390x-core qemu-system-ppc64 qemu-user-static osbuild osbuild-depsolve-dnf cloud-utils
+      - name: Manually start qemu-user-static
+        run: |
+          # mostly for debugging
+          mount | grep binfmt_misc || true
+          ls /proc/sys/fs/binfmt_misc/
+          sudo mount -t binfmt_misc none /proc/sys/fs/binfmt_misc
+          ls /proc/sys/fs/binfmt_misc/
+          sudo SYSTEMD_LOG_LEVEL=debug /usr/lib/systemd/systemd-binfmt
+          ls /proc/sys/fs/binfmt_misc/
+          sudo SYSTEMD_LOG_LEVEL=debug /usr/lib/systemd/systemd-binfmt /usr/lib/binfmt.d/qemu-*.conf
+          ls /proc/sys/fs/binfmt_misc/
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v6
+      - name: Cross arch integration test
+        run: |
+          pip install .
+          sudo -E pytest -rs -k ${{ matrix.arch }} -s -v ./test/test_cross_arch_integration.py

cmd/build/main.go

@@ -26,11 +26,12 @@ func u(s string) string {
 
 func run() error {
 	// common args
-	var outputDir, osbuildStore, rpmCacheRoot, repositories string
+	var outputDir, osbuildStore, rpmCacheRoot, repositories, archName string
 	flag.StringVar(&outputDir, "output", ".", "artifact output directory")
 	flag.StringVar(&osbuildStore, "store", ".osbuild", "osbuild store for intermediate pipeline trees")
 	flag.StringVar(&rpmCacheRoot, "rpmmd", "/tmp/rpmmd", "rpm metadata cache directory")
 	flag.StringVar(&repositories, "repositories", "test/data/repositories", "path to repository file or directory")
+	flag.StringVar(&archName, "arch", "", "target architecture")
 
 	// osbuild checkpoint arg
 	var checkpoints cmdutil.MultiValue
@@ -64,8 +65,10 @@ func run() error {
 		return fmt.Errorf("invalid or unsupported distribution: %q", distroName)
 	}
 
-	archName := arch.Current().String()
-	arch, err := distribution.GetArch(archName)
+	if archName == "" {
+		archName = arch.Current().String()
+	}
+	archi, err := distribution.GetArch(archName)
 	if err != nil {
 		return fmt.Errorf("invalid arch name %q for distro %q: %w", archName, distroName, err)
 	}
@@ -76,7 +79,7 @@ func run() error {
 		return fmt.Errorf("failed to create target directory: %w", err)
 	}
 
-	imgType, err := arch.GetImageType(imgTypeName)
+	imgType, err := archi.GetImageType(imgTypeName)
 	if err != nil {
 		return fmt.Errorf("invalid image type %q for distro %q and arch %q: %w", imgTypeName, distroName, archName, err)
 	}
@@ -109,6 +112,9 @@ func run() error {
 		OverrideRepos:  overrideRepos,
 		CustomSeed:     &seedArg,
 	}
+	if archName != arch.Current().String() {
+		manifestOpts.UseBootstrapContainer = true
+	}
 	// add RHSM fact to detect changes
 	config.Options.Facts = &facts.ImageOptions{
 		APIType: facts.TEST_APITYPE,

test/scripts/boot-image

@@ -134,14 +134,12 @@ class CannotRunQemuTest(Exception):
         self.skip_reason = skip_reason
 
 
-def ensure_can_run_qemu_test(arch, image_path, config_file):
+def ensure_can_run_qemu_test(image_path, config_file):
     """
     Check if the given image_path, config_file is capable of running a
     qemu based test. Will return a bool and a skip_reason.
     """
     # keep in sync with imagetestlib.py:CAN_BOOT_TEST
-    if arch not in ["x86_64"]:
-        raise CannotRunQemuTest(f"no qemu boot test support for {arch} yet")
     manifest_path = pathlib.Path(image_path).parent / "../manifest.json"
     manifest = json.loads(manifest_path.read_text(encoding="utf8"))
     # Note that this needs adjustment when we switch to librepo
@@ -193,7 +191,7 @@ def qemu_cmd_scp_and_run(vm, cmd, privkey_path):
 
 
 def boot_qemu(arch, image_path, config_file, keep_booted=False):
-    ensure_can_run_qemu_test(arch, image_path, config_file)
+    ensure_can_run_qemu_test(image_path, config_file)
     cmd = [BASE_TEST_SCRIPT, config_file]
     with contextlib.ExitStack() as cm:
         uncompressed_image_path = cm.enter_context(ensure_uncompressed(image_path))
@@ -214,7 +212,7 @@ def boot_qemu(arch, image_path, config_file, keep_booted=False):
 
 
 def boot_qemu_iso_no_unattended_support(arch, installer_iso_path, config_file):
-    ensure_can_run_qemu_test(arch, installer_iso_path, config_file)
+    ensure_can_run_qemu_test(installer_iso_path, config_file)
     # this is for ISOs that have no "unattneded" support in their blueprint,
     # manually create one and modify the ISO
     rootpw = "".join(
@@ -273,7 +271,7 @@ def boot_qemu_iso_no_unattended_support(arch, installer_iso_path, config_file):
 
 
 def boot_qemu_iso(arch, installer_iso_path, config_file):
-    ensure_can_run_qemu_test(arch, installer_iso_path, config_file)
+    ensure_can_run_qemu_test(installer_iso_path, config_file)
     # We can only test the unattended-iso as the other configs require
     # interactive setup of the installer which we do not support in this
     # test-runner.

test/scripts/build-image

@@ -12,6 +12,7 @@ def main():
     parser.add_argument("distro", type=str, default=None, help="distro for the image to boot test")
     parser.add_argument("image_type", type=str, default=None, help="type of the image to boot test")
     parser.add_argument("config", type=str, help="config used to build the image")
+    parser.add_argument("--arch", default="", type=str, help="target arch of the image")
 
     args = parser.parse_args()
     distro = args.distro
@@ -30,14 +31,17 @@ def main():
 
     cmd = ["sudo", "-E", "./bin/build", "--output", "./build",
            "--distro", distro, "--type", image_type, "--config", config_path]
+    arch = os.uname().machine
+    if args.arch:
+        arch = args.arch
+        cmd += ["--arch", args.arch]
     testlib.runcmd_nc(cmd, extra_env=testlib.rng_seed_env())
 
     print("✅ Build finished!!")
 
     # Build artifacts are owned by root. Make them world accessible.
     testlib.runcmd(["sudo", "chmod", "a+rwX", "-R", "./build"])
 
-    arch = os.uname().machine
     build_dir = os.path.join("build", testlib.gen_build_name(distro, arch, image_type, config_name))
     manifest_path = os.path.join(build_dir, "manifest.json")
     with open(manifest_path, "r", encoding="utf-8") as manifest_fp:

test/test_cross_arch_integration.py

@@ -0,0 +1,32 @@
+import os
+import platform
+import re
+import shutil
+import subprocess
+
+import pytest
+import scripts.imgtestlib as testlib
+
+if os.getuid() != 0:
+    pytest.skip(reason="need root to build the images", allow_module_level=True)
+
+
+@pytest.mark.images_integration
+@pytest.mark.parametrize("arch", ["ppc64le", "s390x"])
+def test_build_boot_cross_arch_smoke(arch):
+    if arch == "s390x":
+        output = subprocess.check_output(["qemu-s390x-static", "--version"], text=True)
+        m = re.match(r'(?m).*version ([0-9]+)\.([0-9]+)\.([0-9]+)', output)
+        major, minor, patch = m.group(1, 2, 3)
+        if not (int(major) >= 10 and int(minor) >= 1 and int(patch) >= 91):
+            pytest.skip("need qemu-user >= 10.2 to run s390x builds")
+
+    # very minimal as this just a smoke test
+    distro = "centos-10"
+    image_type = "qcow2"
+    config_name = "empty"
+    subprocess.check_call(
+        ["./test/scripts/build-image", f"--arch={arch}", distro, image_type, f"test/configs/{config_name}.json"])
+    build_dir = os.path.join("build", testlib.gen_build_name(distro, arch, image_type, config_name))
+    subprocess.check_call(
+        ["./test/scripts/boot-image", build_dir])

vmtest/vm.py

@@ -151,8 +151,7 @@ def find_ovmf():
 
 
 class QEMU(VM):
-
-    def __init__(self, img, arch="", snapshot=True, cdrom=None, extra_args=None, memory="2000"):
+    def __init__(self, img, arch="", snapshot=True, cdrom=None, extra_args=None, memory="2048"):
         super().__init__()
         self._img = pathlib.Path(img)
         self._tmpdir = tempfile.mkdtemp(prefix="vmtest-", suffix=f"-{self._img.name}")
@@ -172,23 +171,43 @@ def __del__(self):
         shutil.rmtree(self._tmpdir)
 
     def _gen_qemu_cmdline(self, snapshot, use_ovmf):
+        virtio_scsi_hd = [
+            "-device", "virtio-scsi-pci,id=scsi",
+            "-device", "scsi-hd,drive=disk0",
+        ]
+        virtio_net_device = "virtio-net-pci"
         if self._arch in ("arm64", "aarch64"):
             qemu_cmdline = [
                 "qemu-system-aarch64",
                 "-machine", "virt",
                 "-cpu", "cortex-a57",
                 "-smp", "2",
                 "-bios", "/usr/share/AAVMF/AAVMF_CODE.fd",
-            ]
+            ] + virtio_scsi_hd
         elif self._arch in ("amd64", "x86_64"):
             qemu_cmdline = [
                 "qemu-system-x86_64",
                 "-M", "accel=kvm",
                 # get "illegal instruction" inside the VM otherwise
                 "-cpu", "host",
-            ]
+            ] + virtio_scsi_hd
             if use_ovmf:
                 qemu_cmdline.extend(["-bios", find_ovmf()])
+        elif self._arch in ("ppc64le", "ppc64"):
+            qemu_cmdline = [
+                "qemu-system-ppc64",
+                "-machine", "pseries",
+                "-smp", "2",
+            ] + virtio_scsi_hd
+        elif self._arch == "s390x":
+            qemu_cmdline = [
+                "qemu-system-s390x",
+                "-machine", "s390-ccw-virtio",
+                "-smp", "2",
+                # sepcial disk setup
+                "-device", "virtio-blk,drive=disk0,bootindex=1",
+            ]
+            virtio_net_device = "virtio-net-ccw"
         else:
             raise ValueError(f"unsupported architecture {self._arch}")
 
@@ -197,9 +216,11 @@ def _gen_qemu_cmdline(self, snapshot, use_ovmf):
             "-m", self._memory,
             "-serial", "stdio",
             "-monitor", "none",
+            "-device", f"{virtio_net_device},netdev=net.0,id=net.0",
             "-netdev", f"user,id=net.0,hostfwd=tcp::{self._ssh_port}-:22",
-            "-device", "e1000,netdev=net.0",
             "-qmp", f"unix:{self._qmp_socket},server,nowait",
+            # boot
+            "-drive", f"file={self._img},if=none,id=disk0,format=qcow2",
         ]
         if not os.environ.get("OSBUILD_TEST_QEMU_GUI"):
             qemu_cmdline.append("-nographic")