[RFC] test: add/use a pytest wrapper for the build/boot tests

.github/workflows/tests.yml

@@ -229,7 +229,7 @@ jobs:
       - name: Testing imgtestlib and test scripts
         run: |
           pip install .
-          python3 -m pytest -v
+          python3 -m pytest -v -k "not images_integration"
 
   python-lint:
     name: "🐍 Lint (test scripts)"

cmd/gen-manifests/main.go

@@ -424,6 +424,10 @@ func main() {
 	flag.Var(&imgTypes, "types", "comma-separated list of image types (globs supported)")
 	flag.Var(&bootcRefs, "bootc-refs", "comma-separated list of bootc-refs")
 
+	// print-only
+	var printOnly bool
+	flag.BoolVar(&printOnly, "print-only", false, "print what manifests would be generate")
+
 	flag.Parse()
 
 	testedRepoRegistry, err := testrepos.New()
@@ -443,7 +447,7 @@ func main() {
 	var configs *BuildConfigs
 	opts := &buildconfig.Options{AllowUnknownFields: buildconfigAllowUnknown}
 	if configPath != "" {
-		fmt.Println("'-config' was provided, thus ignoring '-config-list' option")
+		fmt.Fprintln(os.Stderr, "'-config' was provided, thus ignoring '-config-list' option")
 		configs = loadImgConfig(configPath, opts)
 	} else {
 		configs = loadConfigList(configMapPath, opts)
@@ -460,7 +464,7 @@ func main() {
 	tmpdirRoot := filepath.Join(os.TempDir(), "gen-manifests-tmpdir")
 	defer os.RemoveAll(tmpdirRoot)
 
-	fmt.Println("Collecting jobs")
+	fmt.Fprintln(os.Stderr, "Collecting jobs")
 
 	distros, invalidDistros := distros.ResolveArgValues(testedRepoRegistry.ListDistros())
 	if len(invalidDistros) > 0 {
@@ -503,7 +507,7 @@ func main() {
 				if len(repos) == 0 {
 					fmt.Printf("no repositories defined for %s/%s/%s\n", distroName, archName, imgTypeName)
 					if skipNorepos {
-						fmt.Println("Skipping")
+						fmt.Fprintln(os.Stderr, "Skipping")
 						continue
 					}
 					panic("no repositories found, pass --skip-norepos to skip")
@@ -523,8 +527,12 @@ func main() {
 						continue
 					}
 
-					job := makeManifestJob(itConfig, imgType, distribution, repos, archName, cacheRoot, outputDir, contentResolve, metadata, tmpdirRoot)
-					jobs = append(jobs, job)
+					if printOnly {
+						fmt.Printf("%s,%s,%s,%s\n", distribution.Name(), archName, imgType.Name(), itConfig.Name)
+					} else {
+						job := makeManifestJob(itConfig, imgType, distribution, repos, archName, cacheRoot, outputDir, contentResolve, metadata, tmpdirRoot)
+						jobs = append(jobs, job)
+					}
 				}
 			}
 		}
@@ -577,9 +585,13 @@ func main() {
 						continue
 					}
 
-					var repos []rpmmd.RepoConfig
-					job := makeManifestJob(itConfig, imgType, distribution, repos, archName, cacheRoot, outputDir, contentResolve, metadata, tmpdirRoot)
-					jobs = append(jobs, job)
+					if printOnly {
+						fmt.Printf("%s,%s,%s,%s\n", distribution.Name(), archName, imgType.Name(), itConfig.Name)
+					} else {
+						var repos []rpmmd.RepoConfig
+						job := makeManifestJob(itConfig, imgType, distribution, repos, archName, cacheRoot, outputDir, contentResolve, metadata, tmpdirRoot)
+						jobs = append(jobs, job)
+					}
 				}
 			}
 		}
@@ -669,17 +681,17 @@ func main() {
 	}
 
 	nJobs := len(jobs)
-	fmt.Printf("Collected %d jobs\n", nJobs)
+	fmt.Fprintf(os.Stderr, "Collected %d jobs\n", nJobs)
 
 	// nolint:gosec
 	wq := newWorkerQueue(uint32(nWorkers), uint32(nJobs))
 	wq.start()
-	fmt.Printf("Initialised %d workers\n", nWorkers)
-	fmt.Printf("Submitting %d jobs... ", nJobs)
+	fmt.Fprintf(os.Stderr, "Initialised %d workers\n", nWorkers)
+	fmt.Fprintf(os.Stderr, "Submitting %d jobs... ", nJobs)
 	for _, j := range jobs {
 		wq.submitJob(j)
 	}
-	fmt.Println("done")
+	fmt.Fprintln(os.Stderr, "done")
 	errs := wq.wait()
 	exit := 0
 	if nErrs := len(errs); nErrs > 0 {
@@ -697,6 +709,6 @@ func main() {
 		}
 		exit = 1
 	}
-	fmt.Printf("RPM metadata cache kept in %s\n", cacheRoot)
+	fmt.Fprintf(os.Stderr, "RPM metadata cache kept in %s\n", cacheRoot)
 	os.Exit(exit)
 }

test/conftest.py

@@ -0,0 +1,4 @@
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "images_integration"
+    )

test/scripts/generate-build-config

@@ -13,9 +13,8 @@ build/{distro}/{arch}/{image_type}/{config_name}:
   script:
     - sudo ./test/scripts/setup-osbuild-repo
     - sudo ./test/scripts/install-dependencies
-    - ./test/scripts/build-image "{distro}" "{image_type}" "{config}"
-    - ./test/scripts/boot-image "{image_path}"
-    - ./test/scripts/upload-results "{distro}" "{image_type}" "{config}"
+    - sudo -E pytest -k "{distro}-{arch}-{image_type}-{config_name}" ./test
+    - sudo -E ./test/scripts/upload-results "{distro}" "{image_type}" "{config}"
   extends: .terraform
   tags:
     - {tag}

test/scripts/generate-ostree-build-config

@@ -15,9 +15,8 @@ build/{distro}/{arch}/{image_type}/{config_name}:
     - sudo ./test/scripts/install-dependencies
     - {dl_container}
     - {start_container}
-    - ./test/scripts/build-image "{distro}" "{image_type}" "{config}"
-    - ./test/scripts/boot-image "{image_path}"
-    - ./test/scripts/upload-results "{distro}" "{image_type}" "{config}"
+    - sudo -E pytest -k "{distro}-{arch}-{image_type}-{config_name}" ./test
+    - sudo -E ./test/scripts/upload-results "{distro}" "{image_type}" "{config}"
   extends: .terraform
   variables:
     RUNNER: {runner}-{arch}

test/scripts/install-dependencies

@@ -22,6 +22,7 @@ dnf -y install \
     podman \
     python3 \
     python3-pip \
+    python3-pytest \
     yamllint \
     xz
 

test/test_build_integration.py

@@ -0,0 +1,55 @@
+import os
+import platform
+import subprocess
+
+import pytest
+import scripts.imgtestlib as testlib
+
+if os.getuid() != 0:
+    pytest.skip(reason="need root to build the images", allow_module_level=True)
+
+def _test_cases():
+    # XXX: make testcase a data class
+    all_test_cases = subprocess.check_output([
+        "go", "run", "-buildvcs=false", "./cmd/gen-manifests",
+        # we may consider cross arch tests here at some point but for now
+        # assume we run native
+        "-arches", platform.uname().machine,
+        "-print-only",
+    ], text=True).strip().split("\n")
+    boot_tests = set()
+    for tcase in all_test_cases:
+        distro, arch, image_type, config_name = tcase.split(",")
+        if not (image_type in testlib.CAN_BOOT_TEST["*"] or image_type in testlib.CAN_BOOT_TEST.get(arch, [])):
+            continue
+        # XXX: we need to filter further here, i.e. not all qemu tests can be run currently, e.g.
+        # if sshd is missing (see boot_image.ensure_can_run_qemu_test - however this needs
+        # a build/ dir so we cannot filter right now without also building all manifests. This
+        # should instead be refactored so that we have a testcase class and we can filter more
+        # flexible on that and exclude e.g. all image types with "minmal" and "minimal-pkgs"
+        # as their config_name
+        boot_tests.add(tcase)
+    return {
+        "build_only": set(all_test_cases)-boot_tests,
+        "build_and_boot": boot_tests,
+    }
+
+
+@pytest.mark.images_integration
+@pytest.mark.parametrize("distro,arch,image_type,config_name", [tcase.split(",") for tcase in _test_cases()["build_only"]])
+def test_build_only(distro,arch,image_type,config_name):
+    subprocess.check_call(
+        ["./test/scripts/build-image", distro, image_type, f"test/configs/{config_name}.json"])
+    build_dir = os.path.join("build", testlib.gen_build_name(distro, arch, image_type, config_name))
+    subprocess.check_call(
+        ["./test/scripts/boot-image", build_dir])
+
+
+@pytest.mark.images_integration
+@pytest.mark.parametrize("distro,arch,image_type,config_name", [tcase.split(",") for tcase in _test_cases()["build_and_boot"]])
+def test_build_and_boot(distro,arch,image_type,config_name):
+    subprocess.check_call(
+        ["./test/scripts/build-image", distro, image_type, f"test/configs/{config_name}.json"])
+    build_dir = os.path.join("build", testlib.gen_build_name(distro, arch, image_type, config_name))
+    subprocess.check_call(
+        ["./test/scripts/boot-image", build_dir])