Disable skippable boot tests

test/scripts/boot-image

@@ -5,16 +5,18 @@ import json
 import os
 import pathlib
 import random
-import uuid
-import subprocess
 import shutil
 import string
+import subprocess
 import textwrap
+import uuid
 from tempfile import TemporaryDirectory
 
 import imgtestlib as testlib
+
 from vmtest.vm import QEMU
 
+
 BASE_TEST_SCRIPT = "test/scripts/check-host-config.sh"
 WSL_TEST_SCRIPT = "test/scripts/wsl-entrypoint.bat"
 
@@ -460,6 +462,8 @@ def main():
     bib_image_id = ""
     # Keep test/scripts/imagetestlib.py:CAN_BOOT_TEST in sync as it
     # has the same checks again
+    # WARNING: skipping boot tests that are listed under CAN_BOOT_TEST will cause the skipped image configuration to be
+    # rebuilt every CI run.
     match image_type:
         # Not all qcow2 types can be boot-tested, for example `server-qcow2` uses
         # initial-setup and this blocks the boot.

test/scripts/generate-ostree-build-config

@@ -15,7 +15,8 @@ build/{distro}/{arch}/{image_type}/{config_name}:
     - sudo ./test/scripts/install-dependencies
     - {dl_container}
     - {start_container}
-    - sudo -E pytest -s -k "{distro}-{arch}-{image_type}-{config_name}" ./test
+    - ./test/scripts/build-image "{distro}" "{image_type}" "{config}"
+    - ./test/scripts/boot-image "{image_path}"
     - sudo -E ./test/scripts/upload-results "{distro}" "{image_type}" "{config}"
   extends: .terraform
   variables:

test/scripts/imgtestlib.py

@@ -32,11 +32,15 @@
         "vhd",
         "cloud-ec2",
     ],
-    "x86_64": [
-        "image-installer", "minimal-installer", "network-installer",
-        "qcow2", "generic-qcow2", "cloud-qcow2",
-        "wsl", "generic-wsl",
-    ]
+    # NOTE(akoutsou): Temporarily disabling these because they can be skipped under certain circumstances, based on
+    # specific configs or the presence of specific packages. This causes problems with the cache checks, because the
+    # test is marked as possible to boot here, the boot test is added to pipeline, the test is skipped (and not marked
+    # as boot-success), so the image is built again the next time in order to boot.
+    # "x86_64": [
+    #     "image-installer", "minimal-installer", "network-installer",
+    #     "qcow2", "generic-qcow2", "cloud-qcow2",
+    #     "wsl", "generic-wsl",
+    # ]
 }
 
 BIB_TYPES = [