ci: Unify more of hack/ and tests/

A key thing for me is that the `Justfile` should be a one-stop
shop for development of the project. It can't have everything but
it should answer the basic questions of "how do I build and test
this project".

This aligns the recently added tmt-on-GHA flow a *bit* more closely
with some of that. Biggest is to use the `just build-integration-test-image` as the canonical
way to build a container image with our testing stuff in it;
which uses our main Dockerfile

Other cleanups:
- Change test script to move into tests/tmt/ as a workaround for
  teemtee/tmt#3037 (comment)
- Change the qemu logic to use SMBIOS credentials so we don't
  have to carry around both a disk image and a SSH key
- Change qemu to use `-snapshot` so we can reuse disks
- Change the scripts to accept data via argv[1] and not environment
- Drop the hardcoded testing directory and use `target/` as
  a generic build artifact dir

Signed-off-by: Colin Walters <[email protected]>

Author: cgwalters

.github/workflows/ci.yml

@@ -56,7 +56,9 @@ jobs:
         run: sudo apt update && sudo apt install just
       - uses: actions/checkout@v4
       - name: Build and run container integration tests
-        run: sudo just run-container-integration run-container-external-tests
+        run: |
+          sudo just build
+          sudo just run-container-integration run-container-external-tests
   container-continuous:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'control/skip-ci') }}
     runs-on: ubuntu-24.04
@@ -105,6 +107,7 @@ jobs:
           set -xeu
           # Build images to test; TODO investigate doing single container builds
           # via GHA and pushing to a temporary registry to share among workflows?
+          sudo just build
           sudo just build-integration-test-image
           sudo podman build -t localhost/bootc-fsverity -f ci/Containerfile.install-fsverity
 

.github/workflows/integration.yml

@@ -1,84 +1,85 @@
-name: bootc integration test
+# This workflow builds a container across a matrix of OSes,
+# generates a disk image from that, and runs integration tests
+# using tmt + libvirt (using nested virt support in the default GHA runners).
+name: Build+TMT
 on:
   pull_request:
-    branches: [main]
+    branches: [main]  
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   build:
     strategy:
+      fail-fast: false
       matrix:
-        test_os: [fedora-41, fedora-42, fedora-43, centos-9]
-        test_runner: [ubuntu-latest, ubuntu-24.04-arm]
+        #test_os: [fedora-42, fedora-43, centos-9, centos-10]
+        test_os: [centos-10]
+        test_runner: [ubuntu-24.04]
 
     runs-on: ${{ matrix.test_runner }}
 
     steps:
-      - name: Install podman for heredoc support
+      - name: Install dependencies
         run: |
           set -eux
           echo 'deb [trusted=yes] https://ftp.debian.org/debian/ testing main' | sudo tee /etc/apt/sources.list.d/testing.list
           sudo apt update
-          sudo apt install -y crun/testing podman/testing
+          sudo apt install -y crun/testing podman/testing just qemu-utils
 
       - uses: actions/checkout@v4
 
-      - name: Build bootc and bootc image
-        env:
-          TEST_OS: ${{ matrix.test_os }}
-        run: sudo -E TEST_OS=$TEST_OS tests/build.sh
+      - name: Set architecture variable
+        id: set_arch
+        run: echo "ARCH=$(arch)" >> $GITHUB_ENV
 
-      - name: Grant sudo user permission to archive files
+      - name: Build container and disk image
         run: |
-          sudo chmod 0755 /tmp/tmp-bootc-build/id_rsa
-
-      - name: Archive bootc disk image - disk.raw
-        if: matrix.test_runner == 'ubuntu-latest'
-        uses: actions/upload-artifact@v4
-        with:
-          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-disk
-          path: /tmp/tmp-bootc-build/disk.raw
-          retention-days: 1
+          sudo tests/build.sh ${{ matrix.test_os }}
 
-      - name: Archive SSH private key - id_rsa
-        if: matrix.test_runner == 'ubuntu-latest'
+      - name: Archive disk image
         uses: actions/upload-artifact@v4
         with:
-          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-id_rsa
-          path: /tmp/tmp-bootc-build/id_rsa
+          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ env.ARCH }}-disk
+          path: target/bootc-integration-test.qcow2
           retention-days: 1
 
   test:
     needs: build
     strategy:
+      fail-fast: false
       matrix:
-        test_os: [fedora-41, fedora-42, fedora-43, centos-9]
+        #test_os: [fedora-42, fedora-43, centos-9, centos-10]
+        test_os: [centos-10]
         tmt_plan: [test-01-readonly, test-20-local-upgrade, test-21-logically-bound-switch, test-22-logically-bound-install, test-23-install-outside-container, test-24-local-upgrade-reboot]
 
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install dependence
+      - name: Set architecture variable
+        id: set_arch
+        run: echo "ARCH=$(arch)" >> $GITHUB_ENV
+
+      - name: Install deps
         run: |
           sudo apt-get update
-          sudo apt install -y qemu-kvm qemu-system
-          pip install --user tmt
+          # see https://tmt.readthedocs.io/en/stable/overview.html#install
+          sudo apt install -y libkrb5-dev pkg-config libvirt-dev genisoimage qemu-kvm qemu-utils libvirt-daemon-system
+          pip install --user "tmt[provision-virtual]"
 
       - name: Create folder to save disk image
-        run: mkdir -p /tmp/tmp-bootc-build
+        run: mkdir -p target
 
       - name: Download disk.raw
         uses: actions/download-artifact@v4
         with:
-          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-disk
-          path: /tmp/tmp-bootc-build
-
-      - name: Download id_rsa
-        uses: actions/download-artifact@v4
-        with:
-          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-id_rsa
-          path: /tmp/tmp-bootc-build
+          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ env.ARCH }}-disk
+          path: target
 
       - name: Enable KVM group perms
         run: |
@@ -87,14 +88,17 @@ jobs:
           sudo udevadm trigger --name-match=kvm
           ls -l /dev/kvm
 
+      - name: Workaround https://github.com/teemtee/testcloud/issues/18
+        run: sudo rm -f /usr/bin/chcon && sudo ln -sr /usr/bin/true /usr/bin/chcon
+
       - name: Run test
-        env:
-          TMT_PLAN_NAME: ${{ matrix.tmt_plan }}
-        run: chmod 600 /tmp/tmp-bootc-build/id_rsa && tests/test.sh
+        run: |
+          ls -al target
+          tests/run-tmt.sh plan  --name "/tmt/plans/integration/"${{ matrix.tmt_plan }}
 
       - name: Archive TMT logs
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ matrix.tmt_plan }}
+          name: tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ env.ARCH }}-${{ matrix.tmt_plan }}
           path: /var/tmp/tmt

.packit.yaml

@@ -59,24 +59,26 @@ jobs:
     owner: rhcontainerbot
     project: bootc
     enable_net: true
+    # TODO
     notifications:
       failure_comment:
         message: "bootc Copr build failed for {commit_sha}. @admin check logs {logs_url} and packit dashboard {packit_dashboard_url}"
 
-  - job: tests
-    trigger: pull_request
-    targets:
-      - centos-stream-9-x86_64
-      - centos-stream-9-aarch64
-      - centos-stream-10-x86_64
-      - centos-stream-10-aarch64
-      - fedora-42-x86_64
-      - fedora-42-aarch64
-      - fedora-rawhide-x86_64
-      - fedora-rawhide-aarch64
-    tmt_plan: /integration
-    skip_build: true
-    identifier: integration-test
+  # TODO: Readd some tmt tests that install the built RPM and e.g. test out system-reinstall-bootc
+  # - job: tests
+  #   trigger: pull_request
+  #   targets:
+  #     - centos-stream-9-x86_64
+  #     - centos-stream-9-aarch64
+  #     - centos-stream-10-x86_64
+  #     - centos-stream-10-aarch64
+  #     - fedora-42-x86_64
+  #     - fedora-42-aarch64
+  #     - fedora-rawhide-x86_64
+  #     - fedora-rawhide-aarch64
+  #   tmt_plan: /integration
+  #   skip_build: true
+  #   identifier: integration-test
 
   - job: propose_downstream
     trigger: release

Justfile

@@ -3,8 +3,10 @@ build *ARGS:
     podman build --jobs=4 -t localhost/bootc {{ARGS}} .
 
 # This container image has additional testing content and utilities
-build-integration-test-image *ARGS: build
+build-integration-test-image *ARGS:
     podman build --jobs=4 -t localhost/bootc-integration -f hack/Containerfile {{ARGS}} .
+    # Keep these in sync with what's used in hack/lbi
+    podman pull -q --retry 5 --retry-delay 5s quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest
 
 # Run container integration tests
 run-container-integration: build-integration-test-image

hack/Containerfile

@@ -1,4 +1,7 @@
-# This injects some extra testing stuff into our image
+# Build a container image that has extra testing stuff in it, such
+# as nushell, some preset logically bound images, etc. This expects
+# to create an image derived FROM localhost/bootc which was created
+# by the Dockerfile at top.
 
 FROM scratch as context
 # We only need this stuff in the initial context
@@ -11,7 +14,15 @@ ARG variant=
 # And this layer has additional stuff for testing, such as nushell etc.
 RUN --mount=type=bind,from=context,target=/run/context <<EORUN
 set -xeuo pipefail
-/run/context/hack/provision-derived.sh "$variant"
+cd /run/context/hack
+./provision-derived.sh "$variant"
+
+# For test-22-logically-bound-install
+cp -a lbi/usr/. /usr
+for x in curl.container curl-base.image podman.image; do
+    ln -s /usr/share/containers/systemd/$x /usr/lib/bootc/bound-images.d/$x
+done
+
 # Add some testing kargs into our dev builds
 install -D -t /usr/lib/bootc/kargs.d /run/context/hack/test-kargs/*
 # Also copy in some default install configs we use for testing

hack/lbi/usr/share/containers/systemd/curl-base.image

@@ -0,0 +1,2 @@
+[Image]
+Image=quay.io/curl/curl-base:latest

hack/lbi/usr/share/containers/systemd/curl.container

@@ -0,0 +1,3 @@
+[Container]
+Image=quay.io/curl/curl:latest
+GlobalArgs=--storage-opt=additionalimagestore=/usr/lib/bootc/storage

hack/lbi/usr/share/containers/systemd/jboss-webserver-5.image

@@ -0,0 +1,6 @@
+# This is not symlinked to bound-images.d so it should not be pulled.
+# It's here to represent an app image that exists
+# in a bootc image but is not logically bound.
+[Image]
+Image=registry.redhat.io/jboss-webserver-5/jws5-rhel8-operator:latest
+AuthFile=/root/auth.json

hack/lbi/usr/share/containers/systemd/podman.image

@@ -0,0 +1,2 @@
+[Image]
+Image=registry.access.redhat.com/ubi9/podman:latest

hack/packages.txt

@@ -0,0 +1,5 @@
+# Needed by tmt
+rsync
+cloud-init
+/usr/bin/flock
+/usr/bin/awk