diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9e5297d1a2..841fdeee38 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -80,7 +80,8 @@ jobs: run: sudo apt update # This is needed for the container resolver dependencies - - name: Install libgpgme devel package + - &apt_install_dependencies + name: Install libgpgme devel package run: sudo apt install -y libgpgme-dev libbtrfs-dev libdevmapper-dev podman # We need to run the test as root, since we use the root @@ -164,8 +165,7 @@ jobs: run: sudo apt install -y libkrb5-dev # This is needed for the container upload dependencies - - name: Install libgpgme devel package - run: sudo apt install -y libgpgme-dev libbtrfs-dev libdevmapper-dev + - *apt_install_dependencies - name: Run golangci-lint uses: golangci/golangci-lint-action@v8 @@ -264,3 +264,30 @@ jobs: # We only care about distro definitions for this check run: | find pkg/distro/defs "(" -iname "*.yaml" -or -iname "*.yml" ")" -exec yq . {} \+ > /dev/null + + bib-manifest-diff: + name: "bib manifest diff validation" + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - *apt_install_dependencies + + - run: sudo apt install -y qemu-user-static + + - name: Check out code into the Go module directory + uses: actions/checkout@v5 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Workaround podman issues in GH actions + run: | + # see https://github.com/osbuild/bootc-image-builder/issues/446 + sudo rm -rf /var/lib/containers/storage + sudo mkdir -p /etc/containers + echo -e "[storage]\ndriver = \"overlay\"\nrunroot = \"/run/containers/storage\"\ngraphroot = \"/var/lib/containers/storage\"" | sudo tee /etc/containers/storage.conf + + - name: Running manifest diff between images/bib + run: ./tools/gen-bootc-diff diff --git a/cmd/gen-manifests/main.go b/cmd/gen-manifests/main.go index 3fca750ab0..b9e806411b 100644 --- a/cmd/gen-manifests/main.go +++ b/cmd/gen-manifests/main.go @@ -23,6 +23,7 @@ import ( "github.com/osbuild/images/internal/cmdutil" "github.com/osbuild/images/pkg/container" "github.com/osbuild/images/pkg/distro" + "github.com/osbuild/images/pkg/distro/bootc" "github.com/osbuild/images/pkg/distrofactory" "github.com/osbuild/images/pkg/dnfjson" "github.com/osbuild/images/pkg/experimentalflags" @@ -400,10 +401,11 @@ func main() { flag.BoolVar(&commits, "commits", false, "resolve ostree commit IDs") // manifest selection args - var arches, distros, imgTypes cmdutil.MultiValue + var arches, distros, imgTypes, bootcRefs cmdutil.MultiValue flag.Var(&arches, "arches", "comma-separated list of architectures (globs supported)") flag.Var(&distros, "distros", "comma-separated list of distributions (globs supported)") flag.Var(&imgTypes, "types", "comma-separated list of image types (globs supported)") + flag.Var(&bootcRefs, "bootc-refs", "comma-separated list of bootc-refs") flag.Parse() @@ -511,6 +513,49 @@ func main() { } } } + for _, bootcRef := range bootcRefs { + distribution, err := bootc.NewBootcDistro(bootcRef) + if err != nil { + panic(err) + } + // XXX: consider making this configurable but for now + // we just need diffable manifests + if distribution.DefaultFs() == "" { + if err := distribution.SetDefaultFs("ext4"); err != nil { + panic(err) + } + } + for _, archName := range arches { + archi, err := distribution.GetArch(archName) + if err != nil { + panic(err) + } + for _, imgTypeName := range imgTypes { + imgType, err := archi.GetImageType(imgTypeName) + if err != nil { + panic(err) + } + // XXX: copied from loop above + imgTypeConfigs := configs.Get(distribution.Name(), archName, imgTypeName) + if len(imgTypeConfigs) == 0 { + if skipNoconfig { + continue + } + panic(fmt.Sprintf("no configs defined for image type %q for %s/%s", imgTypeName, distribution.Name(), archi.Name())) + } + for _, itConfig := range imgTypeConfigs { + if needsSkipping, reason := configs.needsSkipping(distribution.Name(), itConfig); needsSkipping { + fmt.Printf("Skipping %s for %s/%s (reason: %v)\n", itConfig.Name, imgTypeName, distribution.Name(), reason) + continue + } + + var repos []rpmmd.RepoConfig + job := makeManifestJob(itConfig, imgType, distribution, repos, archName, cacheRoot, outputDir, contentResolve, metadata, tmpdirRoot) + jobs = append(jobs, job) + } + } + } + } nJobs := len(jobs) fmt.Printf("Collected %d jobs\n", nJobs) diff --git a/pkg/distro/bootc/bootc.go b/pkg/distro/bootc/bootc.go index 69bfd00431..9e315ca6c1 100644 --- a/pkg/distro/bootc/bootc.go +++ b/pkg/distro/bootc/bootc.go @@ -90,6 +90,10 @@ func (d *BootcDistro) SetDefaultFs(defaultFs string) error { return nil } +func (d *BootcDistro) DefaultFs() string { + return d.defaultFs +} + func (d *BootcDistro) Name() string { return d.name } diff --git a/tools/gen-bootc-diff b/tools/gen-bootc-diff new file mode 100755 index 0000000000..57ce4031e8 --- /dev/null +++ b/tools/gen-bootc-diff @@ -0,0 +1,123 @@ +#!/bin/bash +# +# This tool is meant as a temporary measure to ensure that we +# produce identical manifests from the bootc integration of +# images. We need it because we have no manifest checksums +# for bootc based images (yet). + +set -e +set -o pipefail + +(cd "$(dirname "$0")"/../cmd/gen-manifests && go build -buildvcs=false) + +# we only need to test a single type, all bootc disk manifests +# contain all possible types by default +TYPE=qcow2 + +BPDIR=$(mktemp -d) +trap 'rm -rf -- "$BPDIR"' EXIT +BP_EMPTY="$BPDIR/empty.json" +cat < "$BP_EMPTY" +{} +EOF +BP_FULL_FS="$BPDIR/full.json" +cat < "$BP_FULL_FS" +{ + "customizations": { + "filesystem": [ + { + "mountpoint": "/", + "minsize": "10 GiB" + }, + { + "mountpoint": "/var/data", + "minsize": "20 GiB" + } + ], + "user": [ + { + "name": "alice", + "key": "ssh-rsa AAA ... user@email.com", + "groups": [ + "wheel", + "admins" + ] + } + ], + "group": [ + { + "name": "fancypants" + } + ] + } +} +EOF +BP_LVM="$BPDIR/lvm.json" +cat < "$BP_LVM" +{ + "customizations": { + "disk": { + "partitions": [ + { + "type": "lvm", + "minsize": "22 GiB", + "logical_volumes": [ + { + "mountpoint": "/", + "fs_type": "ext4", + "minsize": "15 GiB" + } + ] + } + ] + } + } +} +EOF + + +for REF in quay.io/centos-bootc/centos-bootc:stream9 \ + quay.io/fedora/fedora-bootc:42; +do + for ARCH in x86_64 aarch64; do + for BP in "$BP_FULL_FS" "$BP_LVM" "$BP_EMPTY"; do + echo "Testing $REF;$ARCH;$(basename "$BP")" + sudo podman pull -q --arch "$ARCH" "$REF" + + TMPDIR=$(mktemp -d) + trap 'rm -rf -- "$TMPDIR"' EXIT + + ROOTFS="" + if grep -q fedora <(echo "$REF"); then + ROOTFS="ext4" + fi + + sudo go run \ + -buildvcs=false \ + github.com/osbuild/bootc-image-builder/bib/cmd/bootc-image-builder@main \ + manifest -v "$REF" \ + --config "$BP" \ + --target-arch "$ARCH" \ + --rootfs "$ROOTFS" \ + --type "$TYPE" | jq > "$TMPDIR/bib.json" + + # oh well, we need to convert our blueprint into the + # gen-manifests config :/ + echo '{"blueprint":' > "$TMPDIR"/gm-conf.json + cat "$BP" >> "$TMPDIR"/gm-conf.json + echo '}' >> "$TMPDIR"/gm-conf.json + sudo ./cmd/gen-manifests/gen-manifests \ + -types "$TYPE" \ + -arches "$ARCH" \ + -distros ignore \ + -metadata=false \ + -config "$TMPDIR"/gm-conf.json \ + -bootc-refs "$REF" \ + -output "$TMPDIR" + + # ensure manifests are identical (but skip uuid diffs) + echo "Comparing images and bootc-image-builder generated manifests" + diff -u <(grep -v uuid "$TMPDIR"/bootc_*.json) <(grep -v uuid "$TMPDIR"/bib.json) + done + done +done