images: provide the ability to generate bootc disk images (HMS-8844)

cmd/gen-manifests/main.go

@@ -23,6 +23,7 @@
 	"github.com/osbuild/images/pkg/blueprint"
 	"github.com/osbuild/images/pkg/container"
 	"github.com/osbuild/images/pkg/distro"
+	"github.com/osbuild/images/pkg/distro/generic"
 	"github.com/osbuild/images/pkg/distrofactory"
 	"github.com/osbuild/images/pkg/dnfjson"
 	"github.com/osbuild/images/pkg/experimentalflags"
@@ -248,7 +249,6 @@
 	options.Facts = &facts.ImageOptions{
 		APIType: facts.TEST_APITYPE,
 	}
-
 	job := func(msgq chan string) (err error) {
 		defer func() {
 			msg := fmt.Sprintf("Finished job %s", filename)
@@ -400,11 +400,13 @@
 	flag.BoolVar(&commits, "commits", false, "resolve ostree commit IDs")
 
 	// manifest selection args
-	var arches, distros, imgTypes cmdutil.MultiValue
+	var arches, distros, imgTypes, bootcRefs cmdutil.MultiValue
 	flag.Var(&arches, "arches", "comma-separated list of architectures (globs supported)")
 	flag.Var(&distros, "distros", "comma-separated list of distributions (globs supported)")
 	flag.Var(&imgTypes, "types", "comma-separated list of image types (globs supported)")
 
+	flag.Var(&bootcRefs, "bootc-refs", "comma-separated list of bootc-refs")
+
 	flag.Parse()
 
 	testedRepoRegistry, err := testrepos.New()
@@ -496,7 +498,7 @@
 					if skipNoconfig {
 						continue
 					}
-					panic(fmt.Sprintf("no configs defined for image type %q for %s", imgTypeName, distribution.Name()))
+					panic(fmt.Sprintf("no configs defined for image type %q for %s/%s", imgTypeName, distribution.Name(), archName))
 				}
 
 				for _, itConfig := range imgTypeConfigs {
@@ -511,6 +513,36 @@
 			}
 		}
 	}
+	for _, bootcRef := range bootcRefs {
+		for _, archName := range arches {
+			for _, imgTypeName := range imgTypes {
+				imgType, err := generic.ImageFromBootc(bootcRef, imgTypeName, archName)
+				if err != nil {
+					panic(err)
+				}
+				distribution := imgType.Arch().Distro()
+
+				// XXX: copied from loop above
+				imgTypeConfigs := configs.Get(distribution.Name(), archName, imgTypeName)
+				if len(imgTypeConfigs) == 0 {
+					if skipNoconfig {
+						continue
+					}
+					panic(fmt.Sprintf("no configs defined for image type %q for %s", imgTypeName, distribution.Name()))
+				}
+				for _, itConfig := range imgTypeConfigs {
+					if needsSkipping, reason := configs.needsSkipping(distribution.Name(), itConfig); needsSkipping {
+						fmt.Printf("Skipping %s for %s/%s (reason: %v)\n", itConfig.Name, imgTypeName, distribution.Name(), reason)
+						continue
+					}
+
+					var repos []rpmmd.RepoConfig
+					job := makeManifestJob(itConfig, imgType, distribution, repos, archName, cacheRoot, outputDir, contentResolve, metadata)
+					jobs = append(jobs, job)
+				}
+			}
+		}
+	}
 
 	nJobs := len(jobs)
 	fmt.Printf("Collected %d jobs\n", nJobs)

pkg/disk/btrfs.go

@@ -19,6 +19,8 @@ type Btrfs struct {
 	Subvolumes []BtrfsSubvolume `json:"subvolumes,omitempty" yaml:"subvolumes,omitempty"`
 }
 
+var _ = MountpointCreator(&Btrfs{})
+
 func init() {
 	payloadEntityMap["btrfs"] = reflect.TypeOf(Btrfs{})
 }
@@ -58,7 +60,7 @@ func (b *Btrfs) GetItemCount() uint {
 func (b *Btrfs) GetChild(n uint) Entity {
 	return &b.Subvolumes[n]
 }
-func (b *Btrfs) CreateMountpoint(mountpoint string, size uint64) (Entity, error) {
+func (b *Btrfs) CreateMountpoint(mountpoint, defaultFs string, size uint64) (Entity, error) {
 	name := mountpoint
 	if name == "/" {
 		name = "root"

pkg/disk/disk.go

@@ -374,7 +374,7 @@ type FSTabEntity interface {
 // CreateMountpoint creates a new mountpoint with the given size and
 // returns the entity that represents the new mountpoint.
 type MountpointCreator interface {
-	CreateMountpoint(mountpoint string, size uint64) (Entity, error)
+	CreateMountpoint(mountpoint, defaultFs string, size uint64) (Entity, error)
 
 	// AlignUp will align the given bytes according to the
 	// requirements of the container type.

pkg/disk/disk_test.go

@@ -80,7 +80,7 @@ func TestDynamicallyResizePartitionTable(t *testing.T) {
 	// math/rand is good enough in this case
 	/* #nosec G404 */
 	rng := rand.New(rand.NewSource(0))
-	newpt, err := disk.NewPartitionTable(&pt, mountpoints, 1024, disk.RawPartitioningMode, arch.ARCH_AARCH64, nil, rng)
+	newpt, err := disk.NewPartitionTable(&pt, mountpoints, 1024, disk.RawPartitioningMode, arch.ARCH_AARCH64, nil, "", rng)
 	assert.NoError(t, err)
 	assert.GreaterOrEqual(t, newpt.Size, expectedSize)
 }
@@ -193,7 +193,7 @@ func TestCreatePartitionTable(t *testing.T) {
 			if ptName == "luks+lvm" {
 				ptMode = disk.AutoLVMPartitioningMode
 			}
-			mpt, err := disk.NewPartitionTable(&pt, bp, uint64(13*MiB), ptMode, arch.ARCH_PPC64LE, nil, rng)
+			mpt, err := disk.NewPartitionTable(&pt, bp, uint64(13*MiB), ptMode, arch.ARCH_PPC64LE, nil, "", rng)
 			require.NoError(t, err, "Partition table generation failed: PT %q BP %q (%s)", ptName, bpName, err)
 			assert.NotNil(mpt, "Partition table generation failed: PT %q BP %q (nil partition table)", ptName, bpName)
 			assert.Greater(mpt.GetSize(), sumSizes(bp))
@@ -217,12 +217,12 @@ func TestCreatePartitionTableLVMify(t *testing.T) {
 	for bpName, tbp := range testBlueprints {
 		for ptName, pt := range testdisk.TestPartitionTables() {
 			if tbp != nil && (ptName == "btrfs" || ptName == "luks") {
-				_, err := disk.NewPartitionTable(&pt, tbp, uint64(13*MiB), disk.AutoLVMPartitioningMode, arch.ARCH_X86_64, nil, rng)
+				_, err := disk.NewPartitionTable(&pt, tbp, uint64(13*MiB), disk.AutoLVMPartitioningMode, arch.ARCH_X86_64, nil, "", rng)
 				assert.Error(err, "PT %q BP %q: should return an error with LVMPartitioningMode", ptName, bpName)
 				continue
 			}
 
-			mpt, err := disk.NewPartitionTable(&pt, tbp, uint64(13*MiB), disk.AutoLVMPartitioningMode, arch.ARCH_X86_64, nil, rng)
+			mpt, err := disk.NewPartitionTable(&pt, tbp, uint64(13*MiB), disk.AutoLVMPartitioningMode, arch.ARCH_X86_64, nil, "", rng)
 			assert.NoError(err, "PT %q BP %q: Partition table generation failed: (%s)", ptName, bpName, err)
 
 			rootPath := disk.EntityPath(mpt, "/")
@@ -253,12 +253,12 @@ func TestCreatePartitionTableBtrfsify(t *testing.T) {
 	for bpName, tbp := range testBlueprints {
 		for ptName, pt := range testdisk.TestPartitionTables() {
 			if ptName == "auto-lvm" || ptName == "luks" || ptName == "luks+lvm" {
-				_, err := disk.NewPartitionTable(&pt, tbp, uint64(13*MiB), disk.BtrfsPartitioningMode, arch.ARCH_X86_64, nil, rng)
+				_, err := disk.NewPartitionTable(&pt, tbp, uint64(13*MiB), disk.BtrfsPartitioningMode, arch.ARCH_X86_64, nil, "", rng)
 				assert.Error(err, "PT %q BP %q: should return an error with BtrfsPartitioningMode", ptName, bpName)
 				continue
 			}
 
-			mpt, err := disk.NewPartitionTable(&pt, tbp, uint64(13*MiB), disk.BtrfsPartitioningMode, arch.ARCH_X86_64, nil, rng)
+			mpt, err := disk.NewPartitionTable(&pt, tbp, uint64(13*MiB), disk.BtrfsPartitioningMode, arch.ARCH_X86_64, nil, "", rng)
 			assert.NoError(err, "PT %q BP %q: Partition table generation failed: (%s)", ptName, bpName, err)
 
 			rootPath := disk.EntityPath(mpt, "/")
@@ -289,12 +289,12 @@ func TestCreatePartitionTableLVMOnly(t *testing.T) {
 	for bpName, tbp := range testBlueprints {
 		for ptName, pt := range testdisk.TestPartitionTables() {
 			if ptName == "btrfs" || ptName == "luks" {
-				_, err := disk.NewPartitionTable(&pt, tbp, uint64(13*MiB), disk.LVMPartitioningMode, arch.ARCH_S390X, nil, rng)
+				_, err := disk.NewPartitionTable(&pt, tbp, uint64(13*MiB), disk.LVMPartitioningMode, arch.ARCH_S390X, nil, "", rng)
 				assert.Error(err, "PT %q BP %q: should return an error with LVMPartitioningMode", ptName, bpName)
 				continue
 			}
 
-			mpt, err := disk.NewPartitionTable(&pt, tbp, uint64(13*MiB), disk.LVMPartitioningMode, arch.ARCH_S390X, nil, rng)
+			mpt, err := disk.NewPartitionTable(&pt, tbp, uint64(13*MiB), disk.LVMPartitioningMode, arch.ARCH_S390X, nil, "", rng)
 			require.NoError(t, err, "PT %q BP %q: Partition table generation failed: (%s)", ptName, bpName, err)
 
 			rootPath := disk.EntityPath(mpt, "/")
@@ -446,7 +446,7 @@ func TestMinimumSizes(t *testing.T) {
 
 	for idx, tc := range testCases {
 		{ // without LVM
-			mpt, err := disk.NewPartitionTable(&pt, tc.Blueprint, uint64(3*GiB), disk.RawPartitioningMode, arch.ARCH_X86_64, nil, rng)
+			mpt, err := disk.NewPartitionTable(&pt, tc.Blueprint, uint64(3*GiB), disk.RawPartitioningMode, arch.ARCH_X86_64, nil, "", rng)
 			assert.NoError(err)
 			for mnt, minSize := range tc.ExpectedMinSizes {
 				path := disk.EntityPath(mpt, mnt)
@@ -460,7 +460,7 @@ func TestMinimumSizes(t *testing.T) {
 		}
 
 		{ // with LVM
-			mpt, err := disk.NewPartitionTable(&pt, tc.Blueprint, uint64(3*GiB), disk.AutoLVMPartitioningMode, arch.ARCH_X86_64, nil, rng)
+			mpt, err := disk.NewPartitionTable(&pt, tc.Blueprint, uint64(3*GiB), disk.AutoLVMPartitioningMode, arch.ARCH_X86_64, nil, "", rng)
 			assert.NoError(err)
 			for mnt, minSize := range tc.ExpectedMinSizes {
 				path := disk.EntityPath(mpt, mnt)
@@ -547,7 +547,7 @@ func TestLVMExtentAlignment(t *testing.T) {
 	}
 
 	for idx, tc := range testCases {
-		mpt, err := disk.NewPartitionTable(&pt, tc.Blueprint, uint64(3*GiB), disk.AutoLVMPartitioningMode, arch.ARCH_X86_64, nil, rng)
+		mpt, err := disk.NewPartitionTable(&pt, tc.Blueprint, uint64(3*GiB), disk.AutoLVMPartitioningMode, arch.ARCH_X86_64, nil, "", rng)
 		assert.NoError(err)
 		for mnt, expSize := range tc.ExpectedSizes {
 			path := disk.EntityPath(mpt, mnt)
@@ -576,7 +576,7 @@ func TestNewBootWithSizeLVMify(t *testing.T) {
 		},
 	}
 
-	mpt, err := disk.NewPartitionTable(&pt, custom, uint64(3*GiB), disk.AutoLVMPartitioningMode, arch.ARCH_AARCH64, nil, rng)
+	mpt, err := disk.NewPartitionTable(&pt, custom, uint64(3*GiB), disk.AutoLVMPartitioningMode, arch.ARCH_AARCH64, nil, "", rng)
 	assert.NoError(err)
 
 	for idx, c := range custom {
@@ -919,7 +919,7 @@ func TestMinimumSizesWithRequiredSizes(t *testing.T) {
 
 	for idx, tc := range testCases {
 		{ // without LVM
-			mpt, err := disk.NewPartitionTable(&pt, tc.Blueprint, uint64(3*GiB), disk.RawPartitioningMode, arch.ARCH_AARCH64, map[string]uint64{"/": 1 * GiB, "/usr": 3 * GiB}, rng)
+			mpt, err := disk.NewPartitionTable(&pt, tc.Blueprint, uint64(3*GiB), disk.RawPartitioningMode, arch.ARCH_AARCH64, map[string]uint64{"/": 1 * GiB, "/usr": 3 * GiB}, "", rng)
 			assert.NoError(err)
 			for mnt, minSize := range tc.ExpectedMinSizes {
 				path := disk.EntityPath(mpt, mnt)
@@ -933,7 +933,7 @@ func TestMinimumSizesWithRequiredSizes(t *testing.T) {
 		}
 
 		{ // with LVM
-			mpt, err := disk.NewPartitionTable(&pt, tc.Blueprint, uint64(3*GiB), disk.AutoLVMPartitioningMode, arch.ARCH_AARCH64, map[string]uint64{"/": 1 * GiB, "/usr": 3 * GiB}, rng)
+			mpt, err := disk.NewPartitionTable(&pt, tc.Blueprint, uint64(3*GiB), disk.AutoLVMPartitioningMode, arch.ARCH_AARCH64, map[string]uint64{"/": 1 * GiB, "/usr": 3 * GiB}, "", rng)
 			assert.NoError(err)
 			for mnt, minSize := range tc.ExpectedMinSizes {
 				path := disk.EntityPath(mpt, mnt)

pkg/disk/lvm.go

@@ -21,6 +21,8 @@ type LVMVolumeGroup struct {
 	LogicalVolumes []LVMLogicalVolume `json:"logical_volumes,omitempty" yaml:"logical_volumes,omitempty"`
 }
 
+var _ = MountpointCreator(&LVMVolumeGroup{})
+
 func init() {
 	payloadEntityMap["lvm"] = reflect.TypeOf(LVMVolumeGroup{})
 }
@@ -73,10 +75,13 @@ func (vg *LVMVolumeGroup) GetChild(n uint) Entity {
 	return &vg.LogicalVolumes[n]
 }
 
-func (vg *LVMVolumeGroup) CreateMountpoint(mountpoint string, size uint64) (Entity, error) {
+func (vg *LVMVolumeGroup) CreateMountpoint(mountpoint, defaultFs string, size uint64) (Entity, error) {
+	if defaultFs == "" {
+		defaultFs = "xfs"
+	}
 
 	filesystem := Filesystem{
-		Type:         "xfs",
+		Type:         defaultFs,
 		Mountpoint:   mountpoint,
 		FSTabOptions: "defaults",
 		FSTabFreq:    0,

pkg/disk/lvm_test.go

@@ -19,27 +19,27 @@ func TestLVMVCreateMountpoint(t *testing.T) {
 		Description: "root volume group",
 	}
 
-	entity, err := vg.CreateMountpoint("/", 0)
+	entity, err := vg.CreateMountpoint("/", "", 0)
 	assert.NoError(err)
 	rootlv := entity.(*LVMLogicalVolume)
 	assert.Equal("rootlv", rootlv.Name)
 
-	_, err = vg.CreateMountpoint("/home_test", 0)
+	_, err = vg.CreateMountpoint("/home_test", "", 0)
 	assert.NoError(err)
 
-	entity, err = vg.CreateMountpoint("/home/test", 0)
+	entity, err = vg.CreateMountpoint("/home/test", "", 0)
 	assert.NoError(err)
 
 	dedup := entity.(*LVMLogicalVolume)
 	assert.Equal("home_testlv00", dedup.Name)
 
 	// Lets collide it
 	for i := 0; i < 99; i++ {
-		_, err = vg.CreateMountpoint("/home/test", 0)
+		_, err = vg.CreateMountpoint("/home/test", "", 0)
 		assert.NoError(err)
 	}
 
-	_, err = vg.CreateMountpoint("/home/test", 0)
+	_, err = vg.CreateMountpoint("/home/test", "", 0)
 	assert.Error(err)
 }
 

pkg/disk/partition_table.go

@@ -31,6 +31,8 @@ type PartitionTable struct {
 	StartOffset uint64 `json:"start_offset,omitempty" yaml:"start_offset,omitempty"`
 }
 
+var _ = MountpointCreator(&PartitionTable{})
+
 type PartitioningMode string
 
 const (
@@ -110,15 +112,15 @@ const DefaultBootPartitionSize = 1 * datasizes.GiB
 // containing the root filesystem is grown to fill any left over space on the
 // partition table. Logical Volumes are not grown to fill the space in the
 // Volume Group since they are trivial to grow on a live system.
-func NewPartitionTable(basePT *PartitionTable, mountpoints []blueprint.FilesystemCustomization, imageSize uint64, mode PartitioningMode, architecture arch.Arch, requiredSizes map[string]uint64, rng *rand.Rand) (*PartitionTable, error) {
+func NewPartitionTable(basePT *PartitionTable, mountpoints []blueprint.FilesystemCustomization, imageSize uint64, mode PartitioningMode, architecture arch.Arch, requiredSizes map[string]uint64, defaultFs string, rng *rand.Rand) (*PartitionTable, error) {
 	newPT := basePT.Clone().(*PartitionTable)
 
 	if basePT.features().LVM && (mode == RawPartitioningMode || mode == BtrfsPartitioningMode) {
 		return nil, fmt.Errorf("%s partitioning mode set for a base partition table with LVM, this is unsupported", mode)
 	}
 
 	// first pass: enlarge existing mountpoints and collect new ones
-	newMountpoints, _ := newPT.applyCustomization(mountpoints, false)
+	newMountpoints, _ := newPT.applyCustomization(mountpoints, defaultFs, false)
 
 	var ensureLVM, ensureBtrfs bool
 	switch mode {
@@ -147,7 +149,7 @@ func NewPartitionTable(basePT *PartitionTable, mountpoints []blueprint.Filesyste
 
 	// second pass: deal with new mountpoints and newly created ones, after switching to
 	// the LVM layout, if requested, which might introduce new mount points, i.e. `/boot`
-	_, err := newPT.applyCustomization(newMountpoints, true)
+	_, err := newPT.applyCustomization(newMountpoints, defaultFs, true)
 	if err != nil {
 		return nil, err
 	}
@@ -356,9 +358,12 @@ func (pt *PartitionTable) EnsureDirectorySizes(dirSizeMap map[string]uint64) {
 	}
 }
 
-func (pt *PartitionTable) CreateMountpoint(mountpoint string, size uint64) (Entity, error) {
+func (pt *PartitionTable) CreateMountpoint(mountpoint, defaultFs string, size uint64) (Entity, error) {
+	if defaultFs == "" {
+		defaultFs = "xfs"
+	}
 	filesystem := Filesystem{
-		Type:         "xfs",
+		Type:         defaultFs,
 		Mountpoint:   mountpoint,
 		FSTabOptions: "defaults",
 		FSTabFreq:    0,
@@ -457,7 +462,7 @@ func (pt *PartitionTable) HeaderSize() uint64 {
 // return a list of left-over mountpoints (i.e. mountpoints in the input that
 // were not created). An error can only occur if create is set.
 // Does not relayout the table, i.e. a call to relayout might be needed.
-func (pt *PartitionTable) applyCustomization(mountpoints []blueprint.FilesystemCustomization, create bool) ([]blueprint.FilesystemCustomization, error) {
+func (pt *PartitionTable) applyCustomization(mountpoints []blueprint.FilesystemCustomization, defaultFs string, create bool) ([]blueprint.FilesystemCustomization, error) {
 
 	newMountpoints := []blueprint.FilesystemCustomization{}
 
@@ -469,7 +474,7 @@ func (pt *PartitionTable) applyCustomization(mountpoints []blueprint.FilesystemC
 		} else {
 			if !create {
 				newMountpoints = append(newMountpoints, mnt)
-			} else if err := pt.createFilesystem(mnt.Mountpoint, size); err != nil {
+			} else if err := pt.createFilesystem(mnt.Mountpoint, defaultFs, size); err != nil {
 				return nil, err
 			}
 		}
@@ -543,7 +548,7 @@ func (pt *PartitionTable) relayout(size uint64) uint64 {
 	return start
 }
 
-func (pt *PartitionTable) createFilesystem(mountpoint string, size uint64) error {
+func (pt *PartitionTable) createFilesystem(mountpoint, defaultFs string, size uint64) error {
 	rootPath := entityPath(pt, "/")
 	if rootPath == nil {
 		panic("no root mountpoint for PartitionTable")
@@ -563,7 +568,7 @@ func (pt *PartitionTable) createFilesystem(mountpoint string, size uint64) error
 		panic("could not find root volume container")
 	}
 
-	newVol, err := vc.CreateMountpoint(mountpoint, 0)
+	newVol, err := vc.CreateMountpoint(mountpoint, defaultFs, 0)
 	if err != nil {
 		return fmt.Errorf("failed creating volume: %w", err)
 	}
@@ -752,7 +757,7 @@ func (pt *PartitionTable) ensureLVM() error {
 	// we need a /boot partition to boot LVM, ensure one exists
 	bootPath := entityPath(pt, "/boot")
 	if bootPath == nil {
-		_, err := pt.CreateMountpoint("/boot", DefaultBootPartitionSize)
+		_, err := pt.CreateMountpoint("/boot", "", DefaultBootPartitionSize)
 
 		if err != nil {
 			return err
@@ -811,13 +816,22 @@ func (pt *PartitionTable) ensureBtrfs(architecture arch.Arch) error {
 	// we need a /boot partition to boot btrfs, ensure one exists
 	bootPath := entityPath(pt, "/boot")
 	if bootPath == nil {
-		_, err := pt.CreateMountpoint("/boot", DefaultBootPartitionSize)
+		_, err := pt.CreateMountpoint("/boot", "", DefaultBootPartitionSize)
 		if err != nil {
 			return fmt.Errorf("failed to create /boot partition when ensuring btrfs: %w", err)
 		}
 
 		rootPath = entityPath(pt, "/")
 	}
+	// ensure /boot is not set to "btrfs", this can happen on e.g. bootc partition tables that
+	// do not declare a default fs
+	bootFilesystem, ok := bootPath[0].(*Filesystem)
+	if !ok {
+		return fmt.Errorf("internal error: boot entity not a filesystem: %T", bootPath[0])
+	}
+	if bootFilesystem.Type == "btrfs" {
+		bootFilesystem.Type = "ext4"
+	}
 
 	parent := rootPath[1] // NB: entityPath has reversed order