diff --git a/bib/cmd/bootc-image-builder/image.go b/bib/cmd/bootc-image-builder/legacy_iso.go similarity index 61% rename from bib/cmd/bootc-image-builder/image.go rename to bib/cmd/bootc-image-builder/legacy_iso.go index 78d3b9d7e..cff937c44 100644 --- a/bib/cmd/bootc-image-builder/image.go +++ b/bib/cmd/bootc-image-builder/legacy_iso.go @@ -1,10 +1,7 @@ package main import ( - cryptorand "crypto/rand" "fmt" - "math" - "math/big" "math/rand" "slices" "strconv" @@ -16,6 +13,7 @@ import ( "github.com/osbuild/images/pkg/container" "github.com/osbuild/images/pkg/customizations/anaconda" "github.com/osbuild/images/pkg/customizations/kickstart" + "github.com/osbuild/images/pkg/depsolvednf" "github.com/osbuild/images/pkg/disk" "github.com/osbuild/images/pkg/image" "github.com/osbuild/images/pkg/manifest" @@ -25,17 +23,25 @@ import ( "github.com/osbuild/images/pkg/runner" "github.com/sirupsen/logrus" + podman_container "github.com/osbuild/images/pkg/bib/container" + "github.com/osbuild/bootc-image-builder/bib/internal/distrodef" - "github.com/osbuild/bootc-image-builder/bib/internal/imagetypes" ) +// all possible locations for the bib's distro definitions +// ./data/defs and ./bib/data/defs are for development +// /usr/share/bootc-image-builder/defs is for the production, containerized version +var distroDefPaths = []string{ + "./data/defs", + "./bib/data/defs", + "/usr/share/bootc-image-builder/defs", +} + type ManifestConfig struct { // OCI image path (without the transport, that is always docker://) Imgref string BuildImgref string - ImageTypes imagetypes.ImageTypes - // Build config Config *blueprint.Blueprint @@ -56,6 +62,155 @@ type ManifestConfig struct { UseLibrepo bool } +func manifestFromCobraForLegacyISO(imgref, buildImgref, imgTypeStr, rootFs, rpmCacheRoot string, config *blueprint.Blueprint, useLibrepo bool, cntArch arch.Arch) ([]byte, *mTLSConfig, error) { + container, err := podman_container.New(imgref) + if err != nil { + return nil, nil, err + } + defer func() { + if err := container.Stop(); err != nil { + logrus.Warnf("error stopping container: %v", err) + } + }() + + var rootfsType string + if rootFs != "" { + rootfsType = rootFs + } else { + rootfsType, err = container.DefaultRootfsType() + if err != nil { + return nil, nil, fmt.Errorf("cannot get rootfs type for container: %w", err) + } + if rootfsType == "" { + return nil, nil, fmt.Errorf(`no default root filesystem type specified in container, please use "--rootfs" to set manually`) + } + } + + // Gather some data from the containers distro + sourceinfo, err := osinfo.Load(container.Root()) + if err != nil { + return nil, nil, err + } + + buildContainer := container + buildSourceinfo := sourceinfo + startedBuildContainer := false + defer func() { + if startedBuildContainer { + if err := buildContainer.Stop(); err != nil { + logrus.Warnf("error stopping container: %v", err) + } + } + }() + + if buildImgref != "" { + buildContainer, err = podman_container.New(buildImgref) + if err != nil { + return nil, nil, err + } + startedBuildContainer = true + + // Gather some data from the containers distro + buildSourceinfo, err = osinfo.Load(buildContainer.Root()) + if err != nil { + return nil, nil, err + } + } else { + buildImgref = imgref + } + + // This is needed just for RHEL and RHSM in most cases, but let's run it every time in case + // the image has some non-standard dnf plugins. + if err := buildContainer.InitDNF(); err != nil { + return nil, nil, err + } + solver, err := buildContainer.NewContainerSolver(rpmCacheRoot, cntArch, sourceinfo) + if err != nil { + return nil, nil, err + } + + manifestConfig := &ManifestConfig{ + Architecture: cntArch, + Config: config, + Imgref: imgref, + BuildImgref: buildImgref, + DistroDefPaths: distroDefPaths, + SourceInfo: sourceinfo, + BuildSourceInfo: buildSourceinfo, + RootFSType: rootfsType, + UseLibrepo: useLibrepo, + } + + manifest, repos, err := makeISOManifest(manifestConfig, solver, rpmCacheRoot) + if err != nil { + return nil, nil, err + } + + mTLS, err := extractTLSKeys(repos) + if err != nil { + return nil, nil, err + } + + return manifest, mTLS, nil +} + +func makeISOManifest(c *ManifestConfig, solver *depsolvednf.Solver, cacheRoot string) (manifest.OSBuildManifest, map[string][]rpmmd.RepoConfig, error) { + rng := createRand() + mani, err := manifestForISO(c, rng) + if err != nil { + return nil, nil, fmt.Errorf("cannot get manifest: %w", err) + } + + // depsolve packages + depsolvedSets := make(map[string]depsolvednf.DepsolveResult) + depsolvedRepos := make(map[string][]rpmmd.RepoConfig) + for name, pkgSet := range mani.GetPackageSetChains() { + res, err := solver.Depsolve(pkgSet, 0) + if err != nil { + return nil, nil, fmt.Errorf("cannot depsolve: %w", err) + } + depsolvedSets[name] = *res + depsolvedRepos[name] = res.Repos + } + + // Resolve container - the normal case is that host and target + // architecture are the same. However it is possible to build + // cross-arch images by using qemu-user. This will run everything + // (including the build-root) with the target arch then, it + // is fast enough (given that it's mostly I/O and all I/O is + // run naively via syscall translation) + + // XXX: should NewResolver() take "arch.Arch"? + resolver := container.NewResolver(c.Architecture.String()) + + containerSpecs := make(map[string][]container.Spec) + for plName, sourceSpecs := range mani.GetContainerSourceSpecs() { + for _, c := range sourceSpecs { + resolver.Add(c) + } + specs, err := resolver.Finish() + if err != nil { + return nil, nil, fmt.Errorf("cannot resolve containers: %w", err) + } + for _, spec := range specs { + if spec.Arch != c.Architecture { + return nil, nil, fmt.Errorf("image found is for unexpected architecture %q (expected %q), if that is intentional, please make sure --target-arch matches", spec.Arch, c.Architecture) + } + } + containerSpecs[plName] = specs + } + + var opts manifest.SerializeOptions + if c.UseLibrepo { + opts.RpmDownloader = osbuild.RpmDownloaderLibrepo + } + mf, err := mani.Serialize(depsolvedSets, containerSpecs, nil, &opts) + if err != nil { + return nil, nil, fmt.Errorf("[ERROR] manifest serialization failed: %s", err.Error()) + } + return mf, depsolvedRepos, nil +} + func labelForISO(os *osinfo.OSRelease, arch *arch.Arch) string { switch os.ID { case "fedora": @@ -256,14 +411,3 @@ func getDistroAndRunner(osRelease osinfo.OSRelease) (manifest.Distro, runner.Run logrus.Warnf("Unknown distro %s, using default runner", osRelease.ID) return manifest.DISTRO_NULL, &runner.Linux{}, nil } - -func createRand() *rand.Rand { - seed, err := cryptorand.Int(cryptorand.Reader, big.NewInt(math.MaxInt64)) - if err != nil { - panic("Cannot generate an RNG seed.") - } - - // math/rand is good enough in this case - /* #nosec G404 */ - return rand.New(rand.NewSource(seed.Int64())) -} diff --git a/bib/cmd/bootc-image-builder/main.go b/bib/cmd/bootc-image-builder/main.go index 357b7dce8..4351b1a5e 100644 --- a/bib/cmd/bootc-image-builder/main.go +++ b/bib/cmd/bootc-image-builder/main.go @@ -20,37 +20,24 @@ import ( "github.com/spf13/pflag" "golang.org/x/exp/slices" + "github.com/osbuild/blueprint/pkg/blueprint" repos "github.com/osbuild/images/data/repositories" "github.com/osbuild/images/pkg/arch" "github.com/osbuild/images/pkg/bib/blueprintload" "github.com/osbuild/images/pkg/cloud" "github.com/osbuild/images/pkg/cloud/awscloud" - "github.com/osbuild/images/pkg/container" "github.com/osbuild/images/pkg/distro/bootc" - "github.com/osbuild/images/pkg/depsolvednf" "github.com/osbuild/images/pkg/experimentalflags" "github.com/osbuild/images/pkg/manifest" "github.com/osbuild/images/pkg/manifestgen" - "github.com/osbuild/images/pkg/osbuild" "github.com/osbuild/images/pkg/reporegistry" "github.com/osbuild/images/pkg/rpmmd" - "github.com/osbuild/bootc-image-builder/bib/internal/imagetypes" - podman_container "github.com/osbuild/images/pkg/bib/container" - "github.com/osbuild/images/pkg/bib/osinfo" - "github.com/osbuild/image-builder-cli/pkg/progress" "github.com/osbuild/image-builder-cli/pkg/setup" -) -// all possible locations for the bib's distro definitions -// ./data/defs and ./bib/data/defs are for development -// /usr/share/bootc-image-builder/defs is for the production, containerized version -var distroDefPaths = []string{ - "./data/defs", - "./bib/data/defs", - "/usr/share/bootc-image-builder/defs", -} + "github.com/osbuild/bootc-image-builder/bib/internal/imagetypes" +) var ( osGetuid = os.Getuid @@ -60,29 +47,6 @@ var ( osStderr = os.Stderr ) -// canChownInPath checks if the ownership of files can be set in a given path. -func canChownInPath(path string) (bool, error) { - info, err := os.Stat(path) - if err != nil { - return false, err - } - if !info.IsDir() { - return false, fmt.Errorf("%s is not a directory", path) - } - - checkFile, err := os.CreateTemp(path, ".writecheck") - if err != nil { - return false, err - } - defer func() { - if err := os.Remove(checkFile.Name()); err != nil { - // print the error message for info but don't error out - fmt.Fprintf(os.Stderr, "error deleting %s: %s\n", checkFile.Name(), err.Error()) - } - }() - return checkFile.Chown(osGetuid(), osGetgid()) == nil, nil -} - func inContainerOrUnknown() bool { // no systemd-detect-virt, err on the side of container if _, err := exec.LookPath("systemd-detect-virt"); err != nil { @@ -93,63 +57,6 @@ func inContainerOrUnknown() bool { return err == nil } -func makeManifest(c *ManifestConfig, solver *depsolvednf.Solver, cacheRoot string) (manifest.OSBuildManifest, map[string][]rpmmd.RepoConfig, error) { - rng := createRand() - mani, err := manifestForISO(c, rng) - if err != nil { - return nil, nil, fmt.Errorf("cannot get manifest: %w", err) - } - - // depsolve packages - depsolvedSets := make(map[string]depsolvednf.DepsolveResult) - depsolvedRepos := make(map[string][]rpmmd.RepoConfig) - for name, pkgSet := range mani.GetPackageSetChains() { - res, err := solver.Depsolve(pkgSet, 0) - if err != nil { - return nil, nil, fmt.Errorf("cannot depsolve: %w", err) - } - depsolvedSets[name] = *res - depsolvedRepos[name] = res.Repos - } - - // Resolve container - the normal case is that host and target - // architecture are the same. However it is possible to build - // cross-arch images by using qemu-user. This will run everything - // (including the build-root) with the target arch then, it - // is fast enough (given that it's mostly I/O and all I/O is - // run naively via syscall translation) - - // XXX: should NewResolver() take "arch.Arch"? - resolver := container.NewResolver(c.Architecture.String()) - - containerSpecs := make(map[string][]container.Spec) - for plName, sourceSpecs := range mani.GetContainerSourceSpecs() { - for _, c := range sourceSpecs { - resolver.Add(c) - } - specs, err := resolver.Finish() - if err != nil { - return nil, nil, fmt.Errorf("cannot resolve containers: %w", err) - } - for _, spec := range specs { - if spec.Arch != c.Architecture { - return nil, nil, fmt.Errorf("image found is for unexpected architecture %q (expected %q), if that is intentional, please make sure --target-arch matches", spec.Arch, c.Architecture) - } - } - containerSpecs[plName] = specs - } - - var opts manifest.SerializeOptions - if c.UseLibrepo { - opts.RpmDownloader = osbuild.RpmDownloaderLibrepo - } - mf, err := mani.Serialize(depsolvedSets, containerSpecs, nil, &opts) - if err != nil { - return nil, nil, fmt.Errorf("[ERROR] manifest serialization failed: %s", err.Error()) - } - return mf, depsolvedRepos, nil -} - func saveManifest(ms manifest.OSBuildManifest, fpath string) (err error) { b, err := json.MarshalIndent(ms, "", " ") if err != nil { @@ -240,145 +147,59 @@ func manifestFromCobra(cmd *cobra.Command, args []string, pbar progress.Progress pbar.SetPulseMsgf("Manifest generation step") pbar.Start() - // For now shortcut here and build ding "images" for anything - // that is not the iso - if !imageTypes.BuildsISO() { - distro, err := bootc.NewBootcDistro(imgref) - if err != nil { - return nil, nil, err - } - if err := distro.SetBuildContainer(buildImgref); err != nil { - return nil, nil, err - } - if err := distro.SetDefaultFs(rootFs); err != nil { - return nil, nil, err - } - // XXX: consider target-arch - archi, err := distro.GetArch(cntArch.String()) - if err != nil { - return nil, nil, err - } - // XXX: how to generate for all image types - imgType, err := archi.GetImageType(imgTypes[0]) - if err != nil { - return nil, nil, err - } - - var buf bytes.Buffer - repos, err := reporegistry.New(nil, []fs.FS{repos.FS}) - if err != nil { - return nil, nil, err - } - mg, err := manifestgen.New(repos, &manifestgen.Options{ - Output: &buf, - // XXX: hack to skip repo loading for the bootc image. - // We need to add a SkipRepositories or similar to - // manifestgen instead to make this clean - OverrideRepos: []rpmmd.RepoConfig{ - { - BaseURLs: []string{"https://example.com/not-used"}, - }, - }, - }) - if err != nil { - return nil, nil, err - } - if err := mg.Generate(config, distro, imgType, archi, nil); err != nil { - return nil, nil, err - } - return buf.Bytes(), nil, nil + // Note that we only need to pass a single imgType here into the manifest generation because: + // 1. the bootc disk manifests contains exports for all supported image types + // 2. the bootc legacy types (iso, anaconda-iso) always do a single build + imgType := imgTypes[0] + if imageTypes.Legacy() { + return manifestFromCobraForLegacyISO(imgref, buildImgref, imgType, rootFs, rpmCacheRoot, config, useLibrepo, cntArch) } + return manifestFromCobraForDisk(imgref, buildImgref, imgType, rootFs, rpmCacheRoot, config, useLibrepo, cntArch) +} - container, err := podman_container.New(imgref) +func manifestFromCobraForDisk(imgref, buildImgref, imgTypeStr, rootFs, rpmCacheRoot string, config *blueprint.Blueprint, useLibrepo bool, cntArch arch.Arch) ([]byte, *mTLSConfig, error) { + distro, err := bootc.NewBootcDistro(imgref) if err != nil { return nil, nil, err } - defer func() { - if err := container.Stop(); err != nil { - logrus.Warnf("error stopping container: %v", err) - } - }() - - var rootfsType string - if rootFs != "" { - rootfsType = rootFs - } else { - rootfsType, err = container.DefaultRootfsType() - if err != nil { - return nil, nil, fmt.Errorf("cannot get rootfs type for container: %w", err) - } - if rootfsType == "" { - return nil, nil, fmt.Errorf(`no default root filesystem type specified in container, please use "--rootfs" to set manually`) - } - } - - // Gather some data from the containers distro - sourceinfo, err := osinfo.Load(container.Root()) - if err != nil { + if err := distro.SetBuildContainer(buildImgref); err != nil { return nil, nil, err } - - buildContainer := container - buildSourceinfo := sourceinfo - startedBuildContainer := false - defer func() { - if startedBuildContainer { - if err := buildContainer.Stop(); err != nil { - logrus.Warnf("error stopping container: %v", err) - } - } - }() - - if buildImgref != "" { - buildContainer, err = podman_container.New(buildImgref) - if err != nil { - return nil, nil, err - } - startedBuildContainer = true - - // Gather some data from the containers distro - buildSourceinfo, err = osinfo.Load(buildContainer.Root()) - if err != nil { - return nil, nil, err - } - } else { - buildImgref = imgref - } - - // This is needed just for RHEL and RHSM in most cases, but let's run it every time in case - // the image has some non-standard dnf plugins. - if err := buildContainer.InitDNF(); err != nil { + if err := distro.SetDefaultFs(rootFs); err != nil { return nil, nil, err } - solver, err := buildContainer.NewContainerSolver(rpmCacheRoot, cntArch, sourceinfo) + archi, err := distro.GetArch(cntArch.String()) if err != nil { return nil, nil, err } - - manifestConfig := &ManifestConfig{ - Architecture: cntArch, - Config: config, - ImageTypes: imageTypes, - Imgref: imgref, - BuildImgref: buildImgref, - DistroDefPaths: distroDefPaths, - SourceInfo: sourceinfo, - BuildSourceInfo: buildSourceinfo, - RootFSType: rootfsType, - UseLibrepo: useLibrepo, + imgType, err := archi.GetImageType(imgTypeStr) + if err != nil { + return nil, nil, err } - manifest, repos, err := makeManifest(manifestConfig, solver, rpmCacheRoot) + var buf bytes.Buffer + repos, err := reporegistry.New(nil, []fs.FS{repos.FS}) if err != nil { return nil, nil, err } - - mTLS, err := extractTLSKeys(repos) + mg, err := manifestgen.New(repos, &manifestgen.Options{ + Output: &buf, + // XXX: hack to skip repo loading for the bootc image. + // We need to add a SkipRepositories or similar to + // manifestgen instead to make this clean + OverrideRepos: []rpmmd.RepoConfig{ + { + BaseURLs: []string{"https://example.com/not-used"}, + }, + }, + }) if err != nil { return nil, nil, err } - - return manifest, mTLS, nil + if err := mg.Generate(config, distro, imgType, archi, nil); err != nil { + return nil, nil, err + } + return buf.Bytes(), nil, nil } func cmdManifest(cmd *cobra.Command, args []string) error { @@ -562,35 +383,6 @@ func cmdBuild(cmd *cobra.Command, args []string) error { return nil } -func chownR(path string, chown string) error { - if chown == "" { - return nil - } - errFmt := "cannot parse chown: %v" - - var gid int - uidS, gidS, _ := strings.Cut(chown, ":") - uid, err := strconv.Atoi(uidS) - if err != nil { - return fmt.Errorf(errFmt, err) - } - if gidS != "" { - gid, err = strconv.Atoi(gidS) - if err != nil { - return fmt.Errorf(errFmt, err) - } - } else { - gid = osGetgid() - } - - return filepath.Walk(path, func(name string, info os.FileInfo, err error) error { - if err == nil { - err = os.Chown(name, uid, gid) - } - return err - }) -} - var rootLogLevel string func rootPreRunE(cmd *cobra.Command, _ []string) error { diff --git a/bib/cmd/bootc-image-builder/util.go b/bib/cmd/bootc-image-builder/util.go new file mode 100644 index 000000000..d6b2bd989 --- /dev/null +++ b/bib/cmd/bootc-image-builder/util.go @@ -0,0 +1,76 @@ +package main + +import ( + cryptorand "crypto/rand" + "fmt" + "math" + "math/big" + "math/rand" + "os" + "path/filepath" + "strconv" + "strings" +) + +// canChownInPath checks if the ownership of files can be set in a given path. +func canChownInPath(path string) (bool, error) { + info, err := os.Stat(path) + if err != nil { + return false, err + } + if !info.IsDir() { + return false, fmt.Errorf("%s is not a directory", path) + } + + checkFile, err := os.CreateTemp(path, ".writecheck") + if err != nil { + return false, err + } + defer func() { + if err := os.Remove(checkFile.Name()); err != nil { + // print the error message for info but don't error out + fmt.Fprintf(os.Stderr, "error deleting %s: %s\n", checkFile.Name(), err.Error()) + } + }() + return checkFile.Chown(osGetuid(), osGetgid()) == nil, nil +} + +func chownR(path string, chown string) error { + if chown == "" { + return nil + } + errFmt := "cannot parse chown: %v" + + var gid int + uidS, gidS, _ := strings.Cut(chown, ":") + uid, err := strconv.Atoi(uidS) + if err != nil { + return fmt.Errorf(errFmt, err) + } + if gidS != "" { + gid, err = strconv.Atoi(gidS) + if err != nil { + return fmt.Errorf(errFmt, err) + } + } else { + gid = osGetgid() + } + + return filepath.Walk(path, func(name string, info os.FileInfo, err error) error { + if err == nil { + err = os.Chown(name, uid, gid) + } + return err + }) +} + +func createRand() *rand.Rand { + seed, err := cryptorand.Int(cryptorand.Reader, big.NewInt(math.MaxInt64)) + if err != nil { + panic("Cannot generate an RNG seed.") + } + + // math/rand is good enough in this case + /* #nosec G404 */ + return rand.New(rand.NewSource(seed.Int64())) +} diff --git a/bib/internal/imagetypes/imagetypes.go b/bib/internal/imagetypes/imagetypes.go index 8e788ae6b..de396ce4b 100644 --- a/bib/internal/imagetypes/imagetypes.go +++ b/bib/internal/imagetypes/imagetypes.go @@ -10,17 +10,22 @@ import ( type imageType struct { Export string ISO bool + Legacy bool } var supportedImageTypes = map[string]imageType{ - "ami": imageType{Export: "image"}, - "qcow2": imageType{Export: "qcow2"}, - "raw": imageType{Export: "image"}, - "vmdk": imageType{Export: "vmdk"}, - "vhd": imageType{Export: "vpc"}, - "gce": imageType{Export: "gce"}, - "anaconda-iso": imageType{Export: "bootiso", ISO: true}, - "iso": imageType{Export: "bootiso", ISO: true}, + // XXX: ideally we would look how to consolidate all + // knownledge about disk based image types into the images + // library + "ami": imageType{Export: "image"}, + "qcow2": imageType{Export: "qcow2"}, + "raw": imageType{Export: "image"}, + "vmdk": imageType{Export: "vmdk"}, + "vhd": imageType{Export: "vpc"}, + "gce": imageType{Export: "gce"}, + // the iso image types are RPM based and legacy/deprecated + "anaconda-iso": imageType{Export: "bootiso", ISO: true, Legacy: true}, + "iso": imageType{Export: "bootiso", ISO: true, Legacy: true}, } // Available() returns a comma-separated list of supported image types @@ -86,3 +91,12 @@ func (it ImageTypes) BuildsISO() bool { // XXX: this assumes a valid ImagTypes object return supportedImageTypes[it[0]].ISO } + +func (it ImageTypes) Legacy() bool { + for _, name := range it { + if supportedImageTypes[name].Legacy { + return true + } + } + return false +}