Merge pull request #6442 from nalind/metadata-file

Introduce CommitResults(), add --metadata-file

Author: openshift-merge-bot[bot]

cmd/buildah/commit.go

@@ -69,6 +69,7 @@ type commitInputOptions struct {
 	unsetAnnotation    []string
 	annotation         []string
 	createdAnnotation  bool
+	metadataFile       string
 }
 
 func init() {
@@ -123,6 +124,9 @@ func commitListFlagSet(cmd *cobra.Command, opts *commitInputOptions) {
 	_ = cmd.RegisterFlagCompletionFunc("manifest", completion.AutocompleteNone)
 	flags.StringVar(&opts.iidfile, "iidfile", "", "write the image ID to the file")
 	_ = cmd.RegisterFlagCompletionFunc("iidfile", completion.AutocompleteDefault)
+	flags.StringVar(&opts.metadataFile, "metadata-file", "", "`file` to write metadata about the image to")
+	_ = cmd.RegisterFlagCompletionFunc("metadata-file", completion.AutocompleteDefault)
+
 	flags.BoolVar(&opts.omitTimestamp, "omit-timestamp", false, "set created timestamp to epoch 0 to allow for deterministic builds")
 	sourceDateEpochUsageDefault := "current time"
 	if v := os.Getenv(internal.SourceDateEpochName); v != "" {
@@ -401,10 +405,12 @@ func commitCmd(c *cobra.Command, args []string, iopts commitInputOptions) error
 	if !iopts.quiet {
 		options.ReportWriter = os.Stderr
 	}
-	id, ref, _, err := builder.Commit(ctx, dest, options)
+	results, err := builder.CommitResults(ctx, dest, options)
 	if err != nil {
 		return util.GetFailureCause(err, fmt.Errorf("committing container %q to %q: %w", builder.Container, image, err))
 	}
+	ref := results.Canonical
+	id := results.ImageID
 	if ref != nil && id != "" {
 		logrus.Debugf("wrote image %s with ID %s", ref, id)
 	} else if ref != nil {
@@ -417,6 +423,15 @@ func commitCmd(c *cobra.Command, args []string, iopts commitInputOptions) error
 	if options.IIDFile == "" && id != "" {
 		fmt.Printf("%s\n", id)
 	}
+	if iopts.metadataFile != "" {
+		metadataBytes, err := json.Marshal(results.Metadata)
+		if err != nil {
+			return fmt.Errorf("encoding contents for %q: %w", iopts.metadataFile, err)
+		}
+		if err := os.WriteFile(iopts.metadataFile, metadataBytes, 0o644); err != nil {
+			return err
+		}
+	}
 
 	if iopts.rm {
 		return builder.Delete()

commit.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/containers/buildah/internal/metadata"
 	"github.com/containers/buildah/pkg/blobcache"
 	"github.com/containers/buildah/util"
 	encconfig "github.com/containers/ocicrypt/config"
@@ -76,7 +77,8 @@ type CommitOptions struct {
 	// github.com/containers/image/types SystemContext to hold credentials
 	// and other authentication/authorization information.
 	SystemContext *types.SystemContext
-	// IIDFile tells the builder to write the image ID to the specified file
+	// IIDFile tells the builder to write the image's ID, preceded by
+	// "sha256:", to the specified file.
 	IIDFile string
 	// Squash tells the builder to produce an image with a single layer
 	// instead of with possibly more than one layer.
@@ -304,11 +306,36 @@ func (b *Builder) addManifest(ctx context.Context, manifestName string, imageSpe
 	return imageID, err
 }
 
+// CommitResults is a structure returned when CommitResults() succeeds.
+type CommitResults struct {
+	ImageID       string              // a local image ID, or part of the digest of the image's config blob
+	Canonical     reference.Canonical // set if destination included a DockerReference
+	MediaType     string              // image manifest MIME type, always returned
+	ImageManifest []byte              // raw image manifest, always returned
+	Digest        digest.Digest       // digest of the manifest, always returned
+	Metadata      map[string]any      // always returned, format is flexible
+}
+
 // Commit writes the contents of the container, along with its updated
 // configuration, to a new image in the specified location, and if we know how,
 // add any additional tags that were specified. Returns the ID of the new image
-// if commit was successful and the image destination was local.
+// if commit was successful and the image destination was local, a canonical
+// reference if the destination ImageReference include a DockerReference, and
+// the digest of the written image's manifest.
+// Commit() is implemented as a wrapper around CommitResults().
 func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options CommitOptions) (string, reference.Canonical, digest.Digest, error) {
+	results, err := b.CommitResults(ctx, dest, options)
+	if err != nil {
+		return "", nil, "", err
+	}
+	return results.ImageID, results.Canonical, results.Digest, nil
+}
+
+// CommitResults writes the contents of the container, along with its updated
+// configuration, to a new image in the specified location, and if we know how,
+// add any additional tags that were specified.  Returns a CommitResults
+// structure.
+func (b *Builder) CommitResults(ctx context.Context, dest types.ImageReference, options CommitOptions) (*CommitResults, error) {
 	var (
 		imgID                string
 		src                  types.ImageReference
@@ -325,7 +352,7 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 	// work twice.
 	if options.OmitTimestamp {
 		if options.HistoryTimestamp != nil {
-			return imgID, nil, "", fmt.Errorf("OmitTimestamp and HistoryTimestamp can not be used together")
+			return nil, fmt.Errorf("OmitTimestamp and HistoryTimestamp can not be used together")
 		}
 		timestamp := time.Unix(0, 0).UTC()
 		options.HistoryTimestamp = &timestamp
@@ -339,7 +366,7 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 		nameToRemove = stringid.GenerateRandomID() + "-tmp"
 		dest2, err := is.Transport.ParseStoreReference(b.store, nameToRemove)
 		if err != nil {
-			return imgID, nil, "", fmt.Errorf("creating temporary destination reference for image: %w", err)
+			return nil, fmt.Errorf("creating temporary destination reference for image: %w", err)
 		}
 		dest = dest2
 	}
@@ -348,23 +375,23 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 
 	blocked, err := isReferenceBlocked(dest, systemContext)
 	if err != nil {
-		return "", nil, "", fmt.Errorf("checking if committing to registry for %q is blocked: %w", transports.ImageName(dest), err)
+		return nil, fmt.Errorf("checking if committing to registry for %q is blocked: %w", transports.ImageName(dest), err)
 	}
 	if blocked {
-		return "", nil, "", fmt.Errorf("commit access to registry for %q is blocked by configuration", transports.ImageName(dest))
+		return nil, fmt.Errorf("commit access to registry for %q is blocked by configuration", transports.ImageName(dest))
 	}
 
 	// Load the system signing policy.
 	commitPolicy, err := signature.DefaultPolicy(systemContext)
 	if err != nil {
-		return "", nil, "", fmt.Errorf("obtaining default signature policy: %w", err)
+		return nil, fmt.Errorf("obtaining default signature policy: %w", err)
 	}
 	// Override the settings for local storage to make sure that we can always read the source "image".
 	commitPolicy.Transports[is.Transport.Name()] = storageAllowedPolicyScopes
 
 	policyContext, err := signature.NewPolicyContext(commitPolicy)
 	if err != nil {
-		return imgID, nil, "", fmt.Errorf("creating new signature policy context: %w", err)
+		return nil, fmt.Errorf("creating new signature policy context: %w", err)
 	}
 	defer func() {
 		if err2 := policyContext.Destroy(); err2 != nil {
@@ -375,11 +402,11 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 	// Check if the commit is blocked by $BUILDER_REGISTRY_SOURCES.
 	insecure, err := checkRegistrySourcesAllows("commit to", dest)
 	if err != nil {
-		return imgID, nil, "", err
+		return nil, err
 	}
 	if insecure {
 		if systemContext.DockerInsecureSkipTLSVerify == types.OptionalBoolFalse {
-			return imgID, nil, "", fmt.Errorf("can't require tls verification on an insecured registry")
+			return nil, fmt.Errorf("can't require tls verification on an insecured registry")
 		}
 		systemContext.DockerInsecureSkipTLSVerify = types.OptionalBoolTrue
 		systemContext.OCIInsecureSkipTLSVerify = true
@@ -393,7 +420,7 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 	if len(options.SBOMScanOptions) != 0 {
 		var scansDirectory string
 		if extraImageContent, extraLocalContent, scansDirectory, err = b.sbomScan(ctx, options); err != nil {
-			return imgID, nil, "", fmt.Errorf("scanning rootfs to generate SBOM for container %q: %w", b.ContainerID, err)
+			return nil, fmt.Errorf("scanning rootfs to generate SBOM for container %q: %w", b.ContainerID, err)
 		}
 		if scansDirectory != "" {
 			defer func() {
@@ -418,7 +445,7 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 	// Build an image reference from which we can copy the finished image.
 	src, err = b.makeContainerImageRef(options)
 	if err != nil {
-		return imgID, nil, "", fmt.Errorf("computing layer digests and building metadata for container %q: %w", b.ContainerID, err)
+		return nil, fmt.Errorf("computing layer digests and building metadata for container %q: %w", b.ContainerID, err)
 	}
 	// In case we're using caching, decide how to handle compression for a cache.
 	// If we're using blob caching, set it up for the source.
@@ -431,12 +458,12 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 		}
 		cache, err := blobcache.NewBlobCache(src, options.BlobDirectory, compress)
 		if err != nil {
-			return imgID, nil, "", fmt.Errorf("wrapping image reference %q in blob cache at %q: %w", transports.ImageName(src), options.BlobDirectory, err)
+			return nil, fmt.Errorf("wrapping image reference %q in blob cache at %q: %w", transports.ImageName(src), options.BlobDirectory, err)
 		}
 		maybeCachedSrc = cache
 		cache, err = blobcache.NewBlobCache(dest, options.BlobDirectory, compress)
 		if err != nil {
-			return imgID, nil, "", fmt.Errorf("wrapping image reference %q in blob cache at %q: %w", transports.ImageName(dest), options.BlobDirectory, err)
+			return nil, fmt.Errorf("wrapping image reference %q in blob cache at %q: %w", transports.ImageName(dest), options.BlobDirectory, err)
 		}
 		maybeCachedDest = cache
 	}
@@ -457,7 +484,7 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 
 	var manifestBytes []byte
 	if manifestBytes, err = retryCopyImage(ctx, policyContext, maybeCachedDest, maybeCachedSrc, dest, getCopyOptions(b.store, options.ReportWriter, nil, systemContext, "", false, options.SignBy, options.OciEncryptLayers, options.OciEncryptConfig, nil, destinationTimestamp), options.MaxRetries, options.RetryDelay); err != nil {
-		return imgID, nil, "", fmt.Errorf("copying layers and metadata for container %q: %w", b.ContainerID, err)
+		return nil, fmt.Errorf("copying layers and metadata for container %q: %w", b.ContainerID, err)
 	}
 	// If we've got more names to attach, and we know how to do that for
 	// the transport that we're writing the new image to, add them now.
@@ -466,10 +493,10 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 		case is.Transport.Name():
 			_, img, err := is.ResolveReference(dest)
 			if err != nil {
-				return imgID, nil, "", fmt.Errorf("locating just-written image %q: %w", transports.ImageName(dest), err)
+				return nil, fmt.Errorf("locating just-written image %q: %w", transports.ImageName(dest), err)
 			}
 			if err = util.AddImageNames(b.store, "", systemContext, img, options.AdditionalTags); err != nil {
-				return imgID, nil, "", fmt.Errorf("setting image names to %v: %w", append(img.Names, options.AdditionalTags...), err)
+				return nil, fmt.Errorf("setting image names to %v: %w", append(img.Names, options.AdditionalTags...), err)
 			}
 			logrus.Debugf("assigned names %v to image %q", img.Names, img.ID)
 		default:
@@ -480,7 +507,7 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 	if dest.Transport().Name() == is.Transport.Name() {
 		dest2, img, err := is.ResolveReference(dest)
 		if err != nil {
-			return imgID, nil, "", fmt.Errorf("locating image %q in local storage: %w", transports.ImageName(dest), err)
+			return nil, fmt.Errorf("locating image %q in local storage: %w", transports.ImageName(dest), err)
 		}
 		dest = dest2
 		imgID = img.ID
@@ -492,13 +519,13 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 		}
 		if len(toPruneNames) > 0 {
 			if err = b.store.RemoveNames(imgID, toPruneNames); err != nil {
-				return imgID, nil, "", fmt.Errorf("failed to remove temporary name from image %q: %w", imgID, err)
+				return nil, fmt.Errorf("failed to remove temporary name from image %q: %w", imgID, err)
 			}
 			logrus.Debugf("removing %v from assigned names to image %q", nameToRemove, img.ID)
 		}
 		if options.IIDFile != "" {
 			if err = os.WriteFile(options.IIDFile, []byte("sha256:"+img.ID), 0o644); err != nil {
-				return imgID, nil, "", err
+				return nil, err
 			}
 		}
 	}
@@ -521,23 +548,23 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 			return nil
 		}()
 		if err != nil {
-			return imgID, nil, "", err
+			return nil, err
 		}
 	}
 
 	// Calculate the as-written digest of the image's manifest and build the digested
 	// reference for the image.
 	manifestDigest, err := manifest.Digest(manifestBytes)
 	if err != nil {
-		return imgID, nil, "", fmt.Errorf("computing digest of manifest of new image %q: %w", transports.ImageName(dest), err)
+		return nil, fmt.Errorf("computing digest of manifest of new image %q: %w", transports.ImageName(dest), err)
 	}
-	if imgID == "" {
-		parsedManifest, err := manifest.FromBlob(manifestBytes, manifest.GuessMIMEType(manifestBytes))
-		if err != nil {
-			return imgID, nil, "", fmt.Errorf("parsing written manifest to determine the image's ID: %w", err)
-		}
-		configInfo := parsedManifest.ConfigInfo()
-		if configInfo.Size > 2 && configInfo.Digest.Validate() == nil { // don't be returning a digest of "" or "{}"
+	parsedManifest, err := manifest.FromBlob(manifestBytes, manifest.GuessMIMEType(manifestBytes))
+	if err != nil {
+		return nil, fmt.Errorf("parsing written manifest to determine the image's ID: %w", err)
+	}
+	configInfo := parsedManifest.ConfigInfo()
+	if configInfo.Size > 2 && configInfo.Digest.Validate() == nil { // don't be returning a digest of "" or "{}"
+		if imgID == "" {
 			imgID = configInfo.Digest.Encoded()
 		}
 	}
@@ -553,9 +580,28 @@ func (b *Builder) Commit(ctx context.Context, dest types.ImageReference, options
 	if options.Manifest != "" {
 		manifestID, err := b.addManifest(ctx, options.Manifest, imgID)
 		if err != nil {
-			return imgID, nil, "", err
+			return nil, err
 		}
 		logrus.Debugf("added imgID %s to manifestID %s", imgID, manifestID)
 	}
-	return imgID, ref, manifestDigest, nil
+
+	descriptor := v1.Descriptor{
+		MediaType: manifest.GuessMIMEType(manifestBytes),
+		Digest:    manifestDigest,
+		Size:      int64(len(manifestBytes)),
+	}
+	metadata, err := metadata.Build(configInfo.Digest, descriptor)
+	if err != nil {
+		return nil, fmt.Errorf("building metadata map for image: %w", err)
+	}
+
+	results := CommitResults{
+		ImageID:       imgID,
+		Canonical:     ref,
+		MediaType:     descriptor.MediaType,
+		ImageManifest: manifestBytes,
+		Digest:        manifestDigest,
+		Metadata:      metadata,
+	}
+	return &results, nil
 }

define/build.go

@@ -418,4 +418,7 @@ type BuildOptions struct {
 	// CreatedAnnotation controls whether or not an "org.opencontainers.image.created"
 	// annotation is present in the output image.
 	CreatedAnnotation types.OptionalBool
+	// MetadataFile is the name of a file to which the builder should write a JSON map
+	// containing metadata about the built image.
+	MetadataFile string
 }

docker/types.go

@@ -257,3 +257,10 @@ type V2S2Manifest struct {
 	// configuration.
 	Layers []V2S2Descriptor `json:"layers"`
 }
+
+const (
+	// github.com/moby/buildkit/exporter/containerimage/exptypes/types.go
+	ExporterImageDigestKey       = "containerimage.digest"
+	ExporterImageConfigDigestKey = "containerimage.config.digest"
+	ExporterImageDescriptorKey   = "containerimage.descriptor"
+)

docs/buildah-build.1.md

@@ -625,6 +625,12 @@ The format of `LIMIT` is `<number>[<unit>]`. Unit can be `b` (bytes),
 `k` (kilobytes), `m` (megabytes), or `g` (gigabytes). If you don't specify a
 unit, `b` is used. Set LIMIT to `-1` to enable unlimited swap.
 
+**--metadata-file** *MetadataFile*
+
+Write information about the built image to the named file.  When `--platform`
+is specified more than once, attempting to use this option will trigger an
+error.
+
 **--network**, **--net**=*mode*
 
 Sets the configuration for network namespaces when handling `RUN` instructions.

docs/buildah-commit.1.md

@@ -176,6 +176,10 @@ Write the image ID to the file.
 Name of the manifest list to which the built image will be added. Creates the manifest list
 if it does not exist. This option is useful for building multi architecture images.
 
+**--metadata-file** *MetadataFile*
+
+Write information about the committed image to the named file.
+
 **--omit-history** *bool-value*
 
 Omit build history information in the built image. (default false).

imagebuildah/build.go

@@ -86,6 +86,9 @@ func BuildDockerfiles(ctx context.Context, store storage.Store, options define.B
 	if len(options.Platforms) > 1 && options.IIDFile != "" {
 		return "", nil, fmt.Errorf("building multiple images, but iidfile %q can only be used to store one image ID", options.IIDFile)
 	}
+	if len(options.Platforms) > 1 && options.MetadataFile != "" {
+		return "", nil, fmt.Errorf("building multiple images, but metadata file %q can only be used to store information about one image", options.MetadataFile)
+	}
 
 	logger := logrus.New()
 	if options.Err != nil {