support multi-image (docker) archives

Support loading and saving tarballs with more than one image.
Add a new `/libpod/images/export` endpoint to the rest API to
allow for exporting/saving multiple images into an archive.

Note that a non-release version of containers/image is vendored.
A release version must be vendored before cutting a new Podman
release.  We force the containers/image version via a replace in
the go.mod file; this way go won't try to match the versions.

Signed-off-by: Valentin Rothberg <[email protected]>

cmd/podman/images/save.go

@@ -16,7 +16,10 @@ import (
 	"golang.org/x/crypto/ssh/terminal"
 )
 
-var validFormats = []string{define.OCIManifestDir, define.OCIArchive, define.V2s2ManifestDir, define.V2s2Archive}
+var (
+	validFormats    = []string{define.OCIManifestDir, define.OCIArchive, define.V2s2ManifestDir, define.V2s2Archive}
+	containerConfig = registry.PodmanConfig()
+)
 
 var (
 	saveDescription = `Save an image to docker-archive or oci-archive on the local machine. Default is docker-archive.`
@@ -79,7 +82,7 @@ func saveFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&saveOpts.Format, "format", define.V2s2Archive, "Save image to oci-archive, oci-dir (directory with oci manifest type), docker-archive, docker-dir (directory with v2s2 manifest type)")
 	flags.StringVarP(&saveOpts.Output, "output", "o", "", "Write to a specified file (default: stdout, which must be redirected)")
 	flags.BoolVarP(&saveOpts.Quiet, "quiet", "q", false, "Suppress the output")
-
+	flags.BoolVarP(&saveOpts.MultiImageArchive, "multi-image-archive", "m", containerConfig.Engine.MultiImageArchive, "Interpret additional arguments as images not tags and create a multi-image-archive (only for docker-archive)")
 }
 
 func save(cmd *cobra.Command, args []string) (finalErr error) {
@@ -118,6 +121,13 @@ func save(cmd *cobra.Command, args []string) (finalErr error) {
 	if len(args) > 1 {
 		tags = args[1:]
 	}
+
+	// Decide whether c/image's progress bars should use stderr or stdout.
+	// If the output is set of stdout, any log message there would corrupt
+	// the tarfile.
+	if saveOpts.Output == os.Stdout.Name() {
+		saveOpts.Quiet = true
+	}
 	err := registry.ImageEngine().Save(context.Background(), args[0], tags, saveOpts)
 	if err == nil {
 		succeeded = true

docs/source/markdown/podman-save.1.md

@@ -40,6 +40,10 @@ Save image to **oci-archive, oci-dir** (directory with oci manifest type), or **
 --format docker-dir
 ```
 
+**--multi-image-archive**, **-m**
+
+Allow for creating archives with more than one image.  Additional names will be interpreted as images instead of tags.  Only supported for **docker-archive**.
+
 **--quiet**, **-q**
 
 Suppress the output

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/containernetworking/cni v0.8.0
 	github.com/containernetworking/plugins v0.8.7
 	github.com/containers/buildah v1.15.1-0.20200813183340-0a8dc1f8064c
-	github.com/containers/common v0.20.3-0.20200827091701-a550d6a98aa3
+	github.com/containers/common v0.21.0
 	github.com/containers/conmon v2.0.20+incompatible
 	github.com/containers/image/v5 v5.5.2
 	github.com/containers/psgo v1.5.1
@@ -60,8 +60,10 @@ require (
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
 	golang.org/x/net v0.0.0-20200707034311-ab3426394381
 	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
-	golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1
+	golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed
 	k8s.io/api v0.0.0-20190620084959-7cf5895f2711
 	k8s.io/apimachinery v0.19.0
 	k8s.io/client-go v0.0.0-20190620085101-78d2af792bab
 )
+
+replace github.com/containers/image/v5 => github.com/containers/image/v5 v5.5.2-0.20200902171422-1c313b2d23e0

libpod/image/image.go

@@ -17,6 +17,7 @@ import (
 	"github.com/containers/common/pkg/retry"
 	cp "github.com/containers/image/v5/copy"
 	"github.com/containers/image/v5/directory"
+	"github.com/containers/image/v5/docker/archive"
 	dockerarchive "github.com/containers/image/v5/docker/archive"
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/image"
@@ -173,13 +174,182 @@ func (ir *Runtime) New(ctx context.Context, name, signaturePolicyPath, authfile
 	return newImage, nil
 }
 
+// SaveImages stores one more images in a multi-image archive.
+// Note that only `docker-archive` supports storing multiple
+// image.
+func (ir *Runtime) SaveImages(ctx context.Context, namesOrIDs []string, format string, outputFile string, quiet bool) (finalErr error) {
+	if format != DockerArchive {
+		return errors.Errorf("multi-image archives are only supported in in the %q format", DockerArchive)
+	}
+
+	sys := GetSystemContext("", "", false)
+
+	archWriter, err := archive.NewWriter(sys, outputFile)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err := archWriter.Close()
+		if err == nil {
+			return
+		}
+		if finalErr == nil {
+			finalErr = err
+			return
+		}
+		finalErr = errors.Wrap(finalErr, err.Error())
+	}()
+
+	// Decide whether c/image's progress bars should use stderr or stdout.
+	// Use stderr in case we need to be quiet or if the output is set to
+	// stdout.  If the output is set of stdout, any log message there would
+	// corrupt the tarfile.
+	writer := os.Stdout
+	if quiet {
+		writer = os.Stderr
+	}
+
+	// extend an image with additional tags
+	type imageData struct {
+		*Image
+		tags []reference.NamedTagged
+	}
+
+	// Look up the images (and their tags) in the local storage.
+	imageMap := make(map[string]*imageData) // to group tags for an image
+	imageQueue := []string{}                // to preserve relative image order
+	for _, nameOrID := range namesOrIDs {
+		// Look up the name or ID in the local image storage.
+		localImage, err := ir.NewFromLocal(nameOrID)
+		if err != nil {
+			return err
+		}
+		id := localImage.ID()
+
+		iData, exists := imageMap[id]
+		if !exists {
+			imageQueue = append(imageQueue, id)
+			iData = &imageData{Image: localImage}
+			imageMap[id] = iData
+		}
+
+		// Unless we referred to an ID, add the input as a tag.
+		if !strings.HasPrefix(id, nameOrID) {
+			tag, err := NormalizedTag(nameOrID)
+			if err != nil {
+				return err
+			}
+			refTagged, isTagged := tag.(reference.NamedTagged)
+			if isTagged {
+				iData.tags = append(iData.tags, refTagged)
+			}
+		}
+	}
+
+	policyContext, err := getPolicyContext(sys)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := policyContext.Destroy(); err != nil {
+			logrus.Errorf("failed to destroy policy context: %q", err)
+		}
+	}()
+
+	// Now copy the images one-by-one.
+	for _, id := range imageQueue {
+		dest, err := archWriter.NewReference(nil)
+		if err != nil {
+			return err
+		}
+
+		img := imageMap[id]
+		copyOptions := getCopyOptions(sys, writer, nil, nil, SigningOptions{}, "", img.tags)
+		copyOptions.DestinationCtx.SystemRegistriesConfPath = registries.SystemRegistriesConfPath()
+
+		// For copying, we need a source reference that we can create
+		// from the image.
+		src, err := is.Transport.NewStoreReference(img.imageruntime.store, nil, id)
+		if err != nil {
+			return errors.Wrapf(err, "error getting source imageReference for %q", img.InputName)
+		}
+		_, err = cp.Image(ctx, policyContext, dest, src, copyOptions)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// LoadAllImagesFromDockerArchive loads all images from the docker archive that
+// fileName points to.
+func (ir *Runtime) LoadAllImagesFromDockerArchive(ctx context.Context, fileName string, signaturePolicyPath string, writer io.Writer) ([]*Image, error) {
+	if signaturePolicyPath == "" {
+		signaturePolicyPath = ir.SignaturePolicyPath
+	}
+
+	sc := GetSystemContext(signaturePolicyPath, "", false)
+	reader, err := archive.NewReader(sc, fileName)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err := reader.Close(); err != nil {
+			logrus.Errorf(err.Error())
+		}
+	}()
+
+	refLists, err := reader.List()
+	if err != nil {
+		return nil, err
+	}
+
+	refPairs := []pullRefPair{}
+	for _, refList := range refLists {
+		for _, ref := range refList {
+			pairs, err := ir.getPullRefPairsFromDockerArchiveReference(ctx, reader, ref, sc)
+			if err != nil {
+				return nil, err
+			}
+			refPairs = append(refPairs, pairs...)
+		}
+	}
+
+	goal := pullGoal{
+		pullAllPairs:         true,
+		usedSearchRegistries: false,
+		refPairs:             refPairs,
+		searchedRegistries:   nil,
+	}
+
+	defer goal.cleanUp()
+	imageNames, err := ir.doPullImage(ctx, sc, goal, writer, SigningOptions{}, &DockerRegistryOptions{}, &retry.RetryOptions{}, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	newImages := make([]*Image, 0, len(imageNames))
+	for _, name := range imageNames {
+		newImage, err := ir.NewFromLocal(name)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error retrieving local image after pulling %s", name)
+		}
+		newImages = append(newImages, newImage)
+	}
+	ir.newImageEvent(events.LoadFromArchive, "")
+	return newImages, nil
+}
+
 // LoadFromArchiveReference creates a new image object for images pulled from a tar archive and the like (podman load)
 // This function is needed because it is possible for a tar archive to have multiple tags for one image
 func (ir *Runtime) LoadFromArchiveReference(ctx context.Context, srcRef types.ImageReference, signaturePolicyPath string, writer io.Writer) ([]*Image, error) {
 	if signaturePolicyPath == "" {
 		signaturePolicyPath = ir.SignaturePolicyPath
 	}
-	imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{}, &retry.RetryOptions{MaxRetry: maxRetry})
+
+	imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{}, &retry.RetryOptions{})
 	if err != nil {
 		return nil, errors.Wrapf(err, "unable to pull %s", transports.ImageName(srcRef))
 	}