DO NOT MERGE IRRESPONSIBLE: Cosign integration

copy/copy.go

@@ -123,15 +123,17 @@ type ImageListSelection int
 
 // Options allows supplying non-default configuration modifying the behavior of CopyImage.
 type Options struct {
-	RemoveSignatures bool            // Remove any pre-existing signatures. SignBy will still add a new signature.
-	SignBy           string          // If non-empty, asks for a signature to be added during the copy, and specifies a key ID, as accepted by signature.NewGPGSigningMechanism().SignDockerManifest(),
-	SignPassphrase   string          // Passphare to use when signing with the key ID from `SignBy`.
-	SignIdentity     reference.Named // Identify to use when signing, defaults to the docker reference of the destination
-	ReportWriter     io.Writer
-	SourceCtx        *types.SystemContext
-	DestinationCtx   *types.SystemContext
-	ProgressInterval time.Duration                 // time to wait between reports to signal the progress channel
-	Progress         chan types.ProgressProperties // Reported to when ProgressInterval has arrived for a single artifact+offset.
+	RemoveSignatures               bool            // Remove any pre-existing signatures. SignBy will still add a new signature.
+	SignBy                         string          // If non-empty, asks for a signature to be added during the copy, and specifies a key ID, as accepted by signature.NewGPGSigningMechanism().SignDockerManifest(),
+	SignPassphrase                 string          // Passphare to use when signing with the key ID from `SignBy`.
+	SignByCosignPrivateKeyFile     string          // If non-empty, asks for a signature to be added during the copy, using a Cosign private key file at the provided path.
+	SignCosignPrivateKeyPassphrase []byte          // Passphare to use when signing with `SignByCosignPrivateKeyFile`.
+	SignIdentity                   reference.Named // Identify to use when signing, defaults to the docker reference of the destination
+	ReportWriter                   io.Writer
+	SourceCtx                      *types.SystemContext
+	DestinationCtx                 *types.SystemContext
+	ProgressInterval               time.Duration                 // time to wait between reports to signal the progress channel
+	Progress                       chan types.ProgressProperties // Reported to when ProgressInterval has arrived for a single artifact+offset.
 
 	// Preserve digests, and fail if we cannot.
 	PreserveDigests bool
@@ -575,6 +577,13 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur
 		}
 		sigs = append(sigs, newSig)
 	}
+	if options.SignByCosignPrivateKeyFile != "" {
+		newSig, err := c.createCosignSignature(manifestList, options.SignByCosignPrivateKeyFile, options.SignCosignPrivateKeyPassphrase, options.SignIdentity)
+		if err != nil {
+			return nil, err
+		}
+		sigs = append(sigs, newSig)
+	}
 
 	c.Printf("Storing list signatures\n")
 	if err := c.dest.PutSignaturesWithFormat(ctx, sigs, nil); err != nil {
@@ -645,7 +654,7 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli
 		sigs = []internalsig.Signature{}
 	} else {
 		c.Printf("Getting image source signatures\n")
-		s, err := src.SignaturesWithFormat(ctx)
+		s, err := src.UntrustedSignatures(ctx)
 		if err != nil {
 			return nil, "", "", perrors.Wrap(err, "reading signatures")
 		}
@@ -688,7 +697,7 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli
 	//   We do intend the RecordDigestUncompressedPair calls to only work with reliable data, but at least there’s a risk
 	//   that the compressed version coming from a third party may be designed to attack some other decompressor implementation,
 	//   and we would reuse and sign it.
-	ic.canSubstituteBlobs = ic.cannotModifyManifestReason == "" && options.SignBy == ""
+	ic.canSubstituteBlobs = ic.cannotModifyManifestReason == "" && options.SignBy == "" && options.SignByCosignPrivateKeyFile == ""
 
 	if err := ic.updateEmbeddedDockerReference(); err != nil {
 		return nil, "", "", err
@@ -719,7 +728,7 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli
 
 	// If enabled, fetch and compare the destination's manifest. And as an optimization skip updating the destination iff equal
 	if options.OptimizeDestinationImageAlreadyExists {
-		shouldUpdateSigs := len(sigs) > 0 || options.SignBy != "" // TODO: Consider allowing signatures updates only and skipping the image's layers/manifest copy if possible
+		shouldUpdateSigs := len(sigs) > 0 || options.SignBy != "" || options.SignByCosignPrivateKeyFile != "" // TODO: Consider allowing signatures updates only and skipping the image's layers/manifest copy if possible
 		noPendingManifestUpdates := ic.noPendingManifestUpdates()
 
 		logrus.Debugf("Checking if we can skip copying: has signatures=%t, OCI encryption=%t, no manifest updates=%t", shouldUpdateSigs, destRequiresOciEncryption, noPendingManifestUpdates)
@@ -806,6 +815,13 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli
 		}
 		sigs = append(sigs, newSig)
 	}
+	if options.SignByCosignPrivateKeyFile != "" {
+		newSig, err := c.createCosignSignature(manifestBytes, options.SignByCosignPrivateKeyFile, options.SignCosignPrivateKeyPassphrase, options.SignIdentity)
+		if err != nil {
+			return nil, "", "", err
+		}
+		sigs = append(sigs, newSig)
+	}
 
 	c.Printf("Storing signatures\n")
 	if err := c.dest.PutSignaturesWithFormat(ctx, sigs, targetInstance); err != nil {

copy/sign.go

@@ -6,6 +6,7 @@ import (
 	"github.com/containers/image/v5/docker/reference"
 	internalsig "github.com/containers/image/v5/internal/signature"
 	"github.com/containers/image/v5/signature"
+	"github.com/containers/image/v5/signature/cosign"
 	"github.com/containers/image/v5/transports"
 	perrors "github.com/pkg/errors"
 )
@@ -39,3 +40,24 @@ func (c *copier) createSignature(manifest []byte, keyIdentity string, passphrase
 	}
 	return internalsig.SimpleSigningFromBlob(newSig), nil
 }
+
+// createCosignSignature creates a new Cosign signature of manifest using privateKeyFile and identity.
+func (c *copier) createCosignSignature(manifest []byte, privateKeyFile string, passphrase []byte, identity reference.Named) (internalsig.Signature, error) {
+	if identity != nil {
+		if reference.IsNameOnly(identity) {
+			return nil, fmt.Errorf("Sign identity must be a fully specified reference %s", identity.String())
+		}
+	} else {
+		identity = c.dest.Reference().DockerReference()
+		if identity == nil {
+			return nil, fmt.Errorf("Cannot determine canonical Docker reference for destination %s", transports.ImageName(c.dest.Reference()))
+		}
+	}
+
+	c.Printf("Signing manifest\n")
+	newSig, err := cosign.SignDockerManifestWithPrivateKeyFileUnstable(manifest, identity, privateKeyFile, passphrase)
+	if err != nil {
+		return nil, fmt.Errorf("creating signature: %w", err)
+	}
+	return newSig, nil
+}

docker/docker_client.go

@@ -18,15 +18,19 @@ import (
 
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/internal/iolimits"
+	"github.com/containers/image/v5/manifest"
 	"github.com/containers/image/v5/pkg/docker/config"
 	"github.com/containers/image/v5/pkg/sysregistriesv2"
 	"github.com/containers/image/v5/pkg/tlsclientconfig"
 	"github.com/containers/image/v5/types"
 	"github.com/containers/image/v5/version"
 	"github.com/containers/storage/pkg/homedir"
+	"github.com/docker/distribution/registry/api/errcode"
+	v2 "github.com/docker/distribution/registry/api/v2"
 	clientLib "github.com/docker/distribution/registry/client"
 	"github.com/docker/go-connections/tlsconfig"
 	digest "github.com/opencontainers/go-digest"
+	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 	perrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -102,10 +106,11 @@ type dockerClient struct {
 	// by detectProperties(). Callers can edit tlsClientConfig.InsecureSkipVerify in the meantime.
 	tlsClientConfig *tls.Config
 	// The following members are not set by newDockerClient and must be set by callers if needed.
-	auth          types.DockerAuthConfig
-	registryToken string
-	signatureBase signatureStorageBase
-	scope         authScope
+	auth                 types.DockerAuthConfig
+	registryToken        string
+	signatureBase        signatureStorageBase
+	useCosignAttachments bool
+	scope                authScope
 
 	// The following members are detected registry properties:
 	// They are set after a successful detectProperties(), and never change afterwards.
@@ -210,13 +215,13 @@ func dockerCertDir(sys *types.SystemContext, hostPort string) (string, error) {
 // newDockerClientFromRef returns a new dockerClient instance for refHostname (a host a specified in the Docker image reference, not canonicalized to dockerRegistry)
 // “write” specifies whether the client will be used for "write" access (in particular passed to lookaside.go:toplevelFromSection)
 // signatureBase is always set in the return value
-func newDockerClientFromRef(sys *types.SystemContext, ref dockerReference, write bool, actions string) (*dockerClient, error) {
+func newDockerClientFromRef(sys *types.SystemContext, ref dockerReference, registryConfig *registryConfiguration, write bool, actions string) (*dockerClient, error) {
 	auth, err := config.GetCredentialsForRef(sys, ref.ref)
 	if err != nil {
 		return nil, perrors.Wrapf(err, "getting username and password")
 	}
 
-	sigBase, err := SignatureStorageBaseURL(sys, ref, write)
+	sigBase, err := registryConfig.signatureStorageBaseURL(ref, write)
 	if err != nil {
 		return nil, err
 	}
@@ -231,6 +236,7 @@ func newDockerClientFromRef(sys *types.SystemContext, ref dockerReference, write
 		client.registryToken = sys.DockerBearerRegistryToken
 	}
 	client.signatureBase = sigBase
+	client.useCosignAttachments = registryConfig.useCosignAttachments(ref)
 	client.scope.actions = actions
 	client.scope.remoteName = reference.Path(ref.ref)
 	return client, nil
@@ -801,6 +807,166 @@ func (c *dockerClient) detectProperties(ctx context.Context) error {
 	return c.detectPropertiesError
 }
 
+func (c *dockerClient) fetchManifest(ctx context.Context, ref dockerReference, tagOrDigest string) ([]byte, string, error) {
+	path := fmt.Sprintf(manifestPath, reference.Path(ref.ref), tagOrDigest)
+	headers := map[string][]string{
+		"Accept": manifest.DefaultRequestedManifestMIMETypes,
+	}
+	res, err := c.makeRequest(ctx, http.MethodGet, path, headers, nil, v2Auth, nil)
+	if err != nil {
+		return nil, "", err
+	}
+	logrus.Debugf("Content-Type from manifest GET is %q", res.Header.Get("Content-Type"))
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return nil, "", perrors.Wrapf(registryHTTPResponseToError(res), "reading manifest %s in %s", tagOrDigest, ref.ref.Name())
+	}
+
+	manblob, err := iolimits.ReadAtMost(res.Body, iolimits.MaxManifestBodySize)
+	if err != nil {
+		return nil, "", err
+	}
+	return manblob, simplifyContentType(res.Header.Get("Content-Type")), nil
+}
+
+// getExternalBlob returns the reader of the first available blob URL from urls, which must not be empty.
+// This function can return nil reader when no url is supported by this function. In this case, the caller
+// should fallback to fetch the non-external blob (i.e. pull from the registry).
+func (c *dockerClient) getExternalBlob(ctx context.Context, urls []string) (io.ReadCloser, int64, error) {
+	var (
+		resp *http.Response
+		err  error
+	)
+	if len(urls) == 0 {
+		return nil, 0, errors.New("internal error: getExternalBlob called with no URLs")
+	}
+	for _, u := range urls {
+		url, err := url.Parse(u)
+		if err != nil || (url.Scheme != "http" && url.Scheme != "https") {
+			continue // unsupported url. skip this url.
+		}
+		// NOTE: we must not authenticate on additional URLs as those
+		//       can be abused to leak credentials or tokens.  Please
+		//       refer to CVE-2020-15157 for more information.
+		resp, err = c.makeRequestToResolvedURL(ctx, http.MethodGet, url, nil, nil, -1, noAuth, nil)
+		if err == nil {
+			if resp.StatusCode != http.StatusOK {
+				err = fmt.Errorf("error fetching external blob from %q: %d (%s)", u, resp.StatusCode, http.StatusText(resp.StatusCode))
+				logrus.Debug(err)
+				resp.Body.Close()
+				continue
+			}
+			break
+		}
+	}
+	if resp == nil && err == nil {
+		return nil, 0, nil // fallback to non-external blob
+	}
+	if err != nil {
+		return nil, 0, err
+	}
+	return resp.Body, getBlobSize(resp), nil
+}
+
+func getBlobSize(resp *http.Response) int64 {
+	size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
+	if err != nil {
+		size = -1
+	}
+	return size
+}
+
+// getBlob returns a stream for the specified blob in ref, and the blob’s size (or -1 if unknown).
+// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
+// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
+func (c *dockerClient) getBlob(ctx context.Context, ref dockerReference, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
+	if len(info.URLs) != 0 {
+		r, s, err := c.getExternalBlob(ctx, info.URLs)
+		if err != nil {
+			return nil, 0, err
+		} else if r != nil {
+			return r, s, nil
+		}
+	}
+
+	path := fmt.Sprintf(blobsPath, reference.Path(ref.ref), info.Digest.String())
+	logrus.Debugf("Downloading %s", path)
+	res, err := c.makeRequest(ctx, http.MethodGet, path, nil, nil, v2Auth, nil)
+	if err != nil {
+		return nil, 0, err
+	}
+	if err := httpResponseToError(res, "Error fetching blob"); err != nil {
+		res.Body.Close()
+		return nil, 0, err
+	}
+	cache.RecordKnownLocation(ref.Transport(), bicTransportScope(ref), info.Digest, newBICLocationReference(ref))
+	return res.Body, getBlobSize(res), nil
+}
+
+// getOCIDescriptorContents returns the contents a blob spcified by descriptor in ref, which must fit within limit.
+func (c *dockerClient) getOCIDescriptorContents(ctx context.Context, ref dockerReference, desc imgspecv1.Descriptor, maxSize int, cache types.BlobInfoCache) ([]byte, error) {
+	// Note that this copies all kinds of attachments: attestations, and whatever else is there,
+	// not just signatures. We leave the signature consumers to decide based on the MIME type.
+	reader, _, err := c.getBlob(ctx, ref, manifest.BlobInfoFromOCI1Descriptor(desc), cache)
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Close()
+	payload, err := iolimits.ReadAtMost(reader, iolimits.MaxSignatureBodySize)
+	if err != nil {
+		return nil, fmt.Errorf("reading blob %s in %s: %w", desc.Digest.String(), ref.ref.Name(), err)
+	}
+	return payload, nil
+}
+
+// isManifestUnknownError returns true iff err from client.HandleErrorResponse is a “manifest unknown” error.
+func isManifestUnknownError(err error) bool {
+	errors, ok := err.(errcode.Errors)
+	if !ok || len(errors) == 0 {
+		return false
+	}
+	err = errors[0]
+	ec, ok := err.(errcode.ErrorCoder)
+	if !ok {
+		return false
+	}
+	return ec.ErrorCode() == v2.ErrorCodeManifestUnknown
+}
+
+// getCosignAttachmentManifest loads and parses the manifest for Cosign attachments for
+// digest in ref.
+// It returns (nil, nil) if the manifest does not exist.
+func (c *dockerClient) getCosignAttachmentManifest(ctx context.Context, ref dockerReference, digest digest.Digest) (*manifest.OCI1, error) {
+	tag := cosignAttachmentTag(digest)
+	cosignRef, err := reference.WithTag(reference.TrimNamed(ref.ref), tag)
+	if err != nil {
+		return nil, err
+	}
+	logrus.Debugf("Looking for Cosign attachments in %s", cosignRef.String())
+	manifestBlob, mimeType, err := c.fetchManifest(ctx, ref, tag)
+	if err != nil {
+		// FIXME: Are we going to need better heuristics??
+		// This alone is probably a good enough reason for Cosign to be opt-in only,
+		// otherwise we would just break ordinary copies.
+		if isManifestUnknownError(err) {
+			logrus.Debugf("Fetching Cosign attachment manifest failed, assuming it does not exist: %v", err)
+			return nil, nil
+		}
+		logrus.Debugf("Fetching Cosign attachment manifest failed: %v", err)
+		return nil, err
+	}
+	if mimeType != imgspecv1.MediaTypeImageManifest {
+		// FIXME: Try anyway??
+		return nil, fmt.Errorf("unexpected MIME type for Cosign attachment manifest %s: %q",
+			cosignRef.String(), mimeType)
+	}
+	res, err := manifest.OCI1FromManifest(manifestBlob)
+	if err != nil {
+		return nil, fmt.Errorf("parsing manifest %s: %w", cosignRef.String(), err)
+	}
+	return res, nil
+}
+
 // getExtensionsSignatures returns signatures from the X-Registry-Supports-Signatures API extension,
 // using the original data structures.
 func (c *dockerClient) getExtensionsSignatures(ctx context.Context, ref dockerReference, manifestDigest digest.Digest) (*extensionSignatureList, error) {
@@ -826,3 +992,8 @@ func (c *dockerClient) getExtensionsSignatures(ctx context.Context, ref dockerRe
 	}
 	return &parsedBody, nil
 }
+
+// cosignAttachmentTag returns a Cosign attachment tag for the specified digest.
+func cosignAttachmentTag(d digest.Digest) string {
+	return strings.Replace(d.String(), ":", "-", 1) + ".sig"
+}

docker/docker_image.go

@@ -60,8 +60,12 @@ func GetRepositoryTags(ctx context.Context, sys *types.SystemContext, ref types.
 		return nil, errors.New("ref must be a dockerReference")
 	}
 
+	registryConfig, err := loadRegistryConfiguration(sys)
+	if err != nil {
+		return nil, err
+	}
 	path := fmt.Sprintf(tagsPath, reference.Path(dr.ref))
-	client, err := newDockerClientFromRef(sys, dr, false, "pull")
+	client, err := newDockerClientFromRef(sys, dr, registryConfig, false, "pull")
 	if err != nil {
 		return nil, perrors.Wrap(err, "failed to create client")
 	}
@@ -125,7 +129,11 @@ func GetDigest(ctx context.Context, sys *types.SystemContext, ref types.ImageRef
 		return "", err
 	}
 
-	client, err := newDockerClientFromRef(sys, dr, false, "pull")
+	registryConfig, err := loadRegistryConfiguration(sys)
+	if err != nil {
+		return "", err
+	}
+	client, err := newDockerClientFromRef(sys, dr, registryConfig, false, "pull")
 	if err != nil {
 		return "", perrors.Wrap(err, "failed to create client")
 	}