Support to download remote OCI layers (with TLS)

docker/docker_client.go

@@ -8,19 +8,18 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net"
 	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/containers/image/docker/reference"
+	"github.com/containers/image/pkg/tlsclientconfig"
 	"github.com/containers/image/types"
 	"github.com/containers/storage/pkg/homedir"
 	"github.com/docker/distribution/registry/client"
 	helperclient "github.com/docker/docker-credential-helpers/client"
-	"github.com/docker/go-connections/sockets"
 	"github.com/docker/go-connections/tlsconfig"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
@@ -113,27 +112,7 @@ func serverDefault() *tls.Config {
 	}
 }
 
-func newTransport() *http.Transport {
-	direct := &net.Dialer{
-		Timeout:   30 * time.Second,
-		KeepAlive: 30 * time.Second,
-		DualStack: true,
-	}
-	tr := &http.Transport{
-		Proxy:               http.ProxyFromEnvironment,
-		Dial:                direct.Dial,
-		TLSHandshakeTimeout: 10 * time.Second,
-		// TODO(dmcgowan): Call close idle connections when complete and use keep alive
-		DisableKeepAlives: true,
-	}
-	proxyDialer, err := sockets.DialerFromEnvironment(direct)
-	if err == nil {
-		tr.Dial = proxyDialer.Dial
-	}
-	return tr
-}
-
-// dockerCertDir returns a path to a directory to be consumed by setupCertificates() depending on ctx and hostPort.
+// dockerCertDir returns a path to a directory to be consumed by tlsclientconfig.SetupCertificates() depending on ctx and hostPort.
 func dockerCertDir(ctx *types.SystemContext, hostPort string) string {
 	if ctx != nil && ctx.DockerCertPath != "" {
 		return ctx.DockerCertPath
@@ -149,69 +128,6 @@ func dockerCertDir(ctx *types.SystemContext, hostPort string) string {
 	return filepath.Join(hostCertDir, hostPort)
 }
 
-func setupCertificates(dir string, tlsc *tls.Config) error {
-	logrus.Debugf("Looking for TLS certificates and private keys in %s", dir)
-	fs, err := ioutil.ReadDir(dir)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		if os.IsPermission(err) {
-			logrus.Debugf("Skipping scan of %s due to permission error: %v", dir, err)
-			return nil
-		}
-		return err
-	}
-
-	for _, f := range fs {
-		fullPath := filepath.Join(dir, f.Name())
-		if strings.HasSuffix(f.Name(), ".crt") {
-			systemPool, err := tlsconfig.SystemCertPool()
-			if err != nil {
-				return errors.Wrap(err, "unable to get system cert pool")
-			}
-			tlsc.RootCAs = systemPool
-			logrus.Debugf(" crt: %s", fullPath)
-			data, err := ioutil.ReadFile(fullPath)
-			if err != nil {
-				return err
-			}
-			tlsc.RootCAs.AppendCertsFromPEM(data)
-		}
-		if strings.HasSuffix(f.Name(), ".cert") {
-			certName := f.Name()
-			keyName := certName[:len(certName)-5] + ".key"
-			logrus.Debugf(" cert: %s", fullPath)
-			if !hasFile(fs, keyName) {
-				return errors.Errorf("missing key %s for client certificate %s. Note that CA certificates should use the extension .crt", keyName, certName)
-			}
-			cert, err := tls.LoadX509KeyPair(filepath.Join(dir, certName), filepath.Join(dir, keyName))
-			if err != nil {
-				return err
-			}
-			tlsc.Certificates = append(tlsc.Certificates, cert)
-		}
-		if strings.HasSuffix(f.Name(), ".key") {
-			keyName := f.Name()
-			certName := keyName[:len(keyName)-4] + ".cert"
-			logrus.Debugf(" key: %s", fullPath)
-			if !hasFile(fs, certName) {
-				return errors.Errorf("missing client certificate %s for key %s", certName, keyName)
-			}
-		}
-	}
-	return nil
-}
-
-func hasFile(files []os.FileInfo, name string) bool {
-	for _, f := range files {
-		if f.Name() == name {
-			return true
-		}
-	}
-	return false
-}
-
 // newDockerClient returns a new dockerClient instance for refHostname (a host a specified in the Docker image reference, not canonicalized to dockerRegistry)
 // “write” specifies whether the client will be used for "write" access (in particular passed to lookaside.go:toplevelFromSection)
 func newDockerClient(ctx *types.SystemContext, ref dockerReference, write bool, actions string) (*dockerClient, error) {
@@ -223,15 +139,15 @@ func newDockerClient(ctx *types.SystemContext, ref dockerReference, write bool,
 	if err != nil {
 		return nil, err
 	}
-	tr := newTransport()
+	tr := tlsclientconfig.NewTransport()
 	tr.TLSClientConfig = serverDefault()
 	// It is undefined whether the host[:port] string for dockerHostname should be dockerHostname or dockerRegistry,
 	// because docker/docker does not read the certs.d subdirectory at all in that case.  We use the user-visible
 	// dockerHostname here, because it is more symmetrical to read the configuration in that case as well, and because
 	// generally the UI hides the existence of the different dockerRegistry.  But note that this behavior is
 	// undocumented and may change if docker/docker changes.
 	certDir := dockerCertDir(ctx, reference.Domain(ref.ref))
-	if err := setupCertificates(certDir, tr.TLSClientConfig); err != nil {
+	if err := tlsclientconfig.SetupCertificates(certDir, tr.TLSClientConfig); err != nil {
 		return nil, err
 	}
 	if ctx != nil && ctx.DockerInsecureSkipTLSVerify {
@@ -364,7 +280,7 @@ func (c *dockerClient) getBearerToken(ctx context.Context, realm, service, scope
 	if c.username != "" && c.password != "" {
 		authReq.SetBasicAuth(c.username, c.password)
 	}
-	tr := newTransport()
+	tr := tlsclientconfig.NewTransport()
 	// TODO(runcom): insecure for now to contact the external token service
 	tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
 	client := &http.Client{Transport: tr}

oci/layout/fixtures/accepted_certs/cacert.crt

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIICCjCCAWygAwIBAgIQFg60+EkXWgVv8xqI3yrxQjAKBggqhkjOPQQDBDASMRAw
+DgYDVQQKEwdBY21lIENvMB4XDTE3MDkyMjEwMzkzOFoXDTE4MDkyMjEwMzkzOFow
+EjEQMA4GA1UEChMHQWNtZSBDbzCBmzAQBgcqhkjOPQIBBgUrgQQAIwOBhgAEAI3p
+xckijV44L3ffAlLOqB4oA/HpP7S5gTpWrIUU+2SxFJU/bcTKDLPk1cEC87vW+UCY
+IXAyYGlyMAGSm0GxAFHnAIIrQzx9m3yiHbUyIPvRMW4BoDKsLaf5+GIZMm9nOq2q
+njvHr9ag2J3IzxEqQ8KZ95ivmHYrh3VsnfisI7c3opiro2EwXzAOBgNVHQ8BAf8E
+BAMCAqQwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA8GA1UdEwEB/wQF
+MAMBAf8wHQYDVR0RBBYwFIIJbG9jYWxob3N0gQdhQGEuY29tMAoGCCqGSM49BAME
+A4GLADCBhwJCAbk0YzSo4Pf673WlVJ5kitJ1ti0Y6NT47up5683/Y6V2/WM668Zb
+x0qVoa1KUKjqpmdcNS22efqB0P2Ns+2kdh4XAkEm1toJclyhucvTYIsD6MlAxp6v
+Ji5mpzROSkxZVsuH2BoYuJMjkjLeRImekPKHvYzmx4yoMyO4h4NEmTtkp3RdjQ==
+-----END CERTIFICATE-----

oci/layout/fixtures/accepted_certs/cert.cert

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIICCjCCAWygAwIBAgIQFg60+EkXWgVv8xqI3yrxQjAKBggqhkjOPQQDBDASMRAw
+DgYDVQQKEwdBY21lIENvMB4XDTE3MDkyMjEwMzkzOFoXDTE4MDkyMjEwMzkzOFow
+EjEQMA4GA1UEChMHQWNtZSBDbzCBmzAQBgcqhkjOPQIBBgUrgQQAIwOBhgAEAI3p
+xckijV44L3ffAlLOqB4oA/HpP7S5gTpWrIUU+2SxFJU/bcTKDLPk1cEC87vW+UCY
+IXAyYGlyMAGSm0GxAFHnAIIrQzx9m3yiHbUyIPvRMW4BoDKsLaf5+GIZMm9nOq2q
+njvHr9ag2J3IzxEqQ8KZ95ivmHYrh3VsnfisI7c3opiro2EwXzAOBgNVHQ8BAf8E
+BAMCAqQwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA8GA1UdEwEB/wQF
+MAMBAf8wHQYDVR0RBBYwFIIJbG9jYWxob3N0gQdhQGEuY29tMAoGCCqGSM49BAME
+A4GLADCBhwJCAbk0YzSo4Pf673WlVJ5kitJ1ti0Y6NT47up5683/Y6V2/WM668Zb
+x0qVoa1KUKjqpmdcNS22efqB0P2Ns+2kdh4XAkEm1toJclyhucvTYIsD6MlAxp6v
+Ji5mpzROSkxZVsuH2BoYuJMjkjLeRImekPKHvYzmx4yoMyO4h4NEmTtkp3RdjQ==
+-----END CERTIFICATE-----

oci/layout/fixtures/accepted_certs/cert.key

@@ -0,0 +1,7 @@
+-----BEGIN EC PRIVATE KEY-----
+MIHcAgEBBEIAMDtdVU5PeUWCo1Ndvr+1X+Hry4I7+NdTqxLlU0ZBudm2ov0iJdZj
+O2PdSW6pRHJl9gYL+D/QjcEIwQBK4vsHS3SgBwYFK4EEACOhgYkDgYYABACN6cXJ
+Io1eOC933wJSzqgeKAPx6T+0uYE6VqyFFPtksRSVP23Eygyz5NXBAvO71vlAmCFw
+MmBpcjABkptBsQBR5wCCK0M8fZt8oh21MiD70TFuAaAyrC2n+fhiGTJvZzqtqp47
+x6/WoNidyM8RKkPCmfeYr5h2K4d1bJ34rCO3N6KYqw==
+-----END EC PRIVATE KEY-----

oci/layout/fixtures/manifest/index.json

@@ -0,0 +1 @@
+{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:84afb6189c4d69f2d040c5f1dc4e0a16fed9b539ce9cfb4ac2526ae4e0576cc0","size":496,"annotations":{"org.opencontainers.image.ref.name":"v0.1.1"},"platform":{"architecture":"amd64","os":"linux"}}]}

oci/layout/fixtures/rejected_certs/cert.cert

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIICDDCCAW2gAwIBAgIRAK2Kd9uMIa+8QMULJEerBfkwCgYIKoZIzj0EAwQwEjEQ
+MA4GA1UEChMHQWNtZSBDbzAeFw0xNzA5MjIxMjE2NThaFw0xODA5MjIxMjE2NTha
+MBIxEDAOBgNVBAoTB0FjbWUgQ28wgZswEAYHKoZIzj0CAQYFK4EEACMDgYYABAFH
+j4kX+5EWC7oorajgfWJHDs9Tx2H2VjvzzLruJnmzc6+TVMKDXoaqxByvxUJ6HXEV
+VthYgjemoGF32yWbzV+D5wGTlh37IKWOpHVKGLn6vQ7kFaFUTl65l3IGzpGd+CDm
+xrT2GYVdLjmrRINNby5JC7flsCqv6FPlbD/qA9S8a3U5r6NhMF8wDgYDVR0PAQH/
+BAQDAgKkMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAPBgNVHRMBAf8E
+BTADAQH/MB0GA1UdEQQWMBSCCWxvY2FsaG9zdIEHYkBiLmNvbTAKBggqhkjOPQQD
+BAOBjAAwgYgCQgGEJ3gBvWcs3KOYi3akz5H/QY8+NROGTRZ8VHRmySgPNKiUq9by
+5kxb9LhwnzIHhcYDQS7jJCaZpzzD+rkYrG9MEAJCAcsyzixyufE8iOrKc1qHKnya
+5poyGv5+WoloVozs8NreR7ZhYsOwPHXt10ciGPDN+7EZnzVvYI+e1ZMLz1NtF0Gr
+-----END CERTIFICATE-----

oci/layout/fixtures/rejected_certs/cert.key

@@ -0,0 +1,7 @@
+-----BEGIN EC PRIVATE KEY-----
+MIHcAgEBBEIB3BPUEOohwxGCV8V2fwIBdZ3S7yWADrbz5w17YITBt0p6j1C0NKRx
+xL9V7Cq+P2OkfQa6rxiD7cM8DjP/6y1//XKgBwYFK4EEACOhgYkDgYYABAFHj4kX
++5EWC7oorajgfWJHDs9Tx2H2VjvzzLruJnmzc6+TVMKDXoaqxByvxUJ6HXEVVthY
+gjemoGF32yWbzV+D5wGTlh37IKWOpHVKGLn6vQ7kFaFUTl65l3IGzpGd+CDmxrT2
+GYVdLjmrRINNby5JC7flsCqv6FPlbD/qA9S8a3U5rw==
+-----END EC PRIVATE KEY-----

oci/layout/oci_dest.go

@@ -66,7 +66,7 @@ func (d *ociImageDestination) ShouldCompressLayers() bool {
 // AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually
 // uploaded to the image destination, true otherwise.
 func (d *ociImageDestination) AcceptsForeignLayerURLs() bool {
-	return false
+	return true
 }
 
 // MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime OS. False otherwise.

oci/layout/oci_src.go

@@ -4,25 +4,43 @@ import (
 	"context"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
+	"strconv"
 
+	"github.com/containers/image/pkg/tlsclientconfig"
 	"github.com/containers/image/types"
+	"github.com/docker/go-connections/tlsconfig"
 	"github.com/opencontainers/go-digest"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
 )
 
 type ociImageSource struct {
 	ref        ociReference
 	descriptor imgspecv1.Descriptor
+	client     *http.Client
 }
 
 // newImageSource returns an ImageSource for reading from an existing directory.
-func newImageSource(ref ociReference) (types.ImageSource, error) {
+func newImageSource(ctx *types.SystemContext, ref ociReference) (types.ImageSource, error) {
+	tr := tlsclientconfig.NewTransport()
+	tr.TLSClientConfig = tlsconfig.ServerDefault()
+
+	if ctx != nil && ctx.OCICertPath != "" {
+		if err := tlsclientconfig.SetupCertificates(ctx.OCICertPath, tr.TLSClientConfig); err != nil {
+			return nil, err
+		}
+		tr.TLSClientConfig.InsecureSkipVerify = ctx.OCIInsecureSkipTLSVerify
+	}
+
+	client := &http.Client{}
+	client.Transport = tr
 	descriptor, err := ref.getManifestDescriptor()
 	if err != nil {
 		return nil, err
 	}
-	return &ociImageSource{ref: ref, descriptor: descriptor}, nil
+	return &ociImageSource{ref: ref, descriptor: descriptor, client: client}, nil
 }
 
 // Reference returns the reference used to set up this source.
@@ -70,6 +88,10 @@ func (s *ociImageSource) GetTargetManifest(digest digest.Digest) ([]byte, string
 
 // GetBlob returns a stream for the specified blob, and the blob's size.
 func (s *ociImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, error) {
+	if len(info.URLs) != 0 {
+		return s.getExternalBlob(info.URLs)
+	}
+
 	path, err := s.ref.blobPath(info.Digest)
 	if err != nil {
 		return nil, 0, err
@@ -89,3 +111,32 @@ func (s *ociImageSource) GetBlob(info types.BlobInfo) (io.ReadCloser, int64, err
 func (s *ociImageSource) GetSignatures(context.Context) ([][]byte, error) {
 	return [][]byte{}, nil
 }
+
+func (s *ociImageSource) getExternalBlob(urls []string) (io.ReadCloser, int64, error) {
+	errWrap := errors.New("failed fetching external blob from all urls")
+	for _, url := range urls {
+		resp, err := s.client.Get(url)
+		if err != nil {
+			errWrap = errors.Wrapf(errWrap, "fetching %s failed %s", url, err.Error())
+			continue
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			resp.Body.Close()
+			errWrap = errors.Wrapf(errWrap, "fetching %s failed, response code not 200", url)
+			continue
+		}
+
+		return resp.Body, getBlobSize(resp), nil
+	}
+
+	return nil, 0, errWrap
+}
+
+func getBlobSize(resp *http.Response) int64 {
+	size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
+	if err != nil {
+		size = -1
+	}
+	return size
+}