Enable blob caching and blobInfo caching

Dockerfile

@@ -16,6 +16,8 @@ COPY --from=builder /go/src/github.com/openshift/builder/openshift-builder /usr/
 COPY imagecontent/policy.json /etc/containers/
 COPY imagecontent/registries.conf /etc/containers/
 COPY imagecontent/storage.conf /etc/containers/
+RUN mkdir /var/cache/blobs
+
 RUN ln -s /usr/bin/openshift-builder /usr/bin/openshift-sti-build && \
     ln -s /usr/bin/openshift-builder /usr/bin/openshift-docker-build && \
     ln -s /usr/bin/openshift-builder /usr/bin/openshift-git-clone && \

Dockerfile-dev

@@ -15,6 +15,7 @@ RUN INSTALL_PKGS=" \
 COPY imagecontent/policy.json /etc/containers/
 COPY imagecontent/registries.conf /etc/containers/
 COPY imagecontent/storage.conf /etc/containers/
+RUN mkdir /var/cache/blobs
 
 COPY openshift-builder /usr/bin
 RUN ln -s /usr/bin/openshift-builder /usr/bin/openshift-sti-build && \

Dockerfile.rhel7

@@ -16,6 +16,8 @@ COPY --from=builder /go/src/github.com/openshift/builder/openshift-builder /usr/
 COPY imagecontent/policy.json /etc/containers/
 COPY imagecontent/registries.conf /etc/containers/
 COPY imagecontent/storage.conf /etc/containers/
+RUN mkdir /var/cache/blobs
+
 RUN ln -s /usr/bin/openshift-builder /usr/bin/openshift-sti-build && \
     ln -s /usr/bin/openshift-builder /usr/bin/openshift-docker-build && \
     ln -s /usr/bin/openshift-builder /usr/bin/openshift-git-clone && \

glide.yaml

@@ -87,14 +87,13 @@ import:
 # we need some of the newer constants
 - package: golang.org/x/sys
   version: d641721ec2dead6fe5ca284096fe4b1fcd49e427
-- package: github.com/containers/storage
-  version: master
 # builds
 - package: github.com/containers/buildah
   version: master
 - package: github.com/containers/image
-  repo:    https://github.com/nalind/image
-  version: ip-registry
+  version: master
+- package: github.com/containers/storage
+  version: master
 # new enough to know how to disable itself when cross-compiling
 - package: github.com/opencontainers/selinux
   version: master

pkg/build/builder/cmd/builder.go

@@ -57,6 +57,7 @@ type builderConfig struct {
 	buildsClient    buildclientv1.BuildInterface
 	cleanup         func()
 	store           storage.Store
+	blobCache       string
 }
 
 func newBuilderConfigFromEnvironment(out io.Writer, needsDocker bool) (*builderConfig, error) {
@@ -131,7 +132,15 @@ func newBuilderConfigFromEnvironment(out io.Writer, needsDocker bool) (*builderC
 		}
 		istorage.Transport.SetStore(store)
 
-		dockerClient, err := bld.GetDaemonlessClient(systemContext, store, os.Getenv("BUILD_ISOLATION"))
+		// Default to using /var/cache/blobs as a blob cache, but allow its location
+		// to be changed by setting $BUILD_BLOBCACHE_DIR.  Setting the location to an
+		// empty value disables the cache.
+		cfg.blobCache = "/var/cache/blobs"
+		if blobCacheDir, isSet := os.LookupEnv("BUILD_BLOBCACHE_DIR"); isSet {
+			cfg.blobCache = blobCacheDir
+		}
+
+		dockerClient, err := bld.GetDaemonlessClient(systemContext, store, os.Getenv("BUILD_ISOLATION"), cfg.blobCache)
 		if err != nil {
 			return nil, fmt.Errorf("no daemonless store: %v", err)
 		}
@@ -260,7 +269,7 @@ func (c *builderConfig) extractImageContent() error {
 	}()
 
 	buildDir := bld.InputContentPath
-	return bld.ExtractImageContent(ctx, c.dockerClient, c.store, buildDir, c.build)
+	return bld.ExtractImageContent(ctx, c.dockerClient, c.store, buildDir, c.build, c.blobCache)
 }
 
 // execute is responsible for running a build

pkg/build/builder/daemonless.go

@@ -15,7 +15,6 @@ import (
 	"github.com/containers/buildah/imagebuildah"
 	"github.com/containers/buildah/util"
 	"github.com/containers/image/pkg/docker/config"
-	"github.com/containers/image/pkg/sysregistriesv2"
 	"github.com/containers/image/transports/alltransports"
 	"github.com/containers/image/types"
 	"github.com/containers/storage"
@@ -29,7 +28,7 @@ import (
 	"github.com/openshift/library-go/pkg/image/reference"
 )
 
-func pullDaemonlessImage(sc types.SystemContext, store storage.Store, imageName string, authConfig docker.AuthConfiguration) error {
+func pullDaemonlessImage(sc types.SystemContext, store storage.Store, imageName string, authConfig docker.AuthConfiguration, blobCacheDirectory string) error {
 	glog.V(2).Infof("Asked to pull fresh copy of %q.", imageName)
 
 	if imageName == "" {
@@ -47,11 +46,6 @@ func pullDaemonlessImage(sc types.SystemContext, store storage.Store, imageName
 	// }
 	systemContext.AuthFilePath = "/tmp/config.json"
 
-	registries, err := sysregistriesv2.GetRegistries(&systemContext)
-	if err != nil {
-		return fmt.Errorf("error reading system registries configuration: %v", err)
-	}
-
 	ref, err := reference.Parse(imageName)
 	if err != nil {
 		return fmt.Errorf("error parsing image name %s: %v", ref, err)
@@ -68,28 +62,17 @@ func pullDaemonlessImage(sc types.SystemContext, store storage.Store, imageName
 		}
 	}
 
-	if registry := sysregistriesv2.FindRegistry(imageName, registries); registry != nil {
-		if registry.Insecure {
-			glog.V(2).Infof("Registry %q is marked as insecure in the registries configuration.", registry.URL)
-			systemContext.DockerInsecureSkipTLSVerify = true
-			systemContext.OCIInsecureSkipTLSVerify = true
-		} else {
-			glog.V(2).Infof("Registry %q is marked as secure in the registries configuration.", registry.URL)
-		}
-	} else {
-		glog.V(2).Infof("Registry for %q is not present in the registries configuration, assuming it is secure.", imageName)
-	}
-
 	options := buildah.PullOptions{
 		ReportWriter:  os.Stderr,
 		Store:         store,
 		SystemContext: &systemContext,
+		BlobDirectory: blobCacheDirectory,
 	}
 	_, err = buildah.Pull(context.TODO(), "docker://"+imageName, options)
 	return err
 }
 
-func buildDaemonlessImage(sc types.SystemContext, store storage.Store, isolation buildah.Isolation, dir string, optimization buildapiv1.ImageOptimizationPolicy, opts *docker.BuildImageOptions) error {
+func buildDaemonlessImage(sc types.SystemContext, store storage.Store, isolation buildah.Isolation, contextDir string, optimization buildapiv1.ImageOptimizationPolicy, opts *docker.BuildImageOptions, blobCacheDirectory string) error {
 	glog.V(2).Infof("Building...")
 
 	args := make(map[string]string)
@@ -144,7 +127,7 @@ func buildDaemonlessImage(sc types.SystemContext, store storage.Store, isolation
 	}
 
 	options := imagebuildah.BuildOptions{
-		ContextDirectory: dir,
+		ContextDirectory: contextDir,
 		PullPolicy:       pullPolicy,
 		Isolation:        isolation,
 		TransientMounts:  transientMounts,
@@ -167,6 +150,7 @@ func buildDaemonlessImage(sc types.SystemContext, store storage.Store, isolation
 		NoCache:                 opts.NoCache,
 		RemoveIntermediateCtrs:  opts.RmTmpContainer,
 		ForceRmIntermediateCtrs: true,
+		BlobDirectory:           blobCacheDirectory,
 	}
 
 	_, _, err := imagebuildah.BuildDockerfiles(opts.Context, store, options, opts.Dockerfile)
@@ -229,7 +213,7 @@ func removeDaemonlessImage(sc types.SystemContext, store storage.Store, buildTag
 	return nil
 }
 
-func pushDaemonlessImage(sc types.SystemContext, store storage.Store, imageName string, authConfig docker.AuthConfiguration) (string, error) {
+func pushDaemonlessImage(sc types.SystemContext, store storage.Store, imageName string, authConfig docker.AuthConfiguration, blobCacheDirectory string) (string, error) {
 	glog.V(2).Infof("Pushing image %q from local storage.", imageName)
 
 	if imageName == "" {
@@ -257,26 +241,12 @@ func pushDaemonlessImage(sc types.SystemContext, store storage.Store, imageName
 		glog.V(2).Infof("No authentication secret provided for pushing to registry.")
 	}
 
-	registries, err := sysregistriesv2.GetRegistries(&systemContext)
-	if err != nil {
-		return "", fmt.Errorf("error reading system registries configuration: %v", err)
-	}
-	if registry := sysregistriesv2.FindRegistry(imageName, registries); registry != nil {
-		if registry.Insecure {
-			glog.V(2).Infof("Registry %q is marked as insecure in the registries configuration.", registry.URL)
-			systemContext.DockerInsecureSkipTLSVerify = true
-			systemContext.OCIInsecureSkipTLSVerify = true
-		} else {
-			glog.V(2).Infof("Registry %q is marked as secure in the registries configuration.", registry.URL)
-		}
-	} else {
-		glog.V(2).Infof("Registry for %q is not present in the registries configuration, assuming it is secure.", imageName)
-	}
-
 	options := buildah.PushOptions{
+		Compression:   archive.Gzip,
 		ReportWriter:  os.Stdout,
 		Store:         store,
 		SystemContext: &systemContext,
+		BlobDirectory: blobCacheDirectory,
 	}
 
 	// TODO - do something with the digest
@@ -369,7 +339,7 @@ func inspectDaemonlessImage(sc types.SystemContext, store storage.Store, name st
 
 // daemonlessRun mimics the 'docker run --rm' CLI command well enough. It creates and
 // starts a container and streams its logs. The container is removed after it terminates.
-func daemonlessRun(ctx context.Context, store storage.Store, isolation buildah.Isolation, createOpts docker.CreateContainerOptions, attachOpts docker.AttachToContainerOptions) error {
+func daemonlessRun(ctx context.Context, store storage.Store, isolation buildah.Isolation, createOpts docker.CreateContainerOptions, attachOpts docker.AttachToContainerOptions, blobCacheDirectory string) error {
 	if createOpts.Config == nil {
 		return fmt.Errorf("error calling daemonlessRun: expected a Config")
 	}
@@ -385,6 +355,7 @@ func daemonlessRun(ctx context.Context, store storage.Store, isolation buildah.I
 			MemorySwap:   createOpts.HostConfig.MemorySwap,
 			CgroupParent: createOpts.HostConfig.CgroupParent,
 		},
+		PullBlobDirectory: blobCacheDirectory,
 	}
 
 	builder, err := buildah.NewBuilder(ctx, store, builderOptions)
@@ -454,15 +425,16 @@ func downloadFromDaemonlessContainer(builder *buildah.Builder, id string, path s
 
 // DaemonlessClient is a daemonless DockerClient-like implementation.
 type DaemonlessClient struct {
-	SystemContext types.SystemContext
-	Store         storage.Store
-	Isolation     buildah.Isolation
-	builders      map[string]*buildah.Builder
+	SystemContext      types.SystemContext
+	Store              storage.Store
+	Isolation          buildah.Isolation
+	BlobCacheDirectory string
+	builders           map[string]*buildah.Builder
 }
 
 // GetDaemonlessClient returns a valid implemenatation of the DockerClient
 // interface, or an error if the implementation couldn't be created.
-func GetDaemonlessClient(systemContext types.SystemContext, store storage.Store, isolationSpec string) (client DockerClient, err error) {
+func GetDaemonlessClient(systemContext types.SystemContext, store storage.Store, isolationSpec, blobCacheDirectory string) (client DockerClient, err error) {
 	isolation := buildah.IsolationDefault
 	switch strings.ToLower(isolationSpec) {
 	case "chroot":
@@ -476,24 +448,29 @@ func GetDaemonlessClient(systemContext types.SystemContext, store storage.Store,
 		return nil, fmt.Errorf("unrecognized BUILD_ISOLATION setting %q", strings.ToLower(isolationSpec))
 	}
 
+	if blobCacheDirectory != "" {
+		glog.V(0).Infof("Caching blobs under %q.", blobCacheDirectory)
+	}
+
 	return &DaemonlessClient{
-		SystemContext: systemContext,
-		Store:         store,
-		Isolation:     isolation,
-		builders:      make(map[string]*buildah.Builder),
+		SystemContext:      systemContext,
+		Store:              store,
+		Isolation:          isolation,
+		BlobCacheDirectory: blobCacheDirectory,
+		builders:           make(map[string]*buildah.Builder),
 	}, nil
 }
 
 func (d *DaemonlessClient) BuildImage(opts docker.BuildImageOptions) error {
-	return buildDaemonlessImage(d.SystemContext, d.Store, d.Isolation, opts.ContextDir, buildapiv1.ImageOptimizationNone, &opts)
+	return buildDaemonlessImage(d.SystemContext, d.Store, d.Isolation, opts.ContextDir, buildapiv1.ImageOptimizationNone, &opts, d.BlobCacheDirectory)
 }
 
 func (d *DaemonlessClient) PushImage(opts docker.PushImageOptions, auth docker.AuthConfiguration) (string, error) {
 	imageName := opts.Name
 	if opts.Tag != "" {
 		imageName = imageName + ":" + opts.Tag
 	}
-	return pushDaemonlessImage(d.SystemContext, d.Store, imageName, auth)
+	return pushDaemonlessImage(d.SystemContext, d.Store, imageName, auth, d.BlobCacheDirectory)
 }
 
 func (d *DaemonlessClient) RemoveImage(name string) error {
@@ -502,8 +479,9 @@ func (d *DaemonlessClient) RemoveImage(name string) error {
 
 func (d *DaemonlessClient) CreateContainer(opts docker.CreateContainerOptions) (*docker.Container, error) {
 	options := buildah.BuilderOptions{
-		FromImage: opts.Config.Image,
-		Container: opts.Name,
+		FromImage:         opts.Config.Image,
+		Container:         opts.Name,
+		PullBlobDirectory: d.BlobCacheDirectory,
 	}
 	builder, err := buildah.NewBuilder(opts.Context, d.Store, options)
 	if err != nil {
@@ -556,7 +534,7 @@ func (d *DaemonlessClient) PullImage(opts docker.PullImageOptions, auth docker.A
 	if opts.Tag != "" {
 		imageName = imageName + ":" + opts.Tag
 	}
-	return pullDaemonlessImage(d.SystemContext, d.Store, imageName, auth)
+	return pullDaemonlessImage(d.SystemContext, d.Store, imageName, auth, d.BlobCacheDirectory)
 }
 
 func (d *DaemonlessClient) TagImage(name string, opts docker.TagImageOptions) error {