Merge pull request #353 from priyawadhwa/cache

Add layer caching to kaniko

Author: priyawadhwa

cmd/executor/cmd/root.go

@@ -95,6 +95,8 @@ func addKanikoOptionsFlags(cmd *cobra.Command) {
 	RootCmd.PersistentFlags().BoolVarP(&opts.Reproducible, "reproducible", "", false, "Strip timestamps out of the image to make it reproducible")
 	RootCmd.PersistentFlags().StringVarP(&opts.Target, "target", "", "", "Set the target build stage to build")
 	RootCmd.PersistentFlags().BoolVarP(&opts.NoPush, "no-push", "", false, "Do not push the image to the registry")
+	RootCmd.PersistentFlags().StringVarP(&opts.CacheRepo, "cache-repo", "", "", "Specify a repository to use as a cache, otherwise one will be inferred from the destination provided")
+	RootCmd.PersistentFlags().BoolVarP(&opts.Cache, "cache", "", false, "Use cache when building image")
 }
 
 // addHiddenFlags marks certain flags as hidden from the executor help text

integration/dockerfiles/Dockerfile_test_cache

@@ -0,0 +1,22 @@
+# Copyright 2018 Google, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Test to make sure the cache works properly
+# If the image is built twice, /date should be the same in both images
+# if the cache is implemented correctly
+
+FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0
+RUN date > /date
+COPY context/foo /foo
+RUN echo hey

integration/dockerfiles/Dockerfile_test_cache_install

@@ -0,0 +1,20 @@
+# Copyright 2018 Google, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Test to make sure the cache works properly
+# /date should be the same regardless of when this image is built
+# if the cache is implemented correctly
+
+FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0
+RUN apt-get update && apt-get install -y make

integration/images.go

@@ -23,6 +23,7 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 )
 
@@ -77,12 +78,16 @@ func GetKanikoImage(imageRepo, dockerfile string) string {
 	return strings.ToLower(imageRepo + kanikoPrefix + dockerfile)
 }
 
+// GetVersionedKanikoImage versions constructs the name of the kaniko image that would be built
+// with the dockerfile and versions it for cache testing
+func GetVersionedKanikoImage(imageRepo, dockerfile string, version int) string {
+	return strings.ToLower(imageRepo + kanikoPrefix + dockerfile + strconv.Itoa(version))
+}
+
 // FindDockerFiles will look for test docker files in the directory dockerfilesPath.
 // These files must start with `Dockerfile_test`. If the file is one we are intentionally
 // skipping, it will not be included in the returned list.
 func FindDockerFiles(dockerfilesPath string) ([]string, error) {
-	// TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed
-	testsToIgnore := map[string]bool{"Dockerfile_test_user_run": true}
 	allDockerfiles, err := filepath.Glob(path.Join(dockerfilesPath, "Dockerfile_test*"))
 	if err != nil {
 		return []string{}, fmt.Errorf("Failed to find docker files at %s: %s", dockerfilesPath, err)
@@ -92,9 +97,8 @@ func FindDockerFiles(dockerfilesPath string) ([]string, error) {
 	for _, dockerfile := range allDockerfiles {
 		// Remove the leading directory from the path
 		dockerfile = dockerfile[len("dockerfiles/"):]
-		if !testsToIgnore[dockerfile] {
-			dockerfiles = append(dockerfiles, dockerfile)
-		}
+		dockerfiles = append(dockerfiles, dockerfile)
+
 	}
 	return dockerfiles, err
 }
@@ -103,7 +107,9 @@ func FindDockerFiles(dockerfilesPath string) ([]string, error) {
 // keeps track of which files have been built.
 type DockerFileBuilder struct {
 	// Holds all available docker files and whether or not they've been built
-	FilesBuilt map[string]bool
+	FilesBuilt           map[string]bool
+	DockerfilesToIgnore  map[string]struct{}
+	TestCacheDockerfiles map[string]struct{}
 }
 
 // NewDockerFileBuilder will create a DockerFileBuilder initialized with dockerfiles, which
@@ -113,6 +119,14 @@ func NewDockerFileBuilder(dockerfiles []string) *DockerFileBuilder {
 	for _, f := range dockerfiles {
 		d.FilesBuilt[f] = false
 	}
+	d.DockerfilesToIgnore = map[string]struct{}{
+		// TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed
+		"Dockerfile_test_user_run": {},
+	}
+	d.TestCacheDockerfiles = map[string]struct{}{
+		"Dockerfile_test_cache":         {},
+		"Dockerfile_test_cache_install": {},
+	}
 	return &d
 }
 
@@ -186,3 +200,31 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do
 	d.FilesBuilt[dockerfile] = true
 	return nil
 }
+
+// buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built
+func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesPath string, version int) error {
+	_, ex, _, _ := runtime.Caller(0)
+	cwd := filepath.Dir(ex)
+
+	cacheFlag := "--cache=true"
+
+	for dockerfile := range d.TestCacheDockerfiles {
+		kanikoImage := GetVersionedKanikoImage(imageRepo, dockerfile, version)
+		kanikoCmd := exec.Command("docker",
+			append([]string{"run",
+				"-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
+				"-v", cwd + ":/workspace",
+				ExecutorImage,
+				"-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
+				"-d", kanikoImage,
+				"-c", buildContextPath,
+				cacheFlag,
+				"--cache-repo", cacheRepo})...,
+		)
+
+		if _, err := RunCommandWithoutTest(kanikoCmd); err != nil {
+			return fmt.Errorf("Failed to build cached image %s with kaniko command \"%s\": %s", kanikoImage, kanikoCmd.Args, err)
+		}
+	}
+	return nil
+}

integration/integration_test.go

@@ -24,8 +24,10 @@ import (
 	"math"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/daemon"
@@ -148,6 +150,7 @@ func TestMain(m *testing.M) {
 		fmt.Printf("error building onbuild base: %v", err)
 		os.Exit(1)
 	}
+
 	pushOnbuildBase := exec.Command("docker", "push", config.onbuildBaseImage)
 	if err := pushOnbuildBase.Run(); err != nil {
 		fmt.Printf("error pushing onbuild base %s: %v", config.onbuildBaseImage, err)
@@ -165,7 +168,6 @@ func TestMain(m *testing.M) {
 		fmt.Printf("error pushing hardlink base %s: %v", config.hardlinkBaseImage, err)
 		os.Exit(1)
 	}
-
 	dockerfiles, err := FindDockerFiles(dockerfilesPath)
 	if err != nil {
 		fmt.Printf("Coudn't create map of dockerfiles: %s", err)
@@ -177,6 +179,12 @@ func TestMain(m *testing.M) {
 func TestRun(t *testing.T) {
 	for dockerfile, built := range imageBuilder.FilesBuilt {
 		t.Run("test_"+dockerfile, func(t *testing.T) {
+			if _, ok := imageBuilder.DockerfilesToIgnore[dockerfile]; ok {
+				t.SkipNow()
+			}
+			if _, ok := imageBuilder.TestCacheDockerfiles[dockerfile]; ok {
+				t.SkipNow()
+			}
 			if !built {
 				err := imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile)
 				if err != nil {
@@ -195,25 +203,8 @@ func TestRun(t *testing.T) {
 			t.Logf("diff = %s", string(diff))
 
 			expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage)
+			checkContainerDiffOutput(t, diff, expected)
 
-			// Let's compare the json objects themselves instead of strings to avoid
-			// issues with spaces and indents
-			var diffInt interface{}
-			var expectedInt interface{}
-
-			err := json.Unmarshal(diff, &diffInt)
-			if err != nil {
-				t.Error(err)
-				t.Fail()
-			}
-
-			err = json.Unmarshal([]byte(expected), &expectedInt)
-			if err != nil {
-				t.Error(err)
-				t.Fail()
-			}
-
-			testutil.CheckErrorAndDeepEqual(t, false, nil, expectedInt, diffInt)
 		})
 	}
 }
@@ -228,6 +219,9 @@ func TestLayers(t *testing.T) {
 	}
 	for dockerfile, built := range imageBuilder.FilesBuilt {
 		t.Run("test_layer_"+dockerfile, func(t *testing.T) {
+			if _, ok := imageBuilder.DockerfilesToIgnore[dockerfile]; ok {
+				t.SkipNow()
+			}
 			if !built {
 				err := imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile)
 				if err != nil {
@@ -244,6 +238,58 @@ func TestLayers(t *testing.T) {
 	}
 }
 
+// Build each image with kaniko twice, and then make sure they're exactly the same
+func TestCache(t *testing.T) {
+	for dockerfile := range imageBuilder.TestCacheDockerfiles {
+		t.Run("test_cache_"+dockerfile, func(t *testing.T) {
+			cache := filepath.Join(config.imageRepo, "cache", fmt.Sprintf("%v", time.Now().UnixNano()))
+			// Build the initial image which will cache layers
+			if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, 0); err != nil {
+				t.Fatalf("error building cached image for the first time: %v", err)
+			}
+			// Build the second image which should pull from the cache
+			if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, 1); err != nil {
+				t.Fatalf("error building cached image for the first time: %v", err)
+			}
+			// Make sure both images are the same
+			kanikoVersion0 := GetVersionedKanikoImage(config.imageRepo, dockerfile, 0)
+			kanikoVersion1 := GetVersionedKanikoImage(config.imageRepo, dockerfile, 1)
+
+			// container-diff
+			containerdiffCmd := exec.Command("container-diff", "diff",
+				kanikoVersion0, kanikoVersion1,
+				"-q", "--type=file", "--type=metadata", "--json")
+
+			diff := RunCommand(containerdiffCmd, t)
+			t.Logf("diff = %s", diff)
+
+			expected := fmt.Sprintf(emptyContainerDiff, kanikoVersion0, kanikoVersion1, kanikoVersion0, kanikoVersion1)
+			checkContainerDiffOutput(t, diff, expected)
+		})
+	}
+}
+
+func checkContainerDiffOutput(t *testing.T, diff []byte, expected string) {
+	// Let's compare the json objects themselves instead of strings to avoid
+	// issues with spaces and indents
+	t.Helper()
+
+	var diffInt interface{}
+	var expectedInt interface{}
+
+	err := json.Unmarshal(diff, &diffInt)
+	if err != nil {
+		t.Error(err)
+	}
+
+	err = json.Unmarshal([]byte(expected), &expectedInt)
+	if err != nil {
+		t.Error(err)
+	}
+
+	testutil.CheckErrorAndDeepEqual(t, false, nil, expectedInt, diffInt)
+}
+
 func checkLayers(t *testing.T, image1, image2 string, offset int) {
 	t.Helper()
 	img1, err := getImageDetails(image1)

pkg/cache/cache.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import (
+	"fmt"
+
+	"github.com/GoogleContainerTools/kaniko/pkg/config"
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/authn/k8schain"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// RetrieveLayer checks the specified cache for a layer with the tag :cacheKey
+func RetrieveLayer(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) {
+	cache, err := Destination(opts, cacheKey)
+	if err != nil {
+		return nil, errors.Wrap(err, "getting cache destination")
+	}
+	logrus.Infof("Checking for cached layer %s...", cache)
+
+	cacheRef, err := name.NewTag(cache, name.WeakValidation)
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("getting reference for %s", cache))
+	}
+	k8sc, err := k8schain.NewNoClient()
+	if err != nil {
+		return nil, err
+	}
+	kc := authn.NewMultiKeychain(authn.DefaultKeychain, k8sc)
+	img, err := remote.Image(cacheRef, remote.WithAuthFromKeychain(kc))
+	if err != nil {
+		return nil, err
+	}
+	_, err = img.Layers()
+	return img, err
+}
+
+// Destination returns the repo where the layer should be stored
+// If no cache is specified, one is inferred from the destination provided
+func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) {
+	cache := opts.CacheRepo
+	if cache == "" {
+		destination := opts.Destinations[0]
+		destRef, err := name.NewTag(destination, name.WeakValidation)
+		if err != nil {
+			return "", errors.Wrap(err, "getting tag for destination")
+		}
+		return fmt.Sprintf("%s/cache:%s", destRef.Context(), cacheKey), nil
+	}
+	return fmt.Sprintf("%s:%s", cache, cacheKey), nil
+}

pkg/config/options.go

@@ -24,11 +24,13 @@ type KanikoOptions struct {
 	Bucket         string
 	TarPath        string
 	Target         string
+	CacheRepo      string
 	Destinations   multiArg
 	BuildArgs      multiArg
 	InsecurePush   bool
 	SkipTLSVerify  bool
 	SingleSnapshot bool
 	Reproducible   bool
 	NoPush         bool
+	Cache          bool
 }