From fcfc2f416baf4cae0191392a3afeff01aa20a495 Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Thu, 30 Aug 2018 17:40:05 -0700 Subject: [PATCH 01/15] comments --- pkg/executor/build.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 22807d6a7a..78635558b0 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -79,6 +79,12 @@ func DoBuild(opts *options.KanikoOptions) (v1.Image, error) { for index, cmd := range stage.Commands { finalCmd := index == len(stage.Commands)-1 dockerCommand, err := commands.GetCommand(cmd, opts.SrcContext) + // Check the cache here. Create the key based on the command. + + // Create key. Pass in command object. + + // Check Registry using image_util.remoteImage + if err != nil { return nil, err } From 8dd6d4756e3aa7806ed5b7d5627367349f8b66c0 Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Tue, 2 Oct 2018 14:23:24 -0700 Subject: [PATCH 02/15] initial commit for persisent volume caching --- cmd/warmer/cmd/root.go | 92 ++++++++++++++++++++++++++++++++++++++++ cmd/warmer/main.go | 29 +++++++++++++ kaniko-cache-claim.yaml | 11 +++++ kaniko-cache-volume.yaml | 14 ++++++ kaniko-test.yaml | 28 ++++++++++++ pkg/cache/cache.go | 30 +++++++++++++ pkg/config/options.go | 1 + pkg/executor/build.go | 2 +- pkg/util/image_util.go | 13 +++++- 9 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 cmd/warmer/cmd/root.go create mode 100644 cmd/warmer/main.go create mode 100644 kaniko-cache-claim.yaml create mode 100644 kaniko-cache-volume.yaml create mode 100644 kaniko-test.yaml diff --git a/cmd/warmer/cmd/root.go b/cmd/warmer/cmd/root.go new file mode 100644 index 0000000000..4108616095 --- /dev/null +++ b/cmd/warmer/cmd/root.go @@ -0,0 +1,92 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/GoogleContainerTools/kaniko/pkg/buildcontext" + "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/GoogleContainerTools/kaniko/pkg/constants" + "github.com/GoogleContainerTools/kaniko/pkg/executor" + "github.com/GoogleContainerTools/kaniko/pkg/util" + "github.com/genuinetools/amicontained/container" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var ( + opts = &config.KanikoOptions{} + logLevel string + force bool +) + +func init() { + RootCmd.PersistentFlags().StringVarP(&logLevel, "verbosity", "v", constants.DefaultLogLevel, "Log level (debug, info, warn, error, fatal, panic") + addKanikoOptionsFlags(RootCmd) + addHiddenFlags(RootCmd) +} + +var RootCmd = &cobra.Command{ + Use: "cache warmer", + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if err := util.ConfigureLogging(logLevel); err != nil { + return err + } + if len(opts.Images) == 0 { + return errors.New("You must select at least one image to cache") + } + return nil + }, + Run: func(cmd *cobra.Command, args []string) { + if !checkContained() { + if !force { + exit(errors.New("kaniko should only be run inside of a container, run with the --force flag if you are sure you want to continue")) + } + logrus.Warn("kaniko is being run outside of a container. This can have dangerous effects on your system") + } + if err := os.Chdir("/"); err != nil { + exit(errors.Wrap(err, "error changing to root dir")) + } + image, err := executor.DoBuild(opts) + if err != nil { + exit(errors.Wrap(err, "error building image")) + } + if err := executor.DoPush(image, opts); err != nil { + exit(errors.Wrap(err, "error pushing image")) + } + }, +} + +// addKanikoOptionsFlags configures opts +func addKanikoOptionsFlags(cmd *cobra.Command) { + RootCmd.PersistentFlags().VarP(&opts.Images, "image", "i", "Image to cache. Set it repeatedly for multiple images.") + RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "c", "/cache", "Directory of the cache.") +} + +// addHiddenFlags marks certain flags as hidden from the executor help text +func addHiddenFlags(cmd *cobra.Command) { +} + +func exit(err error) { + fmt.Println(err) + os.Exit(1) +} diff --git a/cmd/warmer/main.go b/cmd/warmer/main.go new file mode 100644 index 0000000000..68f60c4e98 --- /dev/null +++ b/cmd/warmer/main.go @@ -0,0 +1,29 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + + "github.com/GoogleContainerTools/kaniko/cmd/executor/cmd" +) + +func main() { + if err := cmd.RootCmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/kaniko-cache-claim.yaml b/kaniko-cache-claim.yaml new file mode 100644 index 0000000000..dc30c3a8d8 --- /dev/null +++ b/kaniko-cache-claim.yaml @@ -0,0 +1,11 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: kaniko-cache-claim +spec: + storageClassName: manual + accessModes: + - ReadOnlyMany + resources: + requests: + storage: 8Gi diff --git a/kaniko-cache-volume.yaml b/kaniko-cache-volume.yaml new file mode 100644 index 0000000000..7fc57336cf --- /dev/null +++ b/kaniko-cache-volume.yaml @@ -0,0 +1,14 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: kaniko-cache-volume + labels: + type: local +spec: + storageClassName: manual + capacity: + storage: 10Gi + accessModes: + - ReadOnlyMany + hostPath: + path: "/mnt/data" diff --git a/kaniko-test.yaml b/kaniko-test.yaml new file mode 100644 index 0000000000..4553fe3c3b --- /dev/null +++ b/kaniko-test.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kaniko +spec: + containers: + - name: kaniko + image: gcr.io/kaniko-project/executor:latest + args: ["--dockerfile=dockerfiles/Dockerfile_test_add", + "--context=gs://sharif-test/context_1533688591445158884.tar.gz", + "--destination=gcr.io/sharif-test/kaniko-test:cache"] + volumeMounts: + - name: kaniko-secret + mountPath: /secret + - name: kaniko-cache + mountPath: /cache + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /secret/kaniko-secret.json + restartPolicy: Never + volumes: + - name: kaniko-secret + secret: + secretName: kaniko-secret + - name: kaniko-cache + persistentVolumeClaim: + claimName: kaniko-cache-claim + diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 69682ba484..1cb23a3ad8 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -18,6 +18,7 @@ package cache import ( "fmt" + "path" "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/google/go-containerregistry/pkg/authn" @@ -25,12 +26,17 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) // RetrieveLayer checks the specified cache for a layer with the tag :cacheKey func RetrieveLayer(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) { + if local, _ := LocalDestination(opts, cacheKey); local != nil { + return local, nil + } + cache, err := Destination(opts, cacheKey) if err != nil { return nil, errors.Wrap(err, "getting cache destination") @@ -68,3 +74,27 @@ func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) { } return fmt.Sprintf("%s:%s", cache, cacheKey), nil } + +func LocalDestination(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) { + cache := opts.CacheDir + if cache == "" { + return nil, nil + } + + path := path.Join(cache, cacheKey) + destination := opts.Destinations[0] + destRef, err := name.NewTag(destination, name.WeakValidation) + if err != nil { + return nil, errors.Wrap(err, "volume cache: getting tag for destination") + } + tag, err := name.NewTag(fmt.Sprintf("%s/cache:%s", destRef.Context(), cacheKey), name.WeakValidation) + if err != nil { + return nil, errors.Wrap(err, "volume cache: create new cache tag") + } + imgTar, err := tarball.ImageFromPath(path, &tag) + if err != nil { + return nil, errors.Wrap(err, "getting layer from tarball") + } + + return imgTar, nil +} diff --git a/pkg/config/options.go b/pkg/config/options.go index c9bad39e66..6bf593ebe1 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -25,6 +25,7 @@ type KanikoOptions struct { TarPath string Target string CacheRepo string + CacheDir string Destinations multiArg BuildArgs multiArg InsecurePush bool diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 29f52d6ff4..ba82f9b777 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -54,7 +54,7 @@ type stageBuilder struct { // newStageBuilder returns a new type stageBuilder which contains all the information required to build the stage func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage) (*stageBuilder, error) { - sourceImage, err := util.RetrieveSourceImage(stage, opts.BuildArgs) + sourceImage, err := util.RetrieveSourceImage(stage, opts.BuildArgs, opts) if err != nil { return nil, err } diff --git a/pkg/util/image_util.go b/pkg/util/image_util.go index d19e684f66..26677903ab 100644 --- a/pkg/util/image_util.go +++ b/pkg/util/image_util.go @@ -30,6 +30,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/sirupsen/logrus" + "github.com/GoogleContainerTools/kaniko/pkg/cache" "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/constants" ) @@ -41,7 +42,7 @@ var ( ) // RetrieveSourceImage returns the base image of the stage at index -func RetrieveSourceImage(stage config.KanikoStage, buildArgs []string) (v1.Image, error) { +func RetrieveSourceImage(stage config.KanikoStage, buildArgs []string, opts *config.KanikoOptions) (v1.Image, error) { currentBaseName, err := ResolveEnvironmentReplacement(stage.BaseName, buildArgs, false) if err != nil { return nil, err @@ -57,6 +58,12 @@ func RetrieveSourceImage(stage config.KanikoStage, buildArgs []string) (v1.Image return retrieveTarImage(stage.BaseImageIndex) } + // Next, check if local caching is enabled + // If so, look in the local cache before trying the remote registry + if opts.Cache && opts.CacheDir != "" { + return cachedImage(opts, currentBaseName) + } + // Otherwise, initialize image as usual return retrieveRemoteImage(currentBaseName) } @@ -92,3 +99,7 @@ func remoteImage(image string) (v1.Image, error) { kc := authn.NewMultiKeychain(authn.DefaultKeychain, k8sc) return remote.Image(ref, remote.WithAuthFromKeychain(kc)) } + +func cachedImage(opts *config.KanikoOptions, image string) (v1.Image, error) { + return cache.LocalDestination(opts, image) +} From 979092eddb462665f17099b834902474e7f91690 Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Wed, 3 Oct 2018 15:41:35 -0700 Subject: [PATCH 03/15] cache warmer works --- Makefile | 4 +++ cmd/warmer/cmd/root.go | 26 +++------------ cmd/warmer/main.go | 2 +- pkg/cache/warm.go | 72 ++++++++++++++++++++++++++++++++++++++++++ pkg/config/options.go | 6 ++++ 5 files changed, 87 insertions(+), 23 deletions(-) create mode 100644 pkg/cache/warm.go diff --git a/Makefile b/Makefile index 8d20c2b031..a56e5455b8 100644 --- a/Makefile +++ b/Makefile @@ -35,11 +35,15 @@ GO_LDFLAGS += -w -s # Drop debugging symbols. GO_LDFLAGS += ' EXECUTOR_PACKAGE = $(REPOPATH)/cmd/executor +WARMER_PACKAGE = $(REPOPATH)/cmd/warmer KANIKO_PROJECT = $(REPOPATH)/kaniko out/executor: $(GO_FILES) GOARCH=$(GOARCH) GOOS=linux CGO_ENABLED=0 go build -ldflags $(GO_LDFLAGS) -o $@ $(EXECUTOR_PACKAGE) +out/warmer: $(GO_FILES) + GOARCH=$(GOARCH) GOOS=linux CGO_ENABLED=0 go build -ldflags $(GO_LDFLAGS) -o $@ $(WARMER_PACKAGE) + .PHONY: test test: out/executor @ ./test.sh diff --git a/cmd/warmer/cmd/root.go b/cmd/warmer/cmd/root.go index 4108616095..54c26fce91 100644 --- a/cmd/warmer/cmd/root.go +++ b/cmd/warmer/cmd/root.go @@ -19,22 +19,17 @@ package cmd import ( "fmt" "os" - "path/filepath" - "strings" - "github.com/GoogleContainerTools/kaniko/pkg/buildcontext" + "github.com/GoogleContainerTools/kaniko/pkg/cache" "github.com/GoogleContainerTools/kaniko/pkg/config" "github.com/GoogleContainerTools/kaniko/pkg/constants" - "github.com/GoogleContainerTools/kaniko/pkg/executor" "github.com/GoogleContainerTools/kaniko/pkg/util" - "github.com/genuinetools/amicontained/container" "github.com/pkg/errors" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) var ( - opts = &config.KanikoOptions{} + opts = &config.WarmerOptions{} logLevel string force bool ) @@ -57,21 +52,8 @@ var RootCmd = &cobra.Command{ return nil }, Run: func(cmd *cobra.Command, args []string) { - if !checkContained() { - if !force { - exit(errors.New("kaniko should only be run inside of a container, run with the --force flag if you are sure you want to continue")) - } - logrus.Warn("kaniko is being run outside of a container. This can have dangerous effects on your system") - } - if err := os.Chdir("/"); err != nil { - exit(errors.Wrap(err, "error changing to root dir")) - } - image, err := executor.DoBuild(opts) - if err != nil { - exit(errors.Wrap(err, "error building image")) - } - if err := executor.DoPush(image, opts); err != nil { - exit(errors.Wrap(err, "error pushing image")) + if err := cache.WarmCache(opts); err != nil { + exit(errors.Wrap(err, "Failed warming cache")) } }, } diff --git a/cmd/warmer/main.go b/cmd/warmer/main.go index 68f60c4e98..c91eba44a4 100644 --- a/cmd/warmer/main.go +++ b/cmd/warmer/main.go @@ -19,7 +19,7 @@ package main import ( "os" - "github.com/GoogleContainerTools/kaniko/cmd/executor/cmd" + "github.com/GoogleContainerTools/kaniko/cmd/warmer/cmd" ) func main() { diff --git a/pkg/cache/warm.go b/pkg/cache/warm.go new file mode 100644 index 0000000000..5ccd5a5f53 --- /dev/null +++ b/pkg/cache/warm.go @@ -0,0 +1,72 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "fmt" + "path" + + "github.com/GoogleContainerTools/kaniko/pkg/config" + "github.com/google/go-containerregistry/pkg/authn/k8schain" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +func WarmCache(opts *config.WarmerOptions) error { + cacheDir := opts.CacheDir + images := opts.Images + fmt.Printf("WARMER: cachedir = %s\n", cacheDir) + fmt.Printf("WARMER: images = %v\n", images) + + for _, image := range images { + cacheRef, err := name.NewTag(image, name.WeakValidation) + if err != nil { + fmt.Print(err) + errors.Wrap(err, fmt.Sprintf("Failed to verify image name: %s", image)) + } + k8sc, err := k8schain.NewNoClient() + if err != nil { + fmt.Print(err) + errors.Wrap(err, fmt.Sprintf("Failed to create auth keychain: %s", image)) + } + img, err := remote.Image(cacheRef, remote.WithAuthFromKeychain(k8sc)) + if err != nil { + fmt.Print(err) + errors.Wrap(err, fmt.Sprintf("Failed to retrieve image: %s", image)) + } + + digest, err := img.Digest() + if err != nil { + fmt.Print(err) + errors.Wrap(err, fmt.Sprintf("Failed to retrieve digest: %s", image)) + } + cachePath := path.Join(cacheDir, digest.String()) + err = tarball.WriteToFile(cachePath, cacheRef, img) + if err != nil { + fmt.Print(err) + errors.Wrap(err, fmt.Sprintf("Failed to write %s to cache", image)) + } else { + fmt.Printf("Wrote %s to cache", image) + logrus.Debugf("Wrote %s to cache", image) + } + + } + return nil +} diff --git a/pkg/config/options.go b/pkg/config/options.go index 496434619c..fc15c1f213 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -36,3 +36,9 @@ type KanikoOptions struct { Cache bool Cleanup bool } + +// WarmerOptions are options that are set by command line arguments to the cache warmer. +type WarmerOptions struct { + Images multiArg + CacheDir string +} From ad4f6048c367c3fe2446eb1ad81f1e7a4a400760 Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Wed, 3 Oct 2018 15:46:07 -0700 Subject: [PATCH 04/15] general cleanup --- kaniko-test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kaniko-test.yaml b/kaniko-test.yaml index 4553fe3c3b..f18037d574 100644 --- a/kaniko-test.yaml +++ b/kaniko-test.yaml @@ -6,9 +6,9 @@ spec: containers: - name: kaniko image: gcr.io/kaniko-project/executor:latest - args: ["--dockerfile=dockerfiles/Dockerfile_test_add", - "--context=gs://sharif-test/context_1533688591445158884.tar.gz", - "--destination=gcr.io/sharif-test/kaniko-test:cache"] + args: ["--dockerfile=", + "--context=", + "--destination="] volumeMounts: - name: kaniko-secret mountPath: /secret From 69a760e5a432467dd0ddbd86bd0845fc0db4e3e4 Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Mon, 8 Oct 2018 10:13:59 -0700 Subject: [PATCH 05/15] adding some debugging --- Makefile | 1 + cmd/warmer/cmd/root.go | 2 +- kaniko-cache-volume.yaml | 2 +- pkg/cache/cache.go | 1 + pkg/cache/warm.go | 16 ++++++++-------- pkg/util/image_util.go | 9 ++++++++- 6 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index a56e5455b8..94c18ad400 100644 --- a/Makefile +++ b/Makefile @@ -56,3 +56,4 @@ integration-test: images: docker build -t $(REGISTRY)/executor:latest -f deploy/Dockerfile . docker build -t $(REGISTRY)/executor:debug -f deploy/Dockerfile_debug . + docker build -t $(REGISTRY)/warmer:latest -f deploy/Dockerfile_warmer . diff --git a/cmd/warmer/cmd/root.go b/cmd/warmer/cmd/root.go index 54c26fce91..0e4908d2b2 100644 --- a/cmd/warmer/cmd/root.go +++ b/cmd/warmer/cmd/root.go @@ -31,7 +31,6 @@ import ( var ( opts = &config.WarmerOptions{} logLevel string - force bool ) func init() { @@ -66,6 +65,7 @@ func addKanikoOptionsFlags(cmd *cobra.Command) { // addHiddenFlags marks certain flags as hidden from the executor help text func addHiddenFlags(cmd *cobra.Command) { + RootCmd.PersistentFlags().MarkHidden("azure-container-registry-config") } func exit(err error) { diff --git a/kaniko-cache-volume.yaml b/kaniko-cache-volume.yaml index 7fc57336cf..700aa29991 100644 --- a/kaniko-cache-volume.yaml +++ b/kaniko-cache-volume.yaml @@ -11,4 +11,4 @@ spec: accessModes: - ReadOnlyMany hostPath: - path: "/mnt/data" + path: "/tmp/kaniko-cache" diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 1cb23a3ad8..c3e2bc89ba 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -96,5 +96,6 @@ func LocalDestination(opts *config.KanikoOptions, cacheKey string) (v1.Image, er return nil, errors.Wrap(err, "getting layer from tarball") } + fmt.Printf("Found %s in local cache", tag.Name()) return imgTar, nil } diff --git a/pkg/cache/warm.go b/pkg/cache/warm.go index 5ccd5a5f53..a228b0972e 100644 --- a/pkg/cache/warm.go +++ b/pkg/cache/warm.go @@ -32,38 +32,38 @@ import ( func WarmCache(opts *config.WarmerOptions) error { cacheDir := opts.CacheDir images := opts.Images - fmt.Printf("WARMER: cachedir = %s\n", cacheDir) - fmt.Printf("WARMER: images = %v\n", images) + logrus.Debugf("%s\n", cacheDir) + logrus.Debugf("%s\n", images) for _, image := range images { cacheRef, err := name.NewTag(image, name.WeakValidation) if err != nil { - fmt.Print(err) + logrus.Debugf("Failed to verify image name %v", err) errors.Wrap(err, fmt.Sprintf("Failed to verify image name: %s", image)) } k8sc, err := k8schain.NewNoClient() if err != nil { - fmt.Print(err) + logrus.Debugf("Failed to create auth keychain: %v", err) errors.Wrap(err, fmt.Sprintf("Failed to create auth keychain: %s", image)) } img, err := remote.Image(cacheRef, remote.WithAuthFromKeychain(k8sc)) if err != nil { - fmt.Print(err) + logrus.Debugf("Failed to retrieve image: %v", err) errors.Wrap(err, fmt.Sprintf("Failed to retrieve image: %s", image)) } digest, err := img.Digest() if err != nil { - fmt.Print(err) + logrus.Debugf("Failed to retrieve digest: %v", err) errors.Wrap(err, fmt.Sprintf("Failed to retrieve digest: %s", image)) } cachePath := path.Join(cacheDir, digest.String()) + logrus.Debugf("CACHE PATH = %s\n", cachePath) err = tarball.WriteToFile(cachePath, cacheRef, img) if err != nil { - fmt.Print(err) + logrus.Debugf("Failed to write to cache %v", err) errors.Wrap(err, fmt.Sprintf("Failed to write %s to cache", image)) } else { - fmt.Printf("Wrote %s to cache", image) logrus.Debugf("Wrote %s to cache", image) } diff --git a/pkg/util/image_util.go b/pkg/util/image_util.go index 26677903ab..95ae672f9c 100644 --- a/pkg/util/image_util.go +++ b/pkg/util/image_util.go @@ -17,6 +17,7 @@ limitations under the License. package util import ( + "fmt" "path/filepath" "strconv" @@ -101,5 +102,11 @@ func remoteImage(image string) (v1.Image, error) { } func cachedImage(opts *config.KanikoOptions, image string) (v1.Image, error) { - return cache.LocalDestination(opts, image) + ref, err := name.ParseReference(image, name.WeakValidation) + if err != nil { + return nil, err + } + cacheKey := ref.Name() + fmt.Printf("CACHEKEY=%s", cacheKey) + return cache.LocalDestination(opts, cacheKey) } From c44bd3480165f917c0d0e9e96e853f3bad945e2c Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Mon, 8 Oct 2018 10:38:53 -0700 Subject: [PATCH 06/15] adding missing files --- deploy/Dockerfile_warmer | 42 ++++++++++++++++++++++++++++++++++++++++ kaniko-warmer.yaml | 28 +++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 deploy/Dockerfile_warmer create mode 100644 kaniko-warmer.yaml diff --git a/deploy/Dockerfile_warmer b/deploy/Dockerfile_warmer new file mode 100644 index 0000000000..c74c3d3ed0 --- /dev/null +++ b/deploy/Dockerfile_warmer @@ -0,0 +1,42 @@ +# Copyright 2018 Google, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Builds the static Go image to execute in a Kubernetes job + +FROM golang:1.10 +WORKDIR /go/src/github.com/GoogleContainerTools/kaniko +COPY . . +RUN make +# Get GCR credential helper +ADD https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v1.4.3-static/docker-credential-gcr_linux_amd64-1.4.3.tar.gz /usr/local/bin/ +RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-gcr_linux_amd64-1.4.3.tar.gz +# Get Amazon ECR credential helper +RUN go get -u github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login +RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 + +FROM scratch +COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/warmer /kaniko/warmer +COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr +COPY --from=0 /go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/linux-amd64/docker-credential-ecr-login /kaniko/docker-credential-ecr-login +COPY files/ca-certificates.crt /kaniko/ssl/certs/ +COPY files/config.json /kaniko/.docker/ +ENV HOME /root +ENV USER /root +ENV PATH /usr/local/bin:/kaniko +ENV SSL_CERT_DIR=/kaniko/ssl/certs +ENV DOCKER_CONFIG /kaniko/.docker/ +ENV DOCKER_CREDENTIAL_GCR_CONFIG /kaniko/.config/gcloud/docker_credential_gcr_config.json +WORKDIR /workspace +RUN ["docker-credential-gcr", "config", "--token-source=env"] +ENTRYPOINT ["/kaniko/warmer"] diff --git a/kaniko-warmer.yaml b/kaniko-warmer.yaml new file mode 100644 index 0000000000..e37d6e1cad --- /dev/null +++ b/kaniko-warmer.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kaniko-warmer +spec: + containers: + - name: kaniko-warmer + image: gcr.io/sharif-test/warmer@sha256:9bf8affcf6630a9b17298e54b79247a22516aebff0c88733f04e6e2974ca58c4 + args: ["-c /cache", + "-i gcr.io/google-appengine/debian9", + "-vdebug"] + volumeMounts: + - name: kaniko-secret + mountPath: /secret + - name: kaniko-cache + mountPath: /cache + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /secret/kaniko-secret.json + restartPolicy: Never + volumes: + - name: kaniko-secret + secret: + secretName: default-token-wtlnt + - name: kaniko-cache + persistentVolumeClaim: + claimName: kaniko-cache-claim + From 123d5f30d0519fc36cbddba030a2d2962a70eb9c Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Tue, 9 Oct 2018 13:22:42 -0700 Subject: [PATCH 07/15] Fixing up cache retrieval and cleanup --- cmd/executor/cmd/root.go | 1 + kaniko-test.yaml | 4 +++- kaniko-warmer.yaml | 9 ++++----- pkg/cache/cache.go | 16 ++++------------ pkg/cache/warm.go | 6 ------ pkg/util/image_util.go | 22 ++++++++++++++++++---- pkg/util/image_util_test.go | 6 +++--- 7 files changed, 33 insertions(+), 31 deletions(-) diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index bdd4e52c68..55d256f289 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -99,6 +99,7 @@ func addKanikoOptionsFlags(cmd *cobra.Command) { RootCmd.PersistentFlags().StringVarP(&opts.Target, "target", "", "", "Set the target build stage to build") RootCmd.PersistentFlags().BoolVarP(&opts.NoPush, "no-push", "", false, "Do not push the image to the registry") RootCmd.PersistentFlags().StringVarP(&opts.CacheRepo, "cache-repo", "", "", "Specify a repository to use as a cache, otherwise one will be inferred from the destination provided") + RootCmd.PersistentFlags().StringVarP(&opts.CacheDir, "cache-dir", "", "/cache", "Specify a local directory to use as a cache.") RootCmd.PersistentFlags().BoolVarP(&opts.Cache, "cache", "", false, "Use cache when building image") RootCmd.PersistentFlags().BoolVarP(&opts.Cleanup, "cleanup", "", false, "Clean the filesystem at the end") } diff --git a/kaniko-test.yaml b/kaniko-test.yaml index f18037d574..d6f9904333 100644 --- a/kaniko-test.yaml +++ b/kaniko-test.yaml @@ -8,7 +8,9 @@ spec: image: gcr.io/kaniko-project/executor:latest args: ["--dockerfile=", "--context=", - "--destination="] + "--destination=", + "--cache", + "--cache-dir=/cache"] volumeMounts: - name: kaniko-secret mountPath: /secret diff --git a/kaniko-warmer.yaml b/kaniko-warmer.yaml index e37d6e1cad..318f628782 100644 --- a/kaniko-warmer.yaml +++ b/kaniko-warmer.yaml @@ -5,10 +5,9 @@ metadata: spec: containers: - name: kaniko-warmer - image: gcr.io/sharif-test/warmer@sha256:9bf8affcf6630a9b17298e54b79247a22516aebff0c88733f04e6e2974ca58c4 - args: ["-c /cache", - "-i gcr.io/google-appengine/debian9", - "-vdebug"] + image: gcr.io/kaniko-project/warmer:latest + args: ["--cache-dir=/cache", + "--image=gcr.io/google-appengine/debian9"] volumeMounts: - name: kaniko-secret mountPath: /secret @@ -21,7 +20,7 @@ spec: volumes: - name: kaniko-secret secret: - secretName: default-token-wtlnt + secretName: kaniko-secret - name: kaniko-cache persistentVolumeClaim: claimName: kaniko-cache-claim diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index c3e2bc89ba..679a881a67 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -33,10 +33,6 @@ import ( // RetrieveLayer checks the specified cache for a layer with the tag :cacheKey func RetrieveLayer(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) { - if local, _ := LocalDestination(opts, cacheKey); local != nil { - return local, nil - } - cache, err := Destination(opts, cacheKey) if err != nil { return nil, errors.Wrap(err, "getting cache destination") @@ -75,21 +71,17 @@ func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) { return fmt.Sprintf("%s:%s", cache, cacheKey), nil } -func LocalDestination(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) { +func LocalSource(opts *config.KanikoOptions, cacheKey string, source string) (v1.Image, error) { cache := opts.CacheDir if cache == "" { return nil, nil } path := path.Join(cache, cacheKey) - destination := opts.Destinations[0] - destRef, err := name.NewTag(destination, name.WeakValidation) - if err != nil { - return nil, errors.Wrap(err, "volume cache: getting tag for destination") - } - tag, err := name.NewTag(fmt.Sprintf("%s/cache:%s", destRef.Context(), cacheKey), name.WeakValidation) + + tag, err := name.NewTag(source, name.WeakValidation) if err != nil { - return nil, errors.Wrap(err, "volume cache: create new cache tag") + return nil, errors.Wrap(err, "volume cache: creating new cache tag") } imgTar, err := tarball.ImageFromPath(path, &tag) if err != nil { diff --git a/pkg/cache/warm.go b/pkg/cache/warm.go index a228b0972e..d028b16c77 100644 --- a/pkg/cache/warm.go +++ b/pkg/cache/warm.go @@ -38,30 +38,24 @@ func WarmCache(opts *config.WarmerOptions) error { for _, image := range images { cacheRef, err := name.NewTag(image, name.WeakValidation) if err != nil { - logrus.Debugf("Failed to verify image name %v", err) errors.Wrap(err, fmt.Sprintf("Failed to verify image name: %s", image)) } k8sc, err := k8schain.NewNoClient() if err != nil { - logrus.Debugf("Failed to create auth keychain: %v", err) errors.Wrap(err, fmt.Sprintf("Failed to create auth keychain: %s", image)) } img, err := remote.Image(cacheRef, remote.WithAuthFromKeychain(k8sc)) if err != nil { - logrus.Debugf("Failed to retrieve image: %v", err) errors.Wrap(err, fmt.Sprintf("Failed to retrieve image: %s", image)) } digest, err := img.Digest() if err != nil { - logrus.Debugf("Failed to retrieve digest: %v", err) errors.Wrap(err, fmt.Sprintf("Failed to retrieve digest: %s", image)) } cachePath := path.Join(cacheDir, digest.String()) - logrus.Debugf("CACHE PATH = %s\n", cachePath) err = tarball.WriteToFile(cachePath, cacheRef, img) if err != nil { - logrus.Debugf("Failed to write to cache %v", err) errors.Wrap(err, fmt.Sprintf("Failed to write %s to cache", image)) } else { logrus.Debugf("Wrote %s to cache", image) diff --git a/pkg/util/image_util.go b/pkg/util/image_util.go index 95ae672f9c..25de2e04cf 100644 --- a/pkg/util/image_util.go +++ b/pkg/util/image_util.go @@ -17,7 +17,6 @@ limitations under the License. package util import ( - "fmt" "path/filepath" "strconv" @@ -106,7 +105,22 @@ func cachedImage(opts *config.KanikoOptions, image string) (v1.Image, error) { if err != nil { return nil, err } - cacheKey := ref.Name() - fmt.Printf("CACHEKEY=%s", cacheKey) - return cache.LocalDestination(opts, cacheKey) + + k8sc, err := k8schain.NewNoClient() + if err != nil { + return nil, err + } + kc := authn.NewMultiKeychain(authn.DefaultKeychain, k8sc) + + img, err := remote.Image(ref, remote.WithAuthFromKeychain(kc)) + if err != nil { + return nil, err + } + + cacheKey, err := img.Digest() + if err != nil { + return nil, err + } + + return cache.LocalSource(opts, cacheKey.String(), image) } diff --git a/pkg/util/image_util_test.go b/pkg/util/image_util_test.go index dbd7b8ee1b..8d9377e250 100644 --- a/pkg/util/image_util_test.go +++ b/pkg/util/image_util_test.go @@ -57,7 +57,7 @@ func Test_StandardImage(t *testing.T) { retrieveRemoteImage = mock actual, err := RetrieveSourceImage(config.KanikoStage{ Stage: stages[0], - }, nil) + }, nil, nil) testutil.CheckErrorAndDeepEqual(t, false, err, nil, actual) } func Test_ScratchImage(t *testing.T) { @@ -67,7 +67,7 @@ func Test_ScratchImage(t *testing.T) { } actual, err := RetrieveSourceImage(config.KanikoStage{ Stage: stages[1], - }, nil) + }, nil, nil) expected := empty.Image testutil.CheckErrorAndDeepEqual(t, false, err, expected, actual) } @@ -89,7 +89,7 @@ func Test_TarImage(t *testing.T) { BaseImageStoredLocally: true, BaseImageIndex: 0, Stage: stages[2], - }, nil) + }, nil, nil) testutil.CheckErrorAndDeepEqual(t, false, err, nil, actual) } From 72e71ce6c8883fbe05840acaa61465c61aa24837 Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Tue, 9 Oct 2018 14:03:51 -0700 Subject: [PATCH 08/15] fix tests --- pkg/util/image_util_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/util/image_util_test.go b/pkg/util/image_util_test.go index 8d9377e250..51d4f62db2 100644 --- a/pkg/util/image_util_test.go +++ b/pkg/util/image_util_test.go @@ -57,7 +57,7 @@ func Test_StandardImage(t *testing.T) { retrieveRemoteImage = mock actual, err := RetrieveSourceImage(config.KanikoStage{ Stage: stages[0], - }, nil, nil) + }, nil, &config.KanikoOptions{}) testutil.CheckErrorAndDeepEqual(t, false, err, nil, actual) } func Test_ScratchImage(t *testing.T) { @@ -89,7 +89,7 @@ func Test_TarImage(t *testing.T) { BaseImageStoredLocally: true, BaseImageIndex: 0, Stage: stages[2], - }, nil, nil) + }, nil, &config.KanikoOptions{}) testutil.CheckErrorAndDeepEqual(t, false, err, nil, actual) } From 78c2111749df90d1d578670e3ae6ba0067f3ba81 Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Tue, 9 Oct 2018 15:18:48 -0700 Subject: [PATCH 09/15] removing auth since we only cache public images --- pkg/cache/warm.go | 7 +------ pkg/util/image_util.go | 8 +------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/pkg/cache/warm.go b/pkg/cache/warm.go index d028b16c77..fc7b6b6c87 100644 --- a/pkg/cache/warm.go +++ b/pkg/cache/warm.go @@ -21,7 +21,6 @@ import ( "path" "github.com/GoogleContainerTools/kaniko/pkg/config" - "github.com/google/go-containerregistry/pkg/authn/k8schain" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" @@ -40,11 +39,7 @@ func WarmCache(opts *config.WarmerOptions) error { if err != nil { errors.Wrap(err, fmt.Sprintf("Failed to verify image name: %s", image)) } - k8sc, err := k8schain.NewNoClient() - if err != nil { - errors.Wrap(err, fmt.Sprintf("Failed to create auth keychain: %s", image)) - } - img, err := remote.Image(cacheRef, remote.WithAuthFromKeychain(k8sc)) + img, err := remote.Image(cacheRef) if err != nil { errors.Wrap(err, fmt.Sprintf("Failed to retrieve image: %s", image)) } diff --git a/pkg/util/image_util.go b/pkg/util/image_util.go index 25de2e04cf..dd081e121e 100644 --- a/pkg/util/image_util.go +++ b/pkg/util/image_util.go @@ -106,13 +106,7 @@ func cachedImage(opts *config.KanikoOptions, image string) (v1.Image, error) { return nil, err } - k8sc, err := k8schain.NewNoClient() - if err != nil { - return nil, err - } - kc := authn.NewMultiKeychain(authn.DefaultKeychain, k8sc) - - img, err := remote.Image(ref, remote.WithAuthFromKeychain(kc)) + img, err := remote.Image(ref) if err != nil { return nil, err } From b8ac1bee262d99f1b8e6d5126af7dc796fc41ace Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Tue, 9 Oct 2018 18:16:13 -0700 Subject: [PATCH 10/15] simplifying the caching logic --- pkg/cache/cache.go | 10 +++------- pkg/util/image_util.go | 25 ++++++++++++++++--------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 679a881a67..fceaa51651 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -71,7 +71,7 @@ func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) { return fmt.Sprintf("%s:%s", cache, cacheKey), nil } -func LocalSource(opts *config.KanikoOptions, cacheKey string, source string) (v1.Image, error) { +func LocalSource(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) { cache := opts.CacheDir if cache == "" { return nil, nil @@ -79,15 +79,11 @@ func LocalSource(opts *config.KanikoOptions, cacheKey string, source string) (v1 path := path.Join(cache, cacheKey) - tag, err := name.NewTag(source, name.WeakValidation) - if err != nil { - return nil, errors.Wrap(err, "volume cache: creating new cache tag") - } - imgTar, err := tarball.ImageFromPath(path, &tag) + imgTar, err := tarball.ImageFromPath(path, nil) if err != nil { return nil, errors.Wrap(err, "getting layer from tarball") } - fmt.Printf("Found %s in local cache", tag.Name()) + fmt.Printf("Found %s in local cache", cacheKey) return imgTar, nil } diff --git a/pkg/util/image_util.go b/pkg/util/image_util.go index dd081e121e..e101083ed7 100644 --- a/pkg/util/image_util.go +++ b/pkg/util/image_util.go @@ -106,15 +106,22 @@ func cachedImage(opts *config.KanikoOptions, image string) (v1.Image, error) { return nil, err } - img, err := remote.Image(ref) - if err != nil { - return nil, err - } - - cacheKey, err := img.Digest() - if err != nil { - return nil, err + var cacheKey string + if d, ok := ref.(name.Digest); ok { + cacheKey = d.DigestStr() + } else { + img, err := remote.Image(ref) + if err != nil { + return nil, err + } + + d, err := img.Digest() + if err != nil { + return nil, err + } + + cacheKey = d.String() } - return cache.LocalSource(opts, cacheKey.String(), image) + return cache.LocalSource(opts, cacheKey) } From 27888f3fb8f68a9edf7eaeeab4fd028088d71f65 Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Tue, 9 Oct 2018 18:20:53 -0700 Subject: [PATCH 11/15] fixing logic --- pkg/cache/cache.go | 2 +- pkg/util/image_util.go | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index fceaa51651..7960567878 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -81,7 +81,7 @@ func LocalSource(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) imgTar, err := tarball.ImageFromPath(path, nil) if err != nil { - return nil, errors.Wrap(err, "getting layer from tarball") + return nil, errors.Wrap(err, "getting image from path") } fmt.Printf("Found %s in local cache", cacheKey) diff --git a/pkg/util/image_util.go b/pkg/util/image_util.go index e101083ed7..6eb41aa574 100644 --- a/pkg/util/image_util.go +++ b/pkg/util/image_util.go @@ -61,7 +61,14 @@ func RetrieveSourceImage(stage config.KanikoStage, buildArgs []string, opts *con // Next, check if local caching is enabled // If so, look in the local cache before trying the remote registry if opts.Cache && opts.CacheDir != "" { - return cachedImage(opts, currentBaseName) + cachedImage, err := cachedImage(opts, currentBaseName) + if cachedImage != nil { + return cachedImage, nil + } + + if err != nil { + logrus.Warnf("Error while retrieving image from cache: %v", err) + } } // Otherwise, initialize image as usual From d158f70ee293a4909f013d76c2cb19aa2c5770a0 Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Wed, 10 Oct 2018 19:08:29 -0700 Subject: [PATCH 12/15] adding volume cache to integration tests. remove auth from cache warmer image. --- deploy/Dockerfile_warmer | 10 ---------- integration/images.go | 26 +++++++++++++++++++++++++- integration/integration_test.go | 8 ++++++++ 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/deploy/Dockerfile_warmer b/deploy/Dockerfile_warmer index c74c3d3ed0..2cee0968c5 100644 --- a/deploy/Dockerfile_warmer +++ b/deploy/Dockerfile_warmer @@ -18,17 +18,9 @@ FROM golang:1.10 WORKDIR /go/src/github.com/GoogleContainerTools/kaniko COPY . . RUN make -# Get GCR credential helper -ADD https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v1.4.3-static/docker-credential-gcr_linux_amd64-1.4.3.tar.gz /usr/local/bin/ -RUN tar -C /usr/local/bin/ -xvzf /usr/local/bin/docker-credential-gcr_linux_amd64-1.4.3.tar.gz -# Get Amazon ECR credential helper -RUN go get -u github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login -RUN make -C /go/src/github.com/awslabs/amazon-ecr-credential-helper linux-amd64 FROM scratch COPY --from=0 /go/src/github.com/GoogleContainerTools/kaniko/out/warmer /kaniko/warmer -COPY --from=0 /usr/local/bin/docker-credential-gcr /kaniko/docker-credential-gcr -COPY --from=0 /go/src/github.com/awslabs/amazon-ecr-credential-helper/bin/linux-amd64/docker-credential-ecr-login /kaniko/docker-credential-ecr-login COPY files/ca-certificates.crt /kaniko/ssl/certs/ COPY files/config.json /kaniko/.docker/ ENV HOME /root @@ -36,7 +28,5 @@ ENV USER /root ENV PATH /usr/local/bin:/kaniko ENV SSL_CERT_DIR=/kaniko/ssl/certs ENV DOCKER_CONFIG /kaniko/.docker/ -ENV DOCKER_CREDENTIAL_GCR_CONFIG /kaniko/.config/gcloud/docker_credential_gcr_config.json WORKDIR /workspace -RUN ["docker-credential-gcr", "config", "--token-source=env"] ENTRYPOINT ["/kaniko/warmer"] diff --git a/integration/images.go b/integration/images.go index cf1f901058..464db8cc88 100644 --- a/integration/images.go +++ b/integration/images.go @@ -30,10 +30,13 @@ import ( const ( // ExecutorImage is the name of the kaniko executor image ExecutorImage = "executor-image" + WarmerImage = "warmer-image" dockerPrefix = "docker-" kanikoPrefix = "kaniko-" buildContextPath = "/workspace" + cacheDir = "/workspace/cache" + baseImageToCache = "gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0" ) // Arguments to build Dockerfiles with, used for both docker and kaniko builds @@ -201,6 +204,26 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do return nil } +func populateVolumeCache() error { + _, ex, _, _ := runtime.Caller(0) + cwd := filepath.Dir(ex) + warmerCmd := exec.Command("docker", + append([]string{"run", + "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud", + "-v", cwd + ":/workspace", + WarmerImage, + "-c", cacheDir, + "-i", baseImageToCache}, + )..., + ) + + if _, err := RunCommandWithoutTest(warmerCmd); err != nil { + return fmt.Errorf("Failed to warm kaniko cache: %s", err) + } + + return nil +} + // buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesPath string, version int) error { _, ex, _, _ := runtime.Caller(0) @@ -219,7 +242,8 @@ func (d *DockerFileBuilder) buildCachedImages(imageRepo, cacheRepo, dockerfilesP "-d", kanikoImage, "-c", buildContextPath, cacheFlag, - "--cache-repo", cacheRepo})..., + "--cache-repo", cacheRepo, + "--cache-dir", cacheDir})..., ) if _, err := RunCommandWithoutTest(kanikoCmd); err != nil { diff --git a/integration/integration_test.go b/integration/integration_test.go index e6e4f89596..313d8f0536 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -145,6 +145,13 @@ func TestMain(m *testing.M) { os.Exit(1) } + fmt.Println("Building cache warmer image") + cmd = exec.Command("docker", "build", "-t", WarmerImage, "-f", "../deploy/Dockerfile_warmer", "..") + if _, err = RunCommandWithoutTest(cmd); err != nil { + fmt.Printf("Building kaniko's cache warmer failed: %s", err) + os.Exit(1) + } + fmt.Println("Building onbuild base image") buildOnbuildBase := exec.Command("docker", "build", "-t", config.onbuildBaseImage, "-f", "dockerfiles/Dockerfile_onbuild_base", ".") if err := buildOnbuildBase.Run(); err != nil { @@ -238,6 +245,7 @@ func TestLayers(t *testing.T) { // Build each image with kaniko twice, and then make sure they're exactly the same func TestCache(t *testing.T) { + populateVolumeCache() for dockerfile := range imageBuilder.TestCacheDockerfiles { t.Run("test_cache_"+dockerfile, func(t *testing.T) { cache := filepath.Join(config.imageRepo, "cache", fmt.Sprintf("%v", time.Now().UnixNano())) From 0a550f0300efb20bcdeb3fa1da9f16ed6a2a3149 Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Wed, 10 Oct 2018 19:18:21 -0700 Subject: [PATCH 13/15] add building warmer to integration-test --- integration-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-test.sh b/integration-test.sh index 51fcb6e20b..3217f767ad 100755 --- a/integration-test.sh +++ b/integration-test.sh @@ -34,5 +34,6 @@ fi echo "Running integration tests..." make out/executor +make out/warmer pushd integration go test -v --bucket "${GCS_BUCKET}" --repo "${IMAGE_REPO}" --timeout 30m From 47c8ff4895b606a5484a4489c632d20daa7bc936 Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Thu, 11 Oct 2018 11:47:14 -0700 Subject: [PATCH 14/15] move sample yaml files to examples dir --- kaniko-cache-claim.yaml => examples/kaniko-cache-claim.yaml | 0 kaniko-cache-volume.yaml => examples/kaniko-cache-volume.yaml | 0 kaniko-test.yaml => examples/kaniko-test.yaml | 0 kaniko-warmer.yaml => examples/kaniko-warmer.yaml | 0 pkg/cache/cache.go | 2 +- 5 files changed, 1 insertion(+), 1 deletion(-) rename kaniko-cache-claim.yaml => examples/kaniko-cache-claim.yaml (100%) rename kaniko-cache-volume.yaml => examples/kaniko-cache-volume.yaml (100%) rename kaniko-test.yaml => examples/kaniko-test.yaml (100%) rename kaniko-warmer.yaml => examples/kaniko-warmer.yaml (100%) diff --git a/kaniko-cache-claim.yaml b/examples/kaniko-cache-claim.yaml similarity index 100% rename from kaniko-cache-claim.yaml rename to examples/kaniko-cache-claim.yaml diff --git a/kaniko-cache-volume.yaml b/examples/kaniko-cache-volume.yaml similarity index 100% rename from kaniko-cache-volume.yaml rename to examples/kaniko-cache-volume.yaml diff --git a/kaniko-test.yaml b/examples/kaniko-test.yaml similarity index 100% rename from kaniko-test.yaml rename to examples/kaniko-test.yaml diff --git a/kaniko-warmer.yaml b/examples/kaniko-warmer.yaml similarity index 100% rename from kaniko-warmer.yaml rename to examples/kaniko-warmer.yaml diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 7960567878..fca66e631b 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -84,6 +84,6 @@ func LocalSource(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) return nil, errors.Wrap(err, "getting image from path") } - fmt.Printf("Found %s in local cache", cacheKey) + logrus.Infof("Found %s in local cache", cacheKey) return imgTar, nil } From 78d48985dfb6f4161b2ebcb9898dc0b0ff32303b Mon Sep 17 00:00:00 2001 From: sharifelgamal Date: Thu, 11 Oct 2018 11:58:23 -0700 Subject: [PATCH 15/15] small test fix --- pkg/util/image_util_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/image_util_test.go b/pkg/util/image_util_test.go index 51d4f62db2..44897fe728 100644 --- a/pkg/util/image_util_test.go +++ b/pkg/util/image_util_test.go @@ -67,7 +67,7 @@ func Test_ScratchImage(t *testing.T) { } actual, err := RetrieveSourceImage(config.KanikoStage{ Stage: stages[1], - }, nil, nil) + }, nil, &config.KanikoOptions{}) expected := empty.Image testutil.CheckErrorAndDeepEqual(t, false, err, expected, actual) }