diff --git a/Makefile b/Makefile index 71f547e32d8a..3933f3ac3464 100755 --- a/Makefile +++ b/Makefile @@ -19,8 +19,10 @@ VERSION_BUILD ?= 3 RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).${VERSION_BUILD} VERSION ?= v$(RAW_VERSION) -KUBERNETES_VERSION ?= $(shell egrep "^var DefaultKubernetesVersion" pkg/minikube/constants/constants.go | cut -d \" -f2) +KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/constants/constants.go | cut -d \" -f2) KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2) +PRELOADED_TARBALL_VERSION ?= $(shell egrep "Version =" pkg/minikube/preload/constants.go | cut -d \" -f2) +PRELOADED_VOLUMES_GCS_BUCKET ?= $(shell egrep "PreloadedVolumeTarballsBucket =" pkg/minikube/constants/constants.go | cut -d \" -f2) # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions ISO_VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).3 @@ -521,15 +523,14 @@ kic-base-image: ## builds the base image used for kic. docker rmi -f $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot || true docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --target base . - -.PHONY: kic-preloaded-base-image -kic-preloaded-base-image: generate-preloaded-images-tar ## builds the base image used for kic. - docker rmi -f $(REGISTRY)/kicbase:$(KIC_VERSION)-k8s-${KUBERNETES_VERSION} || true - docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-k8s-${KUBERNETES_VERSION} --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --build-arg KUBERNETES_VERSION=${KUBERNETES_VERSION} . +.PHONY: upload-preloaded-images-tar +upload-preloaded-images-tar: generate-preloaded-images-tar # Upload the preloaded images tar to the GCS bucket. Specify a specific kubernetes version to build via `KUBERNETES_VERSION=vx.y.z make upload-preloaded-images-tar`. + gsutil cp out/preloaded-images-k8s-${PRELOADED_TARBALL_VERSION}-${KUBERNETES_VERSION}-docker-overlay2.tar.lz4 gs://${PRELOADED_VOLUMES_GCS_BUCKET} + gsutil acl ch -u AllUsers:R gs://${PRELOADED_VOLUMES_GCS_BUCKET}/preloaded-images-k8s-${PRELOADED_TARBALL_VERSION}-${KUBERNETES_VERSION}-docker-overlay2.tar.lz4 .PHONY: generate-preloaded-images-tar -generate-preloaded-images-tar: out/minikube - go run ./hack/preload-images/preload_images.go -kubernetes-version ${KUBERNETES_VERSION} +generate-preloaded-images-tar: + go run ./hack/preload-images/preload_images.go -kubernetes-version ${KUBERNETES_VERSION} -preloaded-tarball-version ${PRELOADED_TARBALL_VERSION} .PHONY: push-storage-provisioner-image diff --git a/go.mod b/go.mod index 91462732ffb2..9b876466ee79 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module k8s.io/minikube go 1.13 require ( + cloud.google.com/go v0.45.1 github.com/Parallels/docker-machine-parallels v1.3.0 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/blang/semver v3.5.0+incompatible @@ -69,6 +70,7 @@ require ( golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/sys v0.0.0-20191010194322-b09406accb47 golang.org/x/text v0.3.2 + google.golang.org/api v0.9.0 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect k8s.io/api v0.17.3 k8s.io/apimachinery v0.17.3 diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile index 631f67b5b605..baadf88df3ac 100644 --- a/hack/images/kicbase.Dockerfile +++ b/hack/images/kicbase.Dockerfile @@ -52,13 +52,3 @@ RUN apt-get clean -y && rm -rf \ /usr/share/man/* \ /usr/share/local/* \ RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt" - - -FROM busybox -ARG KUBERNETES_VERSION -COPY out/preloaded-images-k8s-$KUBERNETES_VERSION.tar /preloaded-images.tar -RUN tar xvf /preloaded-images.tar -C / - -FROM base -COPY --from=1 /var/lib/docker /var/lib/docker -COPY --from=1 /var/lib/minikube/binaries /var/lib/minikube/binaries diff --git a/hack/preload-images/preload_images.go b/hack/preload-images/preload_images.go index 82fdba5b4162..92d4d86f04ac 100644 --- a/hack/preload-images/preload_images.go +++ b/hack/preload-images/preload_images.go @@ -25,6 +25,11 @@ import ( "strings" "github.com/pkg/errors" + "k8s.io/minikube/pkg/drivers/kic" + "k8s.io/minikube/pkg/drivers/kic/oci" + "k8s.io/minikube/pkg/minikube/bootstrapper/images" + "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/localpath" ) const ( @@ -33,17 +38,28 @@ const ( ) var ( - kubernetesVersion = "" - tarballFilename = "" + kubernetesVersion = "" + tarballFilename = "" + dockerStorageDriver = "" + preloadedTarballVersion = "" + containerRuntime = "" ) func init() { flag.StringVar(&kubernetesVersion, "kubernetes-version", "", "desired kubernetes version, for example `v1.17.2`") + flag.StringVar(&dockerStorageDriver, "docker-storage-driver", "overlay2", "docker storage driver backend") + flag.StringVar(&preloadedTarballVersion, "preloaded-tarball-version", "", "preloaded tarball version") + flag.StringVar(&containerRuntime, "container-runtime", "docker", "container runtime") + flag.Parse() - tarballFilename = fmt.Sprintf("preloaded-images-k8s-%s.tar", kubernetesVersion) + tarballFilename = fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-%s.tar.lz4", preloadedTarballVersion, kubernetesVersion, containerRuntime, dockerStorageDriver) } func main() { + if err := verifyDockerStorage(); err != nil { + fmt.Println(err) + os.Exit(1) + } if err := executePreloadImages(); err != nil { fmt.Println(err) os.Exit(1) @@ -56,42 +72,74 @@ func executePreloadImages() error { fmt.Println(err) } }() - if err := startMinikube(); err != nil { + + driver := kic.NewDriver(kic.Config{ + KubernetesVersion: kubernetesVersion, + ContainerRuntime: driver.Docker, + OCIBinary: oci.Docker, + MachineName: profile, + ImageDigest: kic.BaseImage, + StorePath: localpath.MiniPath(), + CPU: 2, + Memory: 4000, + APIServerPort: 8080, + }) + + baseDir := filepath.Dir(driver.GetSSHKeyPath()) + defer os.Remove(baseDir) + + if err := os.MkdirAll(baseDir, 0755); err != nil { return err } + if err := driver.Create(); err != nil { + return errors.Wrap(err, "creating kic driver") + } + + // Now, get images to pull + imgs, err := images.Kubeadm("", kubernetesVersion) + if err != nil { + return errors.Wrap(err, "kubeadm images") + } + + for _, img := range append(imgs, kic.OverlayImage) { + cmd := exec.Command("docker", "exec", profile, "docker", "pull", img) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return errors.Wrapf(err, "downloading %s", img) + } + } + + // Create image tarball if err := createImageTarball(); err != nil { return err } return copyTarballToHost() } -func startMinikube() error { - cmd := exec.Command(minikubePath, "start", "-p", profile, "--memory", "4000", "--kubernetes-version", kubernetesVersion, "--wait=false") - cmd.Stdout = os.Stdout - return cmd.Run() -} - func createImageTarball() error { - cmd := exec.Command(minikubePath, "ssh", "-p", profile, "--", "sudo", "tar", "cvf", tarballFilename, "/var/lib/docker", "/var/lib/minikube/binaries") + dirs := []string{ + fmt.Sprintf("./lib/docker/%s", dockerStorageDriver), + "./lib/docker/image", + } + args := []string{"exec", profile, "sudo", "tar", "-I", "lz4", "-C", "/var", "-cvf", tarballFilename} + args = append(args, dirs...) + cmd := exec.Command("docker", args...) cmd.Stdout = os.Stdout - return cmd.Run() + if err := cmd.Run(); err != nil { + return errors.Wrap(err, "creating image tarball") + } + return nil } func copyTarballToHost() error { - sshKey, err := runCmd([]string{minikubePath, "ssh-key", "-p", profile}) - if err != nil { - return errors.Wrap(err, "getting ssh-key") - } - - ip, err := runCmd([]string{minikubePath, "ip", "-p", profile}) - if err != nil { - return errors.Wrap(err, "getting ip") - } - dest := filepath.Join("out/", tarballFilename) - args := []string{"scp", "-o", "StrictHostKeyChecking=no", "-i", sshKey, fmt.Sprintf("docker@%s:/home/docker/%s", ip, tarballFilename), dest} - _, err = runCmd(args) - return err + cmd := exec.Command("docker", "cp", fmt.Sprintf("%s:/%s", profile, tarballFilename), dest) + cmd.Stdout = os.Stdout + if err := cmd.Run(); err != nil { + return errors.Wrap(err, "copying tarball to host") + } + return nil } func deleteMinikube() error { @@ -100,8 +148,15 @@ func deleteMinikube() error { return cmd.Run() } -func runCmd(command []string) (string, error) { - cmd := exec.Command(command[0], command[1:]...) +func verifyDockerStorage() error { + cmd := exec.Command("docker", "info", "-f", "{{.Info.Driver}}") output, err := cmd.Output() - return strings.Trim(string(output), "\n "), err + if err != nil { + return err + } + driver := strings.Trim(string(output), " \n") + if driver != dockerStorageDriver { + return fmt.Errorf("docker storage driver %s does not match requested %s", driver, dockerStorageDriver) + } + return nil } diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index a5abcd372d86..91958457119f 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -36,6 +36,7 @@ import ( "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/preload" ) // Driver represents a kic driver https://minikube.sigs.k8s.io/docs/reference/drivers/docker @@ -90,14 +91,23 @@ func (d *Driver) Create() error { ContainerPort: constants.DockerDaemonPort, }, ) - err := oci.CreateContainerNode(params) - if err != nil { + if err := oci.CreateContainerNode(params); err != nil { return errors.Wrap(err, "create kic node") } if err := d.prepareSSH(); err != nil { return errors.Wrap(err, "prepare kic ssh") } + + t := time.Now() + glog.Infof("Starting extracting preloaded images to volume") + // Extract preloaded images to container + if err := oci.ExtractTarballToVolume(preload.TarballFilepath(d.NodeConfig.KubernetesVersion), params.Name, BaseImage); err != nil { + glog.Infof("Unable to extract preloaded tarball to volume: %v", err) + } else { + glog.Infof("Took %f seconds to extract preloaded images to volume", time.Since(t).Seconds()) + } + return nil } diff --git a/pkg/drivers/kic/oci/volumes.go b/pkg/drivers/kic/oci/volumes.go index d24bd7bb96e0..ecba94cf15ba 100644 --- a/pkg/drivers/kic/oci/volumes.go +++ b/pkg/drivers/kic/oci/volumes.go @@ -103,6 +103,19 @@ func allVolumesByLabel(ociBin string, label string) ([]string, error) { return vols, err } +// ExtractTarballToVolume runs a docker image imageName which extracts the tarball at tarballPath +// to the volume named volumeName +func ExtractTarballToVolume(tarballPath, volumeName, imageName string) error { + if err := PointToHostDockerDaemon(); err != nil { + return errors.Wrap(err, "point host docker-daemon") + } + cmd := exec.Command(Docker, "run", "--rm", "--entrypoint", "/usr/bin/tar", "-v", fmt.Sprintf("%s:/preloaded.tar:ro", tarballPath), "-v", fmt.Sprintf("%s:/extractDir", volumeName), imageName, "-I", "lz4", "-xvf", "/preloaded.tar", "-C", "/extractDir") + if out, err := cmd.CombinedOutput(); err != nil { + return errors.Wrapf(err, "output %s", string(out)) + } + return nil +} + // createDockerVolume creates a docker volume to be attached to the container with correct labels and prefixes based on profile name // Caution ! if volume already exists does NOT return an error and will not apply the minikube labels on it. // TODO: this should be fixed as a part of https://github.com/kubernetes/minikube/issues/6530 diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go index e5b7983a84a6..32e29503961f 100644 --- a/pkg/drivers/kic/types.go +++ b/pkg/drivers/kic/types.go @@ -46,14 +46,16 @@ var ( // Config is configuration for the kic driver used by registry type Config struct { - MachineName string // maps to the container name being created - CPU int // Number of CPU cores assigned to the container - Memory int // max memory in MB - StorePath string // libmachine store path - OCIBinary string // oci tool to use (docker, podman,...) - ImageDigest string // image name with sha to use for the node - Mounts []oci.Mount // mounts - APIServerPort int // kubernetes api server port inside the container - PortMappings []oci.PortMapping // container port mappings - Envs map[string]string // key,value of environment variables passed to the node + MachineName string // maps to the container name being created + CPU int // Number of CPU cores assigned to the container + Memory int // max memory in MB + StorePath string // libmachine store path + OCIBinary string // oci tool to use (docker, podman,...) + ImageDigest string // image name with sha to use for the node + Mounts []oci.Mount // mounts + APIServerPort int // kubernetes api server port inside the container + PortMappings []oci.PortMapping // container port mappings + Envs map[string]string // key,value of environment variables passed to the node + KubernetesVersion string // kubernetes version to install + ContainerRuntime string // container runtime kic is running } diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 96495d083ef8..f173d9861898 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -63,6 +63,9 @@ const ( MinikubeActiveDockerdEnv = "MINIKUBE_ACTIVE_DOCKERD" // PodmanVarlinkBridgeEnv is used for podman settings PodmanVarlinkBridgeEnv = "PODMAN_VARLINK_BRIDGE" + + // PreloadedVolumeTarballsBucket is the name of the GCS bucket where preloaded volume tarballs exist + PreloadedVolumeTarballsBucket = "minikube-preloaded-volume-tarballs" ) var ( diff --git a/pkg/minikube/image/image.go b/pkg/minikube/image/image.go index d7face059f3b..a14ad85f71a1 100644 --- a/pkg/minikube/image/image.go +++ b/pkg/minikube/image/image.go @@ -20,8 +20,10 @@ import ( "context" "io/ioutil" "os" + "os/exec" "path/filepath" "runtime" + "strings" "time" "github.com/docker/docker/client" @@ -31,6 +33,8 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/daemon" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/minikube/constants" ) @@ -76,6 +80,37 @@ func DigestByGoLib(imgName string) string { return cf.Hex } +// WriteImageToDaemon write img to the local docker daemon +func WriteImageToDaemon(img string) error { + glog.Infof("Writing %s to local daemon", img) + if err := oci.PointToHostDockerDaemon(); err != nil { + return errors.Wrap(err, "point host docker-daemon") + } + // Check if image exists locally + cmd := exec.Command("docker", "images", "--format", "{{.Repository}}:{{.Tag}}@{{.Digest}}") + if output, err := cmd.Output(); err == nil { + if strings.Contains(string(output), img) { + glog.Infof("Found %s in local docker daemon, skipping pull", img) + return nil + } + } + // Else, pull it + ref, err := name.ParseReference(img) + if err != nil { + return errors.Wrap(err, "parsing reference") + } + i, err := remote.Image(ref) + if err != nil { + return errors.Wrap(err, "getting remote image") + } + tag, err := name.NewTag(strings.Split(img, "@")[0]) + if err != nil { + return errors.Wrap(err, "getting tag") + } + _, err = daemon.Write(tag, i) + return err +} + func retrieveImage(ref name.Reference) (v1.Image, error) { glog.Infof("retrieving image: %+v", ref) img, err := daemon.Image(ref) diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index 301c3b02fdbe..b35ab7969266 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -19,8 +19,10 @@ package machine import ( "fmt" "os" + "os/exec" "path" "path/filepath" + "strings" "sync" "time" @@ -62,6 +64,12 @@ func CacheImagesForBootstrapper(imageRepository string, version string, clusterB // LoadImages loads previously cached images into the container runtime func LoadImages(cc *config.ClusterConfig, runner command.Runner, images []string, cacheDir string) error { + // Skip loading images if images already exist + if imagesPreloaded(runner, images) { + glog.Infof("Images are preloaded, skipping loading") + return nil + } + glog.Infof("LoadImages start: %s", images) start := time.Now() @@ -99,6 +107,28 @@ func LoadImages(cc *config.ClusterConfig, runner command.Runner, images []string return nil } +func imagesPreloaded(runner command.Runner, images []string) bool { + rr, err := runner.RunCmd(exec.Command("docker", "images", "--format", "{{.Repository}}:{{.Tag}}")) + if err != nil { + return false + } + preloadedImages := map[string]struct{}{} + for _, i := range strings.Split(rr.Stdout.String(), "\n") { + preloadedImages[i] = struct{}{} + } + + glog.Infof("Got preloaded images: %s", rr.Output()) + + // Make sure images == imgs + for _, i := range images { + if _, ok := preloadedImages[i]; !ok { + glog.Infof("%s wasn't preloaded", i) + return false + } + } + return true +} + // needsTransfer returns an error if an image needs to be retransfered func needsTransfer(imgClient *client.Client, imgName string, cr cruntime.Manager) error { imgDgst := "" // for instance sha256:7c92a2c6bbcb6b6beff92d0a940779769c2477b807c202954c537e2e0deb9bed diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index 03b854a99ecd..fce265bf7764 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -24,6 +24,7 @@ import ( "github.com/spf13/viper" "golang.org/x/sync/errgroup" cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config" + "k8s.io/minikube/pkg/drivers/kic" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/exit" @@ -31,6 +32,7 @@ import ( "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/preload" ) // beginCacheRequiredImages caches images required for kubernetes version in the background @@ -44,7 +46,7 @@ func beginCacheRequiredImages(g *errgroup.Group, imageRepository string, k8sVers }) } -func handleDownloadOnly(cacheGroup *errgroup.Group, k8sVersion string) { +func handleDownloadOnly(cacheGroup, kicGroup *errgroup.Group, k8sVersion string) { // If --download-only, complete the remaining downloads and exit. if !viper.GetBool("download-only") { return @@ -56,6 +58,7 @@ func handleDownloadOnly(cacheGroup *errgroup.Group, k8sVersion string) { exit.WithError("Failed to cache kubectl", err) } waitCacheRequiredImages(cacheGroup) + waitDownloadKicArtifacts(kicGroup) if err := saveImagesToTarFromConfig(); err != nil { exit.WithError("Failed to cache images to tar", err) } @@ -79,6 +82,26 @@ func doCacheBinaries(k8sVersion string) error { return machine.CacheBinariesForBootstrapper(k8sVersion, viper.GetString(cmdcfg.Bootstrapper)) } +func beginDownloadKicArtifacts(g *errgroup.Group, k8sVersion, cRuntime string) { + glog.Info("Beginning downloading kic artifacts") + g.Go(func() error { + glog.Infof("Downloading %s to local daemon", kic.BaseImage) + return image.WriteImageToDaemon(kic.BaseImage) + }) + + g.Go(func() error { + glog.Info("Caching tarball of preloaded images") + return preload.CacheTarball(k8sVersion, cRuntime) + }) +} + +func waitDownloadKicArtifacts(g *errgroup.Group) { + if err := g.Wait(); err != nil { + glog.Errorln("Error downloading kic artifacts: ", err) + } + glog.Info("Successfully downloaded all kic artifacts") +} + // waitCacheRequiredImages blocks until the required images are all cached. func waitCacheRequiredImages(g *errgroup.Group) { if !viper.GetBool(cacheImages) { diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 6cd74d1a4355..24f6f185f5bd 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -34,9 +34,18 @@ import ( // Start spins up a guest and starts the kubernetes node. func Start(mc config.ClusterConfig, n config.Node, primary bool, existingAddons map[string]bool) (*kubeconfig.Settings, error) { + k8sVersion := mc.KubernetesConfig.KubernetesVersion + driverName := mc.Driver + + // See if we can create a volume of preloaded images + // If not, pull images in the background while the VM boots. + var kicGroup errgroup.Group + if driver.IsKIC(driverName) { + beginDownloadKicArtifacts(&kicGroup, k8sVersion, mc.KubernetesConfig.ContainerRuntime) + } // Now that the ISO is downloaded, pull images in the background while the VM boots. var cacheGroup errgroup.Group - beginCacheRequiredImages(&cacheGroup, mc.KubernetesConfig.ImageRepository, n.KubernetesVersion) + beginCacheRequiredImages(&cacheGroup, mc.KubernetesConfig.ImageRepository, k8sVersion) // Abstraction leakage alert: startHost requires the config to be saved, to satistfy pkg/provision/buildroot. // Hence, saveConfig must be called before startHost, and again afterwards when we know the IP. @@ -44,10 +53,10 @@ func Start(mc config.ClusterConfig, n config.Node, primary bool, existingAddons exit.WithError("Failed to save config", err) } - k8sVersion := mc.KubernetesConfig.KubernetesVersion - driverName := mc.Driver // exits here in case of --download-only option. - handleDownloadOnly(&cacheGroup, k8sVersion) + handleDownloadOnly(&cacheGroup, &kicGroup, k8sVersion) + waitDownloadKicArtifacts(&kicGroup) + mRunner, preExists, machineAPI, host := startMachine(&mc, &n) defer machineAPI.Close() // configure the runtime (docker, containerd, crio) diff --git a/pkg/minikube/preload/constants.go b/pkg/minikube/preload/constants.go new file mode 100644 index 000000000000..ca09325e37e6 --- /dev/null +++ b/pkg/minikube/preload/constants.go @@ -0,0 +1,22 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package preload + +const ( + // Version is the current version of the preloaded tarball + Version = "v1" +) diff --git a/pkg/minikube/preload/preload.go b/pkg/minikube/preload/preload.go new file mode 100644 index 000000000000..703f5bd39ef9 --- /dev/null +++ b/pkg/minikube/preload/preload.go @@ -0,0 +1,149 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package preload + +import ( + "context" + "crypto/md5" + "fmt" + "io/ioutil" + "net/http" + "os" + "path" + + "cloud.google.com/go/storage" + "google.golang.org/api/option" + + "github.com/golang/glog" + "github.com/hashicorp/go-getter" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/util" +) + +// returns name of the tarball +func tarballName(k8sVersion string) string { + return fmt.Sprintf("preloaded-images-k8s-%s-%s-docker-overlay2.tar.lz4", Version, k8sVersion) +} + +// returns the name of the checksum file +func checksumName(k8sVersion string) string { + return fmt.Sprintf("%s.checksum", tarballName(k8sVersion)) +} + +// returns target dir for all cached items related to preloading +func targetDir() string { + return localpath.MakeMiniPath("cache", "preloaded-tarball") +} + +// ChecksumFilepath returns path to checksum file +func ChecksumFilepath(k8sVersion string) string { + return path.Join(targetDir(), checksumName(k8sVersion)) +} + +// TarballFilepath returns the path to the preloaded tarball +func TarballFilepath(k8sVersion string) string { + return path.Join(targetDir(), tarballName(k8sVersion)) +} + +// remoteTarballURL returns the URL for the remote tarball in GCS +func remoteTarballURL(k8sVersion string) string { + return fmt.Sprintf("https://storage.googleapis.com/%s/%s", constants.PreloadedVolumeTarballsBucket, tarballName(k8sVersion)) +} + +// CacheTarball caches the preloaded images tarball on the host machine +func CacheTarball(k8sVersion, containerRuntime string) error { + if containerRuntime != "docker" { + return nil + } + targetFilepath := TarballFilepath(k8sVersion) + + if _, err := os.Stat(targetFilepath); err == nil { + if err := verifyChecksum(k8sVersion); err == nil { + glog.Infof("Found %s in cache, skipping downloading", targetFilepath) + return nil + } + } + + url := remoteTarballURL(k8sVersion) + + // Make sure we support this k8s version + if _, err := http.Get(url); err != nil { + glog.Infof("Unable to get response from %s, skipping downloading: %v", url, err) + return nil + } + + out.T(out.FileDownload, "Downloading preloaded images tarball for k8s {{.version}} ...", out.V{"version": k8sVersion}) + client := &getter.Client{ + Src: url, + Dst: targetFilepath, + Mode: getter.ClientModeFile, + Options: []getter.ClientOption{getter.WithProgress(util.DefaultProgressBar)}, + } + + glog.Infof("Downloading: %+v", client) + if err := client.Get(); err != nil { + return errors.Wrapf(err, "download failed: %s", url) + } + // Give downloaded drivers a baseline decent file permission + if err := os.Chmod(targetFilepath, 0755); err != nil { + return err + } + // Save checksum file locally + if err := saveChecksumFile(k8sVersion); err != nil { + return errors.Wrap(err, "saving checksum file") + } + return verifyChecksum(k8sVersion) +} + +func saveChecksumFile(k8sVersion string) error { + ctx := context.Background() + client, err := storage.NewClient(ctx, option.WithoutAuthentication()) + if err != nil { + return errors.Wrap(err, "getting storage client") + } + attrs, err := client.Bucket(constants.PreloadedVolumeTarballsBucket).Object(tarballName(k8sVersion)).Attrs(ctx) + if err != nil { + return errors.Wrap(err, "getting storage object") + } + checksum := attrs.MD5 + return ioutil.WriteFile(ChecksumFilepath(k8sVersion), checksum, 0644) +} + +// verifyChecksum returns true if the checksum of the local binary matches +// the checksum of the remote binary +func verifyChecksum(k8sVersion string) error { + // get md5 checksum of tarball path + contents, err := ioutil.ReadFile(TarballFilepath(k8sVersion)) + if err != nil { + return errors.Wrap(err, "reading tarball") + } + checksum := md5.Sum(contents) + + remoteChecksum, err := ioutil.ReadFile(ChecksumFilepath(k8sVersion)) + if err != nil { + return errors.Wrap(err, "reading checksum file") + } + + // create a slice of checksum, which is [16]byte + if string(remoteChecksum) != string(checksum[:]) { + return fmt.Errorf("checksum of %s does not match remote checksum (%s != %s)", TarballFilepath(k8sVersion), string(remoteChecksum), string(checksum[:])) + } + return nil +} diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index 0b66dfdecb4b..1c96796a05c5 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -45,13 +45,15 @@ func init() { func configure(mc config.ClusterConfig) (interface{}, error) { return kic.NewDriver(kic.Config{ - MachineName: mc.Name, - StorePath: localpath.MiniPath(), - ImageDigest: kic.BaseImage, - CPU: mc.CPUs, - Memory: mc.Memory, - OCIBinary: oci.Docker, - APIServerPort: mc.Nodes[0].Port, + MachineName: mc.Name, + StorePath: localpath.MiniPath(), + ImageDigest: kic.BaseImage, + CPU: mc.CPUs, + Memory: mc.Memory, + OCIBinary: oci.Docker, + APIServerPort: mc.Nodes[0].Port, + KubernetesVersion: mc.KubernetesConfig.KubernetesVersion, + ContainerRuntime: mc.KubernetesConfig.ContainerRuntime, }), nil } diff --git a/test/integration/aaa_download_only_test.go b/test/integration/aaa_download_only_test.go index e0bf1fba32ae..7e2f5777c8c2 100644 --- a/test/integration/aaa_download_only_test.go +++ b/test/integration/aaa_download_only_test.go @@ -20,19 +20,26 @@ package integration import ( "context" + "crypto/md5" "encoding/json" "fmt" + "io/ioutil" "os" "os/exec" "path/filepath" "runtime" "strings" "testing" + "time" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/daemon" + "k8s.io/minikube/pkg/drivers/kic" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/minikube/preload" ) func TestDownloadOnly(t *testing.T) { @@ -160,3 +167,52 @@ func TestDownloadOnly(t *testing.T) { }) } +func TestDownloadOnlyDocker(t *testing.T) { + if !runningDockerDriver(StartArgs()) { + t.Skip("this test only runs with the docker driver") + } + + profile := UniqueProfileName("download-docker") + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + defer Cleanup(t, profile, cancel) + + args := []string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr", "--vm-driver=docker"} + rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) + if err != nil { + t.Errorf("%s failed: %v:\n%s", args, err, rr.Output()) + } + + // Make sure the preloaded image tarball exists + tarball := preload.TarballFilepath(constants.DefaultKubernetesVersion) + contents, err := ioutil.ReadFile(tarball) + if err != nil { + t.Errorf("reading tarball: %v", err) + } + // Make sure it has the correct checksum + checksum := md5.Sum(contents) + remoteChecksum, err := ioutil.ReadFile(preload.ChecksumFilepath(constants.DefaultKubernetesVersion)) + if err != nil { + t.Errorf("reading checksum file: %v", err) + } + if string(remoteChecksum) != string(checksum[:]) { + t.Errorf("checksum of %s does not match remote checksum (%s != %s)", tarball, string(remoteChecksum), string(checksum[:])) + } + + // Make sure this image exists in the docker daemon + ref, err := name.ParseReference(kic.BaseImage) + if err != nil { + t.Errorf("parsing reference failed: %v", err) + } + if _, err := daemon.Image(ref); err != nil { + t.Errorf("expected image does not exist in local daemon: %v", err) + } +} + +func runningDockerDriver(startArgs []string) bool { + for _, s := range startArgs { + if s == "--vm-driver=docker" { + return true + } + } + return false +}