Preload images into docker volume

Makefile

@@ -19,7 +19,7 @@ VERSION_BUILD ?= 3
 RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).${VERSION_BUILD}
 VERSION ?= v$(RAW_VERSION)
 
-KUBERNETES_VERSION ?= $(shell egrep "^var DefaultKubernetesVersion" pkg/minikube/constants/constants.go | cut -d \" -f2)
+KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/constants/constants.go | cut -d \" -f2)
 KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2)
 
 # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
@@ -511,11 +511,9 @@ kic-base-image: ## builds the base image used for kic.
 	docker rmi -f $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot || true
 	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --target base .
 
-
-.PHONY: kic-preloaded-base-image
-kic-preloaded-base-image: generate-preloaded-images-tar ## builds the base image used for kic.
-	docker rmi -f $(REGISTRY)/kicbase:$(KIC_VERSION)-k8s-${KUBERNETES_VERSION} || true
-	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-k8s-${KUBERNETES_VERSION}  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --build-arg KUBERNETES_VERSION=${KUBERNETES_VERSION} .
+.PHONY: upload-preloaded-images-tar
+upload-preloaded-images-tar: generate-preloaded-images-tar # Upload the preloaded images tar to the GCS bucket. Specify a specific kubernetes version to build via `KUBERNETES_VERSION=vx.y.z make upload-preloaded-images-tar`.
+	gsutil cp out/preloaded-images-k8s-${KUBERNETES_VERSION}.tar.lz4 gs://minikube-docker-volume-tarballs
 
 .PHONY: generate-preloaded-images-tar
 generate-preloaded-images-tar: out/minikube

hack/images/kicbase.Dockerfile

@@ -44,13 +44,3 @@ RUN rm -rf \
   /usr/share/man/* \
   /usr/share/local/* \
   RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
-
-
-FROM busybox
-ARG KUBERNETES_VERSION
-COPY out/preloaded-images-k8s-$KUBERNETES_VERSION.tar /preloaded-images.tar
-RUN tar xvf /preloaded-images.tar -C /
-
-FROM base
-COPY --from=1 /var/lib/docker /var/lib/docker
-COPY --from=1 /var/lib/minikube/binaries /var/lib/minikube/binaries

hack/preload-images/preload_images.go

@@ -22,7 +22,6 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"strings"
 
 	"github.com/pkg/errors"
 )
@@ -40,7 +39,7 @@ var (
 func init() {
 	flag.StringVar(&kubernetesVersion, "kubernetes-version", "", "desired kubernetes version, for example `v1.17.2`")
 	flag.Parse()
-	tarballFilename = fmt.Sprintf("preloaded-images-k8s-%s.tar", kubernetesVersion)
+	tarballFilename = fmt.Sprintf("preloaded-images-k8s-%s.tar.lz4", kubernetesVersion)
 }
 
 func main() {
@@ -66,42 +65,32 @@ func executePreloadImages() error {
 }
 
 func startMinikube() error {
-	cmd := exec.Command(minikubePath, "start", "-p", profile, "--memory", "4000", "--kubernetes-version", kubernetesVersion, "--wait=false")
+	cmd := exec.Command(minikubePath, "start", "-p", profile, "--memory", "4000", "--kubernetes-version", kubernetesVersion, "--wait=false", "--vm-driver=docker")
 	cmd.Stdout = os.Stdout
 	return cmd.Run()
 }
 
 func createImageTarball() error {
-	cmd := exec.Command(minikubePath, "ssh", "-p", profile, "--", "sudo", "tar", "cvf", tarballFilename, "/var/lib/docker", "/var/lib/minikube/binaries")
+	cmd := exec.Command(minikubePath, "ssh", "-p", profile, "--", "cd", "/var", "&&", "sudo", "tar", "-I", "lz4", "-cvf", tarballFilename, "./lib/docker", "./lib/minikube/binaries")
 	cmd.Stdout = os.Stdout
-	return cmd.Run()
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, "creating image tarball")
+	}
+	return nil
 }
 
 func copyTarballToHost() error {
-	sshKey, err := runCmd([]string{minikubePath, "ssh-key", "-p", profile})
-	if err != nil {
-		return errors.Wrap(err, "getting ssh-key")
-	}
-
-	ip, err := runCmd([]string{minikubePath, "ip", "-p", profile})
-	if err != nil {
-		return errors.Wrap(err, "getting ip")
-	}
-
 	dest := filepath.Join("out/", tarballFilename)
-	args := []string{"scp", "-o", "StrictHostKeyChecking=no", "-i", sshKey, fmt.Sprintf("docker@%s:/home/docker/%s", ip, tarballFilename), dest}
-	_, err = runCmd(args)
-	return err
+	cmd := exec.Command("docker", "cp", fmt.Sprintf("%s:/var/%s", profile, tarballFilename), dest)
+	cmd.Stdout = os.Stdout
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, "copying tarball to host")
+	}
+	return nil
 }
 
 func deleteMinikube() error {
 	cmd := exec.Command(minikubePath, "delete", "-p", profile)
 	cmd.Stdout = os.Stdout
 	return cmd.Run()
 }
-
-func runCmd(command []string) (string, error) {
-	cmd := exec.Command(command[0], command[1:]...)
-	output, err := cmd.Output()
-	return strings.Trim(string(output), "\n "), err
-}

pkg/drivers/kic/kic.go

@@ -22,6 +22,7 @@ import (
 	"os/exec"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/docker/machine/libmachine/drivers"
 	"github.com/docker/machine/libmachine/log"
@@ -88,7 +89,15 @@ func (d *Driver) Create() error {
 			ContainerPort: constants.DockerDaemonPort,
 		},
 	)
-	err := oci.CreateContainerNode(params)
+	t := time.Now()
+	glog.Infof("Starting creating preloaded images volume")
+	volumeName, err := oci.CreatePreloadedImagesVolume(d.NodeConfig.KubernetesVersion, BaseImage)
+	if err != nil {
+		glog.Infof("Unable to create preloaded images volume: %v", err)
+	}
+	glog.Infof("Finished creating preloaded images volume in %f seconds", time.Since(t).Seconds())
+	params.PreloadedVolume = volumeName
+	err = oci.CreateContainerNode(params)
 	if err != nil {
 		return errors.Wrap(err, "create kic node")
 	}

pkg/drivers/kic/oci/oci.go

@@ -99,11 +99,15 @@ func CreateContainerNode(p CreateParams) error {
 		runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var:exec", hostVarVolPath))
 	}
 	if p.OCIBinary == Docker {
-		if err := createDockerVolume(p.Name); err != nil {
-			return errors.Wrapf(err, "creating volume for %s container", p.Name)
+		volumeName := p.PreloadedVolume
+		if volumeName == "" {
+			if err := createDockerVolume(p.Name); err != nil {
+				return errors.Wrapf(err, "creating volume for %s container", p.Name)
+			}
+			glog.Infof("Successfully created a docker volume %s", p.Name)
+			volumeName = p.Name
 		}
-		glog.Infof("Successfully created a docker volume %s", p.Name)
-		runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name))
+		runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", volumeName))
 		// setting resource limit in privileged mode is only supported by docker
 		// podman error: "Error: invalid configuration, cannot set resources with rootless containers not using cgroups v2 unified mode"
 		runArgs = append(runArgs, fmt.Sprintf("--cpus=%s", p.CPUs), fmt.Sprintf("--memory=%s", p.Memory))

pkg/drivers/kic/oci/types.go

@@ -31,18 +31,19 @@ const (
 
 // CreateParams are parameters needed to create a container
 type CreateParams struct {
-	Name          string            // used for container name and hostname
-	Image         string            // container image to use to create the node.
-	ClusterLabel  string            // label the containers we create using minikube so we can clean up
-	Role          string            // currently only role supported is control-plane
-	Mounts        []Mount           // volume mounts
-	APIServerPort int               // kubernetes api server port
-	PortMappings  []PortMapping     // ports to map to container from host
-	CPUs          string            // number of cpu cores assign to container
-	Memory        string            // memory (mbs) to assign to the container
-	Envs          map[string]string // environment variables to pass to the container
-	ExtraArgs     []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
-	OCIBinary     string            // docker or podman
+	Name            string            // used for container name and hostname
+	Image           string            // container image to use to create the node.
+	ClusterLabel    string            // label the containers we create using minikube so we can clean up
+	Role            string            // currently only role supported is control-plane
+	Mounts          []Mount           // volume mounts
+	APIServerPort   int               // kubernetes api server port
+	PortMappings    []PortMapping     // ports to map to container from host
+	CPUs            string            // number of cpu cores assign to container
+	Memory          string            // memory (mbs) to assign to the container
+	Envs            map[string]string // environment variables to pass to the container
+	ExtraArgs       []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
+	OCIBinary       string            // docker or podman
+	PreloadedVolume string            // volume of preloaded images
 }
 
 // createOpt is an option for Create

pkg/drivers/kic/oci/volumes.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/drivers/kic/preload"
 )
 
 // DeleteAllVolumesByLabel deletes all volumes that have a specific label
@@ -88,6 +89,56 @@ func allVolumesByLabel(ociBin string, label string) ([]string, error) {
 	return vols, err
 }
 
+// CreatePreloadedImagesVolume creates a volume with preloaded images
+func CreatePreloadedImagesVolume(k8sVersion, baseImage string) (string, error) {
+	if err := PointToHostDockerDaemon(); err != nil {
+		return "", errors.Wrap(err, "point host docker-daemon")
+	}
+	volumeName := fmt.Sprintf("k8s-%s", k8sVersion)
+	if dockerVolumeExists(volumeName) {
+		return volumeName, nil
+	}
+	if err := createDockerVolume(volumeName); err != nil {
+		return "", errors.Wrap(err, "creating docker volume")
+	}
+	tarballPath := preload.TarballPath(k8sVersion)
+
+	if err := extractTarballToVolume(tarballPath, volumeName, baseImage); err != nil {
+		return "", errors.Wrap(err, "extracting tarball to volume")
+	}
+	return volumeName, nil
+}
+
+func dockerVolumeExists(name string) bool {
+	if err := PointToHostDockerDaemon(); err != nil {
+		return false
+	}
+	cmd := exec.Command(Docker, "volume", "ls", "-q")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return false
+	}
+	names := strings.Split(string(out), "\n")
+	for _, n := range names {
+		if n == name {
+			return true
+		}
+	}
+	return false
+}
+
+func extractTarballToVolume(tarballPath, volumeName, imageName string) error {
+	if err := PointToHostDockerDaemon(); err != nil {
+		return errors.Wrap(err, "point host docker-daemon")
+	}
+	cmd := exec.Command(Docker, "run", "--rm", "--entrypoint", "/usr/bin/tar", "-v", fmt.Sprintf("%s:/preloaded.tar:ro", tarballPath), "-v", fmt.Sprintf("%s:/extractDir", volumeName), imageName, "-I", "lz4", "-xvf", "/preloaded.tar", "-C", "/extractDir")
+	fmt.Println(cmd.Args)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return errors.Wrapf(err, "output %s", string(out))
+	}
+	return nil
+}
+
 // createDockerVolume creates a docker volume to be attached to the container with correct labels and prefixes based on profile name
 // Caution ! if volume already exists does NOT return an error and will not apply the minikube labels on it.
 // TODO: this should be fixed as a part of https://github.com/kubernetes/minikube/issues/6530

pkg/drivers/kic/preload/preload.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package preload
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/golang/glog"
+	"github.com/jimmidyson/go-download"
+	"k8s.io/minikube/pkg/minikube/localpath"
+)
+
+// TarballPath returns the path to the preloaded tarball
+func TarballPath(k8sVersion string) string {
+	targetDir := localpath.MakeMiniPath("cache", "preloaded-tarball")
+	targetFilepath := path.Join(targetDir, fmt.Sprintf("preloaded-images-k8s-%s.tar.lz4", k8sVersion))
+	return targetFilepath
+}
+
+// CacheTarball caches the preloaded images tarball on the host machine
+func CacheTarball(k8sVersion string) error {
+	targetFilepath := TarballPath(k8sVersion)
+
+	if _, err := os.Stat(targetFilepath); err == nil {
+		glog.Infof("Found %s in cache, skipping downloading", targetFilepath)
+		return nil
+	}
+
+	url := fmt.Sprintf("https://storage.googleapis.com/minikube-docker-volume-tarballs/preloaded-images-k8s-%s.tar", k8sVersion)
+	glog.Infof("Downloading %s to %s", url, targetFilepath)
+	return download.ToFile(url, targetFilepath, download.FileOptions{Mkdirs: download.MkdirAll})
+}
+
+// UsingPreloadedVolume returns true if the preloaded tarball exists
+func UsingPreloadedVolume(k8sVersion string) bool {
+	path := TarballPath(k8sVersion)
+	_, err := os.Stat(path)
+	return err == nil
+}

pkg/drivers/kic/types.go

@@ -45,14 +45,15 @@ var (
 
 // Config is configuration for the kic driver used by registry
 type Config struct {
-	MachineName   string            // maps to the container name being created
-	CPU           int               // Number of CPU cores assigned to the container
-	Memory        int               // max memory in MB
-	StorePath     string            // libmachine store path
-	OCIBinary     string            // oci tool to use (docker, podman,...)
-	ImageDigest   string            // image name with sha to use for the node
-	Mounts        []oci.Mount       // mounts
-	APIServerPort int               // kubernetes api server port inside the container
-	PortMappings  []oci.PortMapping // container port mappings
-	Envs          map[string]string // key,value of environment variables passed to the node
+	MachineName       string            // maps to the container name being created
+	CPU               int               // Number of CPU cores assigned to the container
+	Memory            int               // max memory in MB
+	StorePath         string            // libmachine store path
+	OCIBinary         string            // oci tool to use (docker, podman,...)
+	ImageDigest       string            // image name with sha to use for the node
+	Mounts            []oci.Mount       // mounts
+	APIServerPort     int               // kubernetes api server port inside the container
+	PortMappings      []oci.PortMapping // container port mappings
+	Envs              map[string]string // key,value of environment variables passed to the node
+	KubernetesVersion string            // kubernetes version to install
 }

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -40,6 +40,7 @@ import (
 	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/minikube/pkg/drivers/kic"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/drivers/kic/preload"
 	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
@@ -436,8 +437,12 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 		glog.Warningf("unable to stop kubelet: %s command: %q output: %q", err, rr.Command(), rr.Output())
 	}
 
-	if err := bsutil.TransferBinaries(cfg.KubernetesConfig, k.c); err != nil {
-		return errors.Wrap(err, "downloading binaries")
+	// Skip transfer if using preloaded kic volume
+	skipTransfer := driver.IsKIC(cfg.Driver) && preload.UsingPreloadedVolume(cfg.KubernetesConfig.KubernetesVersion)
+	if !skipTransfer {
+		if err := bsutil.TransferBinaries(cfg.KubernetesConfig, k.c); err != nil {
+			return errors.Wrap(err, "downloading binaries")
+		}
 	}
 
 	var cniFile []byte