Preload images into docker volume

Makefile

@@ -19,8 +19,10 @@ VERSION_BUILD ?= 3
 RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).${VERSION_BUILD}
 VERSION ?= v$(RAW_VERSION)
 
-KUBERNETES_VERSION ?= $(shell egrep "^var DefaultKubernetesVersion" pkg/minikube/constants/constants.go | cut -d \" -f2)
+KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/constants/constants.go | cut -d \" -f2)
 KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2)
+PRELOADED_TARBALL_VERSION ?= $(shell egrep "Version =" pkg/minikube/preload/constants.go | cut -d \" -f2)
+PRELOADED_VOLUMES_GCS_BUCKET ?= $(shell egrep "PreloadedVolumeTarballsBucket =" pkg/minikube/constants/constants.go | cut -d \" -f2)
 
 # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
 ISO_VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).3
@@ -521,15 +523,14 @@ kic-base-image: ## builds the base image used for kic.
 	docker rmi -f $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot || true
 	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --target base .
 
-
-.PHONY: kic-preloaded-base-image
-kic-preloaded-base-image: generate-preloaded-images-tar ## builds the base image used for kic.
-	docker rmi -f $(REGISTRY)/kicbase:$(KIC_VERSION)-k8s-${KUBERNETES_VERSION} || true
-	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-k8s-${KUBERNETES_VERSION}  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --build-arg KUBERNETES_VERSION=${KUBERNETES_VERSION} .
+.PHONY: upload-preloaded-images-tar
+upload-preloaded-images-tar: generate-preloaded-images-tar # Upload the preloaded images tar to the GCS bucket. Specify a specific kubernetes version to build via `KUBERNETES_VERSION=vx.y.z make upload-preloaded-images-tar`.
+	gsutil cp out/preloaded-images-k8s-${PRELOADED_TARBALL_VERSION}-${KUBERNETES_VERSION}-docker-overlay2.tar.lz4 gs://${PRELOADED_VOLUMES_GCS_BUCKET}
+	gsutil acl ch -u AllUsers:R gs://${PRELOADED_VOLUMES_GCS_BUCKET}/preloaded-images-k8s-${PRELOADED_TARBALL_VERSION}-${KUBERNETES_VERSION}-docker-overlay2.tar.lz4 
 
 .PHONY: generate-preloaded-images-tar
-generate-preloaded-images-tar: out/minikube
-	go run ./hack/preload-images/preload_images.go -kubernetes-version ${KUBERNETES_VERSION} 
+generate-preloaded-images-tar:
+	go run ./hack/preload-images/preload_images.go -kubernetes-version ${KUBERNETES_VERSION} -preloaded-tarball-version ${PRELOADED_TARBALL_VERSION}
 
 
 .PHONY: push-storage-provisioner-image

go.mod

@@ -3,6 +3,7 @@ module k8s.io/minikube
 go 1.13
 
 require (
+	cloud.google.com/go v0.45.1
 	github.com/Parallels/docker-machine-parallels v1.3.0
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/blang/semver v3.5.0+incompatible
@@ -69,6 +70,7 @@ require (
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
 	golang.org/x/sys v0.0.0-20191010194322-b09406accb47
 	golang.org/x/text v0.3.2
+	google.golang.org/api v0.9.0
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 	k8s.io/api v0.17.3
 	k8s.io/apimachinery v0.17.3

hack/images/kicbase.Dockerfile

@@ -52,13 +52,3 @@ RUN apt-get clean -y && rm -rf \
   /usr/share/man/* \
   /usr/share/local/* \
   RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
-
-
-FROM busybox
-ARG KUBERNETES_VERSION
-COPY out/preloaded-images-k8s-$KUBERNETES_VERSION.tar /preloaded-images.tar
-RUN tar xvf /preloaded-images.tar -C /
-
-FROM base
-COPY --from=1 /var/lib/docker /var/lib/docker
-COPY --from=1 /var/lib/minikube/binaries /var/lib/minikube/binaries

hack/preload-images/preload_images.go

@@ -25,6 +25,11 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/drivers/kic"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/localpath"
 )
 
 const (
@@ -33,17 +38,28 @@ const (
 )
 
 var (
-	kubernetesVersion = ""
-	tarballFilename   = ""
+	kubernetesVersion       = ""
+	tarballFilename         = ""
+	dockerStorageDriver     = ""
+	preloadedTarballVersion = ""
+	containerRuntime        = ""
 )
 
 func init() {
 	flag.StringVar(&kubernetesVersion, "kubernetes-version", "", "desired kubernetes version, for example `v1.17.2`")
+	flag.StringVar(&dockerStorageDriver, "docker-storage-driver", "overlay2", "docker storage driver backend")
+	flag.StringVar(&preloadedTarballVersion, "preloaded-tarball-version", "", "preloaded tarball version")
+	flag.StringVar(&containerRuntime, "container-runtime", "docker", "container runtime")
+
 	flag.Parse()
-	tarballFilename = fmt.Sprintf("preloaded-images-k8s-%s.tar", kubernetesVersion)
+	tarballFilename = fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-%s.tar.lz4", preloadedTarballVersion, kubernetesVersion, containerRuntime, dockerStorageDriver)
 }
 
 func main() {
+	if err := verifyDockerStorage(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
 	if err := executePreloadImages(); err != nil {
 		fmt.Println(err)
 		os.Exit(1)
@@ -56,42 +72,74 @@ func executePreloadImages() error {
 			fmt.Println(err)
 		}
 	}()
-	if err := startMinikube(); err != nil {
+
+	driver := kic.NewDriver(kic.Config{
+		KubernetesVersion: kubernetesVersion,
+		ContainerRuntime:  driver.Docker,
+		OCIBinary:         oci.Docker,
+		MachineName:       profile,
+		ImageDigest:       kic.BaseImage,
+		StorePath:         localpath.MiniPath(),
+		CPU:               2,
+		Memory:            4000,
+		APIServerPort:     8080,
+	})
+
+	baseDir := filepath.Dir(driver.GetSSHKeyPath())
+	defer os.Remove(baseDir)
+
+	if err := os.MkdirAll(baseDir, 0755); err != nil {
 		return err
 	}
+	if err := driver.Create(); err != nil {
+		return errors.Wrap(err, "creating kic driver")
+	}
+
+	// Now, get images to pull
+	imgs, err := images.Kubeadm("", kubernetesVersion)
+	if err != nil {
+		return errors.Wrap(err, "kubeadm images")
+	}
+
+	for _, img := range append(imgs, kic.OverlayImage) {
+		cmd := exec.Command("docker", "exec", profile, "docker", "pull", img)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Run(); err != nil {
+			return errors.Wrapf(err, "downloading %s", img)
+		}
+	}
+
+	// Create image tarball
 	if err := createImageTarball(); err != nil {
 		return err
 	}
 	return copyTarballToHost()
 }
 
-func startMinikube() error {
-	cmd := exec.Command(minikubePath, "start", "-p", profile, "--memory", "4000", "--kubernetes-version", kubernetesVersion, "--wait=false")
-	cmd.Stdout = os.Stdout
-	return cmd.Run()
-}
-
 func createImageTarball() error {
-	cmd := exec.Command(minikubePath, "ssh", "-p", profile, "--", "sudo", "tar", "cvf", tarballFilename, "/var/lib/docker", "/var/lib/minikube/binaries")
+	dirs := []string{
+		fmt.Sprintf("./lib/docker/%s", dockerStorageDriver),
+		"./lib/docker/image",
+	}
+	args := []string{"exec", profile, "sudo", "tar", "-I", "lz4", "-C", "/var", "-cvf", tarballFilename}
+	args = append(args, dirs...)
+	cmd := exec.Command("docker", args...)
 	cmd.Stdout = os.Stdout
-	return cmd.Run()
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, "creating image tarball")
+	}
+	return nil
 }
 
 func copyTarballToHost() error {
-	sshKey, err := runCmd([]string{minikubePath, "ssh-key", "-p", profile})
-	if err != nil {
-		return errors.Wrap(err, "getting ssh-key")
-	}
-
-	ip, err := runCmd([]string{minikubePath, "ip", "-p", profile})
-	if err != nil {
-		return errors.Wrap(err, "getting ip")
-	}
-
 	dest := filepath.Join("out/", tarballFilename)
-	args := []string{"scp", "-o", "StrictHostKeyChecking=no", "-i", sshKey, fmt.Sprintf("docker@%s:/home/docker/%s", ip, tarballFilename), dest}
-	_, err = runCmd(args)
-	return err
+	cmd := exec.Command("docker", "cp", fmt.Sprintf("%s:/%s", profile, tarballFilename), dest)
+	cmd.Stdout = os.Stdout
+	if err := cmd.Run(); err != nil {
+		return errors.Wrap(err, "copying tarball to host")
+	}
+	return nil
 }
 
 func deleteMinikube() error {
@@ -100,8 +148,15 @@ func deleteMinikube() error {
 	return cmd.Run()
 }
 
-func runCmd(command []string) (string, error) {
-	cmd := exec.Command(command[0], command[1:]...)
+func verifyDockerStorage() error {
+	cmd := exec.Command("docker", "info", "-f", "{{.Info.Driver}}")
 	output, err := cmd.Output()
-	return strings.Trim(string(output), "\n "), err
+	if err != nil {
+		return err
+	}
+	driver := strings.Trim(string(output), " \n")
+	if driver != dockerStorageDriver {
+		return fmt.Errorf("docker storage driver %s does not match requested %s", driver, dockerStorageDriver)
+	}
+	return nil
 }

pkg/drivers/kic/kic.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/preload"
 )
 
 // Driver represents a kic driver https://minikube.sigs.k8s.io/docs/reference/drivers/docker
@@ -90,14 +91,23 @@ func (d *Driver) Create() error {
 			ContainerPort: constants.DockerDaemonPort,
 		},
 	)
-	err := oci.CreateContainerNode(params)
-	if err != nil {
+	if err := oci.CreateContainerNode(params); err != nil {
 		return errors.Wrap(err, "create kic node")
 	}
 
 	if err := d.prepareSSH(); err != nil {
 		return errors.Wrap(err, "prepare kic ssh")
 	}
+
+	t := time.Now()
+	glog.Infof("Starting extracting preloaded images to volume")
+	// Extract preloaded images to container
+	if err := oci.ExtractTarballToVolume(preload.TarballFilepath(d.NodeConfig.KubernetesVersion), params.Name, BaseImage); err != nil {
+		glog.Infof("Unable to extract preloaded tarball to volume: %v", err)
+	} else {
+		glog.Infof("Took %f seconds to extract preloaded images to volume", time.Since(t).Seconds())
+	}
+
 	return nil
 }
 

pkg/drivers/kic/oci/volumes.go

@@ -103,6 +103,19 @@ func allVolumesByLabel(ociBin string, label string) ([]string, error) {
 	return vols, err
 }
 
+// ExtractTarballToVolume runs a docker image imageName which extracts the tarball at tarballPath
+// to the volume named volumeName
+func ExtractTarballToVolume(tarballPath, volumeName, imageName string) error {
+	if err := PointToHostDockerDaemon(); err != nil {
+		return errors.Wrap(err, "point host docker-daemon")
+	}
+	cmd := exec.Command(Docker, "run", "--rm", "--entrypoint", "/usr/bin/tar", "-v", fmt.Sprintf("%s:/preloaded.tar:ro", tarballPath), "-v", fmt.Sprintf("%s:/extractDir", volumeName), imageName, "-I", "lz4", "-xvf", "/preloaded.tar", "-C", "/extractDir")
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return errors.Wrapf(err, "output %s", string(out))
+	}
+	return nil
+}
+
 // createDockerVolume creates a docker volume to be attached to the container with correct labels and prefixes based on profile name
 // Caution ! if volume already exists does NOT return an error and will not apply the minikube labels on it.
 // TODO: this should be fixed as a part of https://github.com/kubernetes/minikube/issues/6530

pkg/drivers/kic/types.go

@@ -46,14 +46,16 @@ var (
 
 // Config is configuration for the kic driver used by registry
 type Config struct {
-	MachineName   string            // maps to the container name being created
-	CPU           int               // Number of CPU cores assigned to the container
-	Memory        int               // max memory in MB
-	StorePath     string            // libmachine store path
-	OCIBinary     string            // oci tool to use (docker, podman,...)
-	ImageDigest   string            // image name with sha to use for the node
-	Mounts        []oci.Mount       // mounts
-	APIServerPort int               // kubernetes api server port inside the container
-	PortMappings  []oci.PortMapping // container port mappings
-	Envs          map[string]string // key,value of environment variables passed to the node
+	MachineName       string            // maps to the container name being created
+	CPU               int               // Number of CPU cores assigned to the container
+	Memory            int               // max memory in MB
+	StorePath         string            // libmachine store path
+	OCIBinary         string            // oci tool to use (docker, podman,...)
+	ImageDigest       string            // image name with sha to use for the node
+	Mounts            []oci.Mount       // mounts
+	APIServerPort     int               // kubernetes api server port inside the container
+	PortMappings      []oci.PortMapping // container port mappings
+	Envs              map[string]string // key,value of environment variables passed to the node
+	KubernetesVersion string            // kubernetes version to install
+	ContainerRuntime  string            // container runtime kic is running
 }

pkg/minikube/constants/constants.go

@@ -63,6 +63,9 @@ const (
 	MinikubeActiveDockerdEnv = "MINIKUBE_ACTIVE_DOCKERD"
 	// PodmanVarlinkBridgeEnv is used for podman settings
 	PodmanVarlinkBridgeEnv = "PODMAN_VARLINK_BRIDGE"
+
+	// PreloadedVolumeTarballsBucket is the name of the GCS bucket where preloaded volume tarballs exist
+	PreloadedVolumeTarballsBucket = "minikube-preloaded-volume-tarballs"
 )
 
 var (

pkg/minikube/image/image.go

@@ -20,8 +20,10 @@ import (
 	"context"
 	"io/ioutil"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/docker/docker/client"
@@ -31,6 +33,8 @@ import (
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/daemon"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
@@ -76,6 +80,37 @@ func DigestByGoLib(imgName string) string {
 	return cf.Hex
 }
 
+// WriteImageToDaemon write img to the local docker daemon
+func WriteImageToDaemon(img string) error {
+	glog.Infof("Writing %s to local daemon", img)
+	if err := oci.PointToHostDockerDaemon(); err != nil {
+		return errors.Wrap(err, "point host docker-daemon")
+	}
+	// Check if image exists locally
+	cmd := exec.Command("docker", "images", "--format", "{{.Repository}}:{{.Tag}}@{{.Digest}}")
+	if output, err := cmd.Output(); err == nil {
+		if strings.Contains(string(output), img) {
+			glog.Infof("Found %s in local docker daemon, skipping pull", img)
+			return nil
+		}
+	}
+	// Else, pull it
+	ref, err := name.ParseReference(img)
+	if err != nil {
+		return errors.Wrap(err, "parsing reference")
+	}
+	i, err := remote.Image(ref)
+	if err != nil {
+		return errors.Wrap(err, "getting remote image")
+	}
+	tag, err := name.NewTag(strings.Split(img, "@")[0])
+	if err != nil {
+		return errors.Wrap(err, "getting tag")
+	}
+	_, err = daemon.Write(tag, i)
+	return err
+}
+
 func retrieveImage(ref name.Reference) (v1.Image, error) {
 	glog.Infof("retrieving image: %+v", ref)
 	img, err := daemon.Image(ref)