Fixes image repository flags when using CRI-O and containerd runtime

cmd/minikube/cmd/start.go

@@ -326,7 +326,7 @@ func runStart(cmd *cobra.Command, args []string) {
 	mRunner, preExists, machineAPI, host := startMachine(&config)
 	defer machineAPI.Close()
 	// configure the runtime (docker, containerd, crio)
-	cr := configureRuntimes(mRunner, driver)
+	cr := configureRuntimes(mRunner, driver, config.KubernetesConfig)
 	showVersionInfo(k8sVersion, cr)
 	waitCacheImages(&cacheGroup)
 
@@ -556,9 +556,8 @@ func selectImageRepository(mirrorCountry string, k8sVersion string) (bool, strin
 	}
 
 	checkRepository := func(repo string) error {
-		podInfraContainerImage, _ := images.CachedImages(repo, k8sVersion)
-
-		ref, err := name.ParseReference(podInfraContainerImage, name.WeakValidation)
+		pauseImage := images.PauseImage(repo, k8sVersion)
+		ref, err := name.ParseReference(pauseImage, name.WeakValidation)
 		if err != nil {
 			return err
 		}
@@ -1011,8 +1010,8 @@ func setupKubeAdm(mAPI libmachine.API, kc cfg.KubernetesConfig) bootstrapper.Boo
 }
 
 // configureRuntimes does what needs to happen to get a runtime going.
-func configureRuntimes(runner cruntime.CommandRunner, driver string) cruntime.Manager {
-	config := cruntime.Config{Type: viper.GetString(containerRuntime), Runner: runner}
+func configureRuntimes(runner cruntime.CommandRunner, driver string, k8s cfg.KubernetesConfig) cruntime.Manager {
+	config := cruntime.Config{Type: viper.GetString(containerRuntime), Runner: runner, ImageRepository: k8s.ImageRepository, KubernetesVersion: k8s.KubernetesVersion}
 	cr, err := cruntime.New(config)
 	if err != nil {
 		exit.WithError("Failed runtime", err)

pkg/minikube/bootstrapper/bootstrapper.go

@@ -68,7 +68,7 @@ func GetCachedBinaryList(bootstrapper string) []string {
 func GetCachedImageList(imageRepository string, version string, bootstrapper string) []string {
 	switch bootstrapper {
 	case BootstrapperTypeKubeadm:
-		_, images := images.CachedImages(imageRepository, version)
+		images := images.CachedImages(imageRepository, version)
 		return images
 	default:
 		return []string{}

pkg/minikube/bootstrapper/images/images.go

@@ -25,19 +25,36 @@ import (
 	minikubeVersion "k8s.io/minikube/pkg/version"
 )
 
-// CachedImages gets the images to cache for kubeadm for a version
-func CachedImages(imageRepository string, kubernetesVersionStr string) (string, []string) {
-	minikubeRepository := imageRepository
+// getImageRepositories returns either the k8s image registry on GCR or a mirror if specified
+func getImageRepository(imageRepository string) string {
 	if imageRepository == "" {
 		imageRepository = "k8s.gcr.io"
-		minikubeRepository = "gcr.io/k8s-minikube"
 	}
 	if !strings.HasSuffix(imageRepository, "/") {
 		imageRepository += "/"
 	}
+
+	return imageRepository
+}
+
+// getMinikubeRepository returns either the minikube image registry on GCR or a mirror if specified
+func getMinikubeRepository(imageRepository string) string {
+	minikubeRepository := imageRepository
+	if minikubeRepository == "" {
+		minikubeRepository = "gcr.io/k8s-minikube"
+	}
 	if !strings.HasSuffix(minikubeRepository, "/") {
 		minikubeRepository += "/"
 	}
+
+	return minikubeRepository
+}
+
+// CachedImages gets the images to cache for kubeadm for a version
+func CachedImages(imageRepositoryStr string, kubernetesVersionStr string) []string {
+	imageRepository := getImageRepository(imageRepositoryStr)
+	minikubeRepository := getMinikubeRepository(imageRepositoryStr)
+
 	v1_16plus := semver.MustParseRange(">=1.16.0")
 	v1_14plus := semver.MustParseRange(">=1.14.0 <1.16.0")
 	v1_13 := semver.MustParseRange(">=1.13.0 <1.14.0")
@@ -67,9 +84,8 @@ func CachedImages(imageRepository string, kubernetesVersionStr string) (string,
 		}...)
 	}
 
-	var podInfraContainerImage string
+	podInfraContainerImage := PauseImage(imageRepository, kubernetesVersionStr)
 	if v1_16plus(kubernetesVersion) {
-		podInfraContainerImage = imageRepository + "pause:3.1"
 		images = append(images, []string{
 			podInfraContainerImage,
 			imageRepository + "k8s-dns-kube-dns" + ArchTag(true) + "1.14.13",
@@ -80,7 +96,6 @@ func CachedImages(imageRepository string, kubernetesVersionStr string) (string,
 		}...)
 
 	} else if v1_14plus(kubernetesVersion) {
-		podInfraContainerImage = imageRepository + "pause:3.1"
 		images = append(images, []string{
 			podInfraContainerImage,
 			imageRepository + "k8s-dns-kube-dns" + ArchTag(true) + "1.14.13",
@@ -91,7 +106,6 @@ func CachedImages(imageRepository string, kubernetesVersionStr string) (string,
 		}...)
 
 	} else if v1_13(kubernetesVersion) {
-		podInfraContainerImage = imageRepository + "pause" + ArchTag(false) + "3.1"
 		images = append(images, []string{
 			podInfraContainerImage,
 			imageRepository + "k8s-dns-kube-dns" + ArchTag(true) + "1.14.8",
@@ -102,7 +116,6 @@ func CachedImages(imageRepository string, kubernetesVersionStr string) (string,
 		}...)
 
 	} else if v1_12(kubernetesVersion) {
-		podInfraContainerImage = imageRepository + "pause:3.1"
 		images = append(images, []string{
 			podInfraContainerImage,
 			imageRepository + "k8s-dns-kube-dns" + ArchTag(true) + "1.14.8",
@@ -113,7 +126,6 @@ func CachedImages(imageRepository string, kubernetesVersionStr string) (string,
 		}...)
 
 	} else if v1_11(kubernetesVersion) {
-		podInfraContainerImage = imageRepository + "pause" + ArchTag(false) + "3.1"
 		images = append(images, []string{
 			podInfraContainerImage,
 			imageRepository + "k8s-dns-kube-dns" + ArchTag(true) + "1.14.8",
@@ -122,8 +134,6 @@ func CachedImages(imageRepository string, kubernetesVersionStr string) (string,
 			imageRepository + "etcd" + ArchTag(true) + "3.2.18",
 			imageRepository + "coredns:1.1.3",
 		}...)
-	} else {
-		podInfraContainerImage = imageRepository + "pause" + ArchTag(false) + "3.0"
 	}
 
 	images = append(images, []string{
@@ -132,7 +142,46 @@ func CachedImages(imageRepository string, kubernetesVersionStr string) (string,
 		minikubeRepository + "storage-provisioner" + ArchTag(false) + "v1.8.1",
 	}...)
 
-	return podInfraContainerImage, images
+	return images
+}
+
+// PauseImage returns the image name for pause image (for pod infra)
+func PauseImage(imageRepositoryStr string, kubernetesVersionStr string) string {
+	imageRepository := getImageRepository(imageRepositoryStr)
+
+	v1_16plus := semver.MustParseRange(">=1.16.0")
+	v1_14plus := semver.MustParseRange(">=1.14.0 <1.16.0")
+	v1_13 := semver.MustParseRange(">=1.13.0 <1.14.0")
+	v1_12 := semver.MustParseRange(">=1.12.0 <1.13.0")
+	v1_11 := semver.MustParseRange(">=1.11.0 <1.12.0")
+
+	kubernetesVersion, err := semver.Make(strings.TrimPrefix(kubernetesVersionStr, minikubeVersion.VersionPrefix))
+	if err != nil {
+		glog.Errorln("Error parsing version semver: ", err)
+	}
+
+	var podInfraContainerImage string
+	switch {
+	case v1_16plus(kubernetesVersion):
+		podInfraContainerImage = imageRepository + "pause:3.1"
+
+	case v1_14plus(kubernetesVersion):
+		podInfraContainerImage = imageRepository + "pause:3.1"
+
+	case v1_13(kubernetesVersion):
+		podInfraContainerImage = imageRepository + "pause" + ArchTag(false) + "3.1"
+
+	case v1_12(kubernetesVersion):
+		podInfraContainerImage = imageRepository + "pause:3.1"
+
+	case v1_11(kubernetesVersion):
+		podInfraContainerImage = imageRepository + "pause" + ArchTag(false) + "3.1"
+
+	default:
+		podInfraContainerImage = imageRepository + "pause" + ArchTag(false) + "3.0"
+	}
+
+	return podInfraContainerImage
 }
 
 // ArchTag returns the archtag for images

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -65,6 +65,11 @@ const (
 	kubeletSystemdConfFile = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
 )
 
+const (
+	// Container runtimes
+	remoteContainerRuntime = "remote"
+)
+
 // KubeadmExtraArgsWhitelist is a whitelist of supported kubeadm params that can be supplied to kubeadm through
 // minikube's ExtraArgs parameter. The list is split into two parts - params that can be supplied as flags on the
 // command line and params that have to be inserted into the kubeadm config file. This is because of a kubeadm
@@ -556,9 +561,9 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte,
 		extraOpts["node-ip"] = k8s.NodeIP
 	}
 
-	podInfraContainerImage, _ := images.CachedImages(k8s.ImageRepository, k8s.KubernetesVersion)
-	if _, ok := extraOpts["pod-infra-container-image"]; !ok && k8s.ImageRepository != "" && podInfraContainerImage != "" {
-		extraOpts["pod-infra-container-image"] = podInfraContainerImage
+	pauseImage := images.PauseImage(k8s.ImageRepository, k8s.KubernetesVersion)
+	if _, ok := extraOpts["pod-infra-container-image"]; !ok && k8s.ImageRepository != "" && pauseImage != "" && k8s.ContainerRuntime != remoteContainerRuntime {
+		extraOpts["pod-infra-container-image"] = pauseImage
 	}
 
 	// parses a map of the feature gates for kubelet
@@ -590,7 +595,7 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte,
 
 // UpdateCluster updates the cluster
 func (k *Bootstrapper) UpdateCluster(cfg config.KubernetesConfig) error {
-	_, images := images.CachedImages(cfg.ImageRepository, cfg.KubernetesVersion)
+	images := images.CachedImages(cfg.ImageRepository, cfg.KubernetesVersion)
 	if cfg.ShouldLoadCachedImages {
 		if err := machine.LoadImages(k.c, images, constants.ImageCacheDir); err != nil {
 			out.FailureT("Unable to load cached images: {{.error}}", out.V{"error": err})

pkg/minikube/cruntime/containerd.go

@@ -17,17 +17,99 @@ limitations under the License.
 package cruntime
 
 import (
+	"bytes"
+	"encoding/base64"
 	"fmt"
+	"path"
 	"strings"
+	"text/template"
 
 	"github.com/golang/glog"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/out"
 )
 
+const (
+	// ContainerdConfFile is the path to the containerd configuration
+	containerdConfigFile     = "/etc/containerd/config.toml"
+	containerdConfigTemplate = `root = "/var/lib/containerd"
+state = "/run/containerd"
+oom_score = 0
+
+[grpc]
+  address = "/run/containerd/containerd.sock"
+  uid = 0
+  gid = 0
+  max_recv_message_size = 16777216
+  max_send_message_size = 16777216
+
+[debug]
+  address = ""
+  uid = 0
+  gid = 0
+  level = ""
+
+[metrics]
+  address = ""
+  grpc_histogram = false
+
+[cgroup]
+  path = ""
+
+[plugins]
+  [plugins.cgroups]
+    no_prometheus = false
+  [plugins.cri]
+    stream_server_address = ""
+    stream_server_port = "10010"
+    enable_selinux = false
+    sandbox_image = "{{ .PodInfraContainerImage }}"
+    stats_collect_period = 10
+    systemd_cgroup = false
+    enable_tls_streaming = false
+    max_container_log_line_size = 16384
+    [plugins.cri.containerd]
+      snapshotter = "overlayfs"
+      no_pivot = true
+      [plugins.cri.containerd.default_runtime]
+        runtime_type = "io.containerd.runtime.v1.linux"
+        runtime_engine = ""
+        runtime_root = ""
+      [plugins.cri.containerd.untrusted_workload_runtime]
+        runtime_type = ""
+        runtime_engine = ""
+        runtime_root = ""
+    [plugins.cri.cni]
+      bin_dir = "/opt/cni/bin"
+      conf_dir = "/etc/cni/net.d"
+      conf_template = ""
+    [plugins.cri.registry]
+      [plugins.cri.registry.mirrors]
+        [plugins.cri.registry.mirrors."docker.io"]
+          endpoint = ["https://registry-1.docker.io"]
+  [plugins.diff-service]
+    default = ["walking"]
+  [plugins.linux]
+    shim = "containerd-shim"
+    runtime = "runc"
+    runtime_root = ""
+    no_shim = false
+    shim_debug = false
+  [plugins.scheduler]
+    pause_threshold = 0.02
+    deletion_threshold = 0
+    mutation_threshold = 100
+    schedule_delay = "0s"
+    startup_delay = "100ms"
+`
+)
+
 // Containerd contains containerd runtime state
 type Containerd struct {
-	Socket string
-	Runner CommandRunner
+	Socket            string
+	Runner            CommandRunner
+	ImageRepository   string
+	KubernetesVersion string
 }
 
 // Name is a human readable name for containerd
@@ -79,6 +161,22 @@ func (r *Containerd) Available() error {
 	return r.Runner.Run("command -v containerd")
 }
 
+// generateContainerdConfig sets up /etc/containerd/config.toml
+func generateContainerdConfig(cr CommandRunner, imageRepository string, k8sVersion string) error {
+	cPath := containerdConfigFile
+	t, err := template.New("containerd.config.toml").Parse(containerdConfigTemplate)
+	if err != nil {
+		return err
+	}
+	pauseImage := images.PauseImage(imageRepository, k8sVersion)
+	opts := struct{ PodInfraContainerImage string }{PodInfraContainerImage: pauseImage}
+	var b bytes.Buffer
+	if err := t.Execute(&b, opts); err != nil {
+		return err
+	}
+	return cr.Run(fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", path.Dir(cPath), base64.StdEncoding.EncodeToString(b.Bytes()), cPath))
+}
+
 // Enable idempotently enables containerd on a host
 func (r *Containerd) Enable(disOthers bool) error {
 	if disOthers {
@@ -89,6 +187,9 @@ func (r *Containerd) Enable(disOthers bool) error {
 	if err := populateCRIConfig(r.Runner, r.SocketPath()); err != nil {
 		return err
 	}
+	if err := generateContainerdConfig(r.Runner, r.ImageRepository, r.KubernetesVersion); err != nil {
+		return err
+	}
 	if err := enableIPForwarding(r.Runner); err != nil {
 		return err
 	}