Experimental Docker support (kic) using the Kind image

Makefile

@@ -483,6 +483,13 @@ else
 	docker build -t $(REGISTRY)/storage-provisioner-$(GOARCH):$(STORAGE_PROVISIONER_TAG) -f deploy/storage-provisioner/Dockerfile-$(GOARCH) .
 endif
 
+.PHONY: kic-base-image
+kic-base-image: ## builds the base image used for kic.
+	docker rmi -f $(REGISTRY)/kicbase:v0.0.1-snapshot || true
+	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.1-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
+
+
+
 .PHONY: push-storage-provisioner-image
 push-storage-provisioner-image: storage-provisioner-image ## Push storage-provisioner docker image using gcloud
 ifeq ($(GOARCH),amd64)
@@ -591,6 +598,7 @@ out/mkcmp:
 out/performance-monitor:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/monitor/monitor.go
 
+
 .PHONY: help
 help:
 	@printf "\033[1mAvailable targets for minikube ${VERSION}\033[21m\n"

cmd/minikube/cmd/start.go

@@ -232,7 +232,7 @@ func initNetworkingFlags() {
 	startCmd.Flags().StringSliceVar(&registryMirror, "registry-mirror", nil, "Registry mirrors to pass to the Docker daemon")
 	startCmd.Flags().String(imageRepository, "", "Alternative image repository to pull docker images from. This can be used when you have limited access to gcr.io. Set it to \"auto\" to let minikube decide one for you. For Chinese mainland users, you may use local gcr.io mirrors such as registry.cn-hangzhou.aliyuncs.com/google_containers")
 	startCmd.Flags().String(imageMirrorCountry, "", "Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.")
-	startCmd.Flags().String(serviceCIDR, pkgutil.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.")
+	startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.")
 	startCmd.Flags().StringArrayVar(&dockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)")
 	startCmd.Flags().StringArrayVar(&dockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)")
 }
@@ -430,16 +430,22 @@ func displayEnviron(env []string) {
 }
 
 func setupKubeconfig(h *host.Host, c *cfg.MachineConfig, clusterName string) (*kubeconfig.Settings, error) {
-	addr, err := h.Driver.GetURL()
-	if err != nil {
-		exit.WithError("Failed to get driver URL", err)
+	addr := ""
+	var err error
+	if driver.IsKIC(h.DriverName) {
+		addr = fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(c.KubernetesConfig.NodePort)))
+	} else {
+		addr, err = h.Driver.GetURL()
+		if err != nil {
+			exit.WithError("Failed to get driver URL", err)
+		}
+		addr = strings.Replace(addr, "tcp://", "https://", -1)
+		addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(c.KubernetesConfig.NodePort), -1)
 	}
-	addr = strings.Replace(addr, "tcp://", "https://", -1)
-	addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(c.KubernetesConfig.NodePort), -1)
+
 	if c.KubernetesConfig.APIServerName != constants.APIServerName {
 		addr = strings.Replace(addr, c.KubernetesConfig.NodeIP, c.KubernetesConfig.APIServerName, -1)
 	}
-
 	kcs := &kubeconfig.Settings{
 		ClusterName:          clusterName,
 		ClusterServerAddress: addr,
@@ -980,29 +986,35 @@ func setDockerProxy() {
 }
 
 // autoSetDriverOptions sets the options needed for specific vm-driver automatically.
-func autoSetDriverOptions(cmd *cobra.Command, drvName string) error {
+func autoSetDriverOptions(cmd *cobra.Command, drvName string) (err error) {
+	err = nil
 	hints := driver.FlagDefaults(drvName)
-	if !cmd.Flags().Changed("extra-config") && hints.ExtraOptions != "" {
-		return extraOptions.Set(hints.ExtraOptions)
+	if !cmd.Flags().Changed("extra-config") && len(hints.ExtraOptions) > 0 {
+		for _, eo := range hints.ExtraOptions {
+			glog.Infof("auto setting extra-config to %q.", eo)
+			err = extraOptions.Set(eo)
+			if err != nil {
+				err = errors.Wrapf(err, "setting extra option %s", eo)
+			}
+		}
 	}
 
 	if !cmd.Flags().Changed(cacheImages) {
 		viper.Set(cacheImages, hints.CacheImages)
 	}
 
-	// currently only used for kic
 	if !cmd.Flags().Changed(containerRuntime) && hints.ContainerRuntime != "" {
 		viper.Set(containerRuntime, hints.ContainerRuntime)
-		glog.Infof("auto set container runtime to %s for kic driver.", hints.ContainerRuntime)
-
+		glog.Infof("auto set %s to %q.", containerRuntime, hints.ContainerRuntime)
 	}
-	if !cmd.Flags().Changed("bootstrapper") && hints.Bootstrapper != "" {
+
+	if !cmd.Flags().Changed(cmdcfg.Bootstrapper) && hints.Bootstrapper != "" {
 		viper.Set(cmdcfg.Bootstrapper, hints.Bootstrapper)
-		glog.Infof("auto set bootstrapper to %s for kic driver.", hints.Bootstrapper)
+		glog.Infof("auto set %s to %q.", cmdcfg.Bootstrapper, hints.Bootstrapper)
 
 	}
 
-	return nil
+	return err
 }
 
 // prepareNone prepares the user and host for the joy of the "none" driver

hack/images/kicbase.Dockerfile

@@ -0,0 +1,18 @@
+ARG COMMIT_SHA
+FROM kindest/node:v1.16.2
+USER root
+RUN apt-get update && apt-get install -y \
+  sudo \
+  dnsutils \
+  && apt-get clean -y 
+RUN rm -rf \
+    /var/cache/debconf/* \
+    /var/lib/apt/lists/* \
+    /var/log/* \
+    /tmp/* \
+    /var/tmp/* \
+    /usr/share/doc/* \
+    /usr/share/man/* \
+    /usr/share/local/* \
+    /kind/bin/kubeadm /kind/bin/kubelet /kind/systemd /kind/images /kind/manifests
+RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"

pkg/drivers/kic/kic.go

@@ -29,9 +29,19 @@ import (
 	"k8s.io/minikube/pkg/drivers/kic/node"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/constants"
 )
 
-// https://minikube.sigs.k8s.io/docs/reference/drivers/kic/
+// DefaultPodCIDR is The CIDR to be used for pods inside the node.
+const DefaultPodCIDR = "10.244.0.0/16"
+
+// DefaultBindIPV4 is The default IP the container will bind to.
+const DefaultBindIPV4 = "127.0.0.1"
+
+// BaseImage is the base image is used to spin up kic containers
+const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.1@sha256:c4ad2938877d2ae0d5b7248a5e7182ff58c0603165c3bedfe9d503e2d380a0db"
+
+// Driver represents a kic driver https://minikube.sigs.k8s.io/docs/reference/drivers/kic/
 type Driver struct {
 	*drivers.BaseDriver
 	*pkgdrivers.CommonDriver
@@ -43,16 +53,16 @@ type Driver struct {
 
 // Config is configuration for the kic driver used by registry
 type Config struct {
-	MachineName   string            // maps to the container name being created
-	CPU           int               // Number of CPU cores assigned to the container
-	Memory        int               // max memory in MB
-	StorePath     string            // lib machine store path
-	OCIBinary     string            // oci tool to use (docker, podman,...)
-	ImageDigest   string            // image name with sha to use for the node
-	APIServerPort int32             // port to connect to forward from container to user's machine
-	Mounts        []oci.Mount       // mounts
-	PortMappings  []oci.PortMapping // container port mappings
-	Envs          map[string]string // key,value of environment variables passed to the node
+	MachineName  string            // maps to the container name being created
+	CPU          int               // Number of CPU cores assigned to the container
+	Memory       int               // max memory in MB
+	StorePath    string            // libmachine store path
+	OCIBinary    string            // oci tool to use (docker, podman,...)
+	ImageDigest  string            // image name with sha to use for the node
+	HostBindPort int               // port to connect to forward from container to user's machine
+	Mounts       []oci.Mount       // mounts
+	PortMappings []oci.PortMapping // container port mappings
+	Envs         map[string]string // key,value of environment variables passed to the node
 }
 
 // NewDriver returns a fully configured Kic driver
@@ -64,6 +74,7 @@ func NewDriver(c Config) *Driver {
 		},
 		exec:       command.NewKICRunner(c.MachineName, c.OCIBinary),
 		NodeConfig: c,
+		OCIBinary:  c.OCIBinary,
 	}
 	return d
 }
@@ -77,15 +88,15 @@ func (d *Driver) Create() error {
 		CPUs:         strconv.Itoa(d.NodeConfig.CPU),
 		Memory:       strconv.Itoa(d.NodeConfig.Memory) + "mb",
 		Envs:         d.NodeConfig.Envs,
-		ExtraArgs:    []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.APIServerPort)},
+		ExtraArgs:    []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.HostBindPort)},
 		OCIBinary:    d.NodeConfig.OCIBinary,
 	}
 
 	// control plane specific options
 	params.PortMappings = append(params.PortMappings, oci.PortMapping{
 		ListenAddress: "127.0.0.1",
-		HostPort:      d.NodeConfig.APIServerPort,
-		ContainerPort: 6443,
+		HostPort:      int32(d.NodeConfig.HostBindPort),
+		ContainerPort: constants.APIServerPort,
 	})
 
 	_, err := node.CreateNode(params)
@@ -250,32 +261,3 @@ func (d *Driver) nodeID(nameOrID string) (string, error) {
 	}
 	return string(id), err
 }
-
-func ImageForVersion(ver string) (string, error) {
-	switch ver {
-	case "v1.11.10":
-		return "medyagh/kic:v1.11.10@sha256:23bb7f5e8dd2232ec829132172e87f7b9d8de65269630989e7dac1e0fe993b74", nil
-	case "v1.12.8":
-		return "medyagh/kic:v1.12.8@sha256:c74bc5f3efe3539f6e1ad7f11bf7c09f3091c0547cb28071f4e43067053e5898", nil
-	case "v1.12.9":
-		return "medyagh/kic:v1.12.9@sha256:ff82f58e18dcb22174e8eb09dae14f7edd82d91a83c7ef19e33298d0eba6a0e3", nil
-	case "v1.12.10":
-		return "medyagh/kic:v1.12.10@sha256:2d174bae7c20698e59791e7cca9b6db234053d1a92a009d5bb124e482540c70b", nil
-	case "v1.13.6":
-		return "medyagh/kic:v1.13.6@sha256:cf63e50f824fe17b90374d38d64c5964eb9fe6b3692669e1201fcf4b29af4964", nil
-	case "v1.13.7":
-		return "medyagh/kic:v1.13.7@sha256:1a6a5e1c7534cf3012655e99df680496df9bcf0791a304adb00617d5061233fa", nil
-	case "v1.14.3":
-		return "medyagh/kic:v1.14.3@sha256:cebec21f6af23d5dfa3465b88ddf4a1acb94c2c20a0a6ff8cc1c027b0a4e2cec", nil
-	case "v1.15.0":
-		return "medyagh/kic:v1.15.0@sha256:40d433d00a2837c8be829bd3cb0576988e377472062490bce0b18281c7f85303", nil
-	case "v1.15.3":
-		return "medyagh/kic:v1.15.3@sha256:f05ce52776a86c6ead806942d424de7076af3f115b0999332981a446329e6cf1", nil
-	case "v1.16.1":
-		return "medyagh/kic:v1.16.1@sha256:e74530d22e6a04442a97a09bdbba885ad693fcc813a0d1244da32666410d1ad1", nil
-	case "v1.16.2":
-		return "medyagh/kic:v1.16.2@sha256:3374a30971bf5b0011441a227fa56ef990b76125b36ca0ab8316a3c7e4f137a3", nil
-	default:
-		return "medyagh/kic:v1.16.2@sha256:3374a30971bf5b0011441a227fa56ef990b76125b36ca0ab8316a3c7e4f137a3", nil
-	}
-}

pkg/drivers/kic/node/node.go

@@ -39,8 +39,8 @@ const (
 // Node represents a handle to a kic node
 // This struct must be created by one of: CreateControlPlane
 type Node struct {
-	// must be one of docker container ID or name
-	name      string
+	id        string         // container id
+	name      string         // container name
 	r         command.Runner // Runner
 	ociBinary string
 }
@@ -123,13 +123,15 @@ func CreateNode(p CreateConfig) (*Node, error) {
 
 // Find finds a node
 func Find(ociBinary string, name string, cmder command.Runner) (*Node, error) {
-	_, err := oci.Inspect(ociBinary, name, "{{.Id}}")
+	n, err := oci.Inspect(ociBinary, name, "{{.Id}}")
 	if err != nil {
 		return nil, fmt.Errorf("can't find node %v", err)
 	}
 	return &Node{
-		name: name,
-		r:    cmder,
+		ociBinary: ociBinary,
+		id:        n[0],
+		name:      name,
+		r:         cmder,
 	}, nil
 }
 

pkg/drivers/kic/oci/oci.go

@@ -108,7 +108,6 @@ func Pause(ociBinary string, ociID string) error {
 
 // Inspect return low-level information on containers
 func Inspect(ociBinary string, containerNameOrID, format string) ([]string, error) {
-
 	cmd := exec.Command(ociBinary, "inspect",
 		"-f", format,
 		containerNameOrID) // ... against the "node" container
@@ -359,7 +358,7 @@ func CreateContainer(ociBinary string, image string, opts ...CreateOpt) ([]strin
 	// 125
 
 	if err != nil {
-		return output, errors.Wrapf(err, "CreateContainer %v ", args)
+		return output, errors.Wrapf(err, "args: %v  output: %s ", args, output)
 	}
 	return output, nil
 }

pkg/drivers/kic/oci/types.go

@@ -16,6 +16,11 @@ limitations under the License.
 
 package oci
 
+const (
+	Docker = "docker"
+	Podman = "podman"
+)
+
 /*
 These types are from
 https://github.com/kubernetes/kubernetes/blob/063e7ff358fdc8b0916e6f39beedc0d025734cb1/pkg/kubelet/apis/cri/runtime/v1alpha2/api.pb.go#L183

pkg/minikube/bootstrapper/bootstrapper.go

@@ -70,7 +70,7 @@ func GetCachedImageList(imageRepository string, version string, bootstrapper str
 	case Kubeadm:
 		return images.Kubeadm(imageRepository, version)
 	case KIC:
-		return []string{"alpine"}, nil // for testing purpose just caching alpine for kicbs
+		return images.KIC(imageRepository, version)
 	default:
 		return []string{}, nil
 	}

pkg/minikube/bootstrapper/bsutil/binaries.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil package will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil
 
 import (

pkg/minikube/bootstrapper/bsutil/extraconfig.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil
 
 import (

pkg/minikube/bootstrapper/bsutil/featuregates.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil
 
 import (

pkg/minikube/bootstrapper/bsutil/featuregates_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil
 
 import (

pkg/minikube/bootstrapper/bsutil/files.go

@@ -14,12 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil
 
 import (
 	"path"
 
+	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/vmpath"
@@ -42,7 +43,6 @@ func ConfigFileAssets(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byt
 		assets.NewMemoryAssetTarget(kubeadm, KubeadmYamlPath, "0640"),
 		assets.NewMemoryAssetTarget(kubelet, KubeletSystemdConfFile, "0644"),
 		assets.NewMemoryAssetTarget(kubeletSvc, KubeletServiceFile, "0644"),
-		assets.NewMemoryAssetTarget(defaultCNIConfig, DefaultCNIConfigPath, "0644"),
 	}
 	// Copy the default CNI config (k8s.conf), so that kubelet can successfully
 	// start a Pod in the case a user hasn't manually installed any CNI plugin
@@ -52,3 +52,33 @@ func ConfigFileAssets(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byt
 	}
 	return fs
 }
+
+// AddAddons adds addons to list of files
+func AddAddons(files *[]assets.CopyableFile, data interface{}) error {
+	// add addons to file list
+	// custom addons
+	if err := assets.AddMinikubeDirAssets(files); err != nil {
+		return errors.Wrap(err, "adding minikube dir assets")
+	}
+	// bundled addons
+	for _, addonBundle := range assets.Addons {
+		if isEnabled, err := addonBundle.IsEnabled(); err == nil && isEnabled {
+			for _, addon := range addonBundle.Assets {
+				if addon.IsTemplate() {
+					addonFile, err := addon.Evaluate(data)
+					if err != nil {
+						return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName())
+					}
+
+					*files = append(*files, addonFile)
+				} else {
+					*files = append(*files, addon)
+				}
+			}
+		} else if err != nil {
+			return nil
+		}
+	}
+
+	return nil
+}