kic driver: add multiple profiles and ssh

Makefile

@@ -485,8 +485,8 @@ endif
 
 .PHONY: kic-base-image
 kic-base-image: ## builds the base image used for kic.
-	docker rmi -f $(REGISTRY)/kicbase:v0.0.1-snapshot || true
-	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.1-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
+	docker rmi -f $(REGISTRY)/kicbase:v0.0.2-snapshot || true
+	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.2-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
 
 
 

cmd/minikube/cmd/start.go

@@ -445,15 +445,11 @@ func displayEnviron(env []string) {
 }
 
 func setupKubeconfig(h *host.Host, c *cfg.MachineConfig, clusterName string) (*kubeconfig.Settings, error) {
-	addr := ""
-	var err error
-	if driver.IsKIC(h.DriverName) {
-		addr = fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(c.KubernetesConfig.NodePort)))
-	} else {
-		addr, err = h.Driver.GetURL()
-		if err != nil {
-			exit.WithError("Failed to get driver URL", err)
-		}
+	addr, err := h.Driver.GetURL()
+	if err != nil {
+		exit.WithError("Failed to get driver URL", err)
+	}
+	if !driver.IsKIC(h.DriverName) {
 		addr = strings.Replace(addr, "tcp://", "https://", -1)
 		addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(c.KubernetesConfig.NodePort), -1)
 	}

hack/images/kicbase.Dockerfile

@@ -1,10 +1,21 @@
 ARG COMMIT_SHA
+# for now using node image created by kind https://github.com/kubernetes-sigs/kind
+# could be changed to slim ubuntu with systemd. 
 FROM kindest/node:v1.16.2
 USER root
 RUN apt-get update && apt-get install -y \
   sudo \
   dnsutils \
+  openssh-server \
   && apt-get clean -y 
+# based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile
+# making SSH work for docker container 
+RUN mkdir /var/run/sshd
+RUN echo 'root:root' |chpasswd
+RUN sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config
+RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
+EXPOSE 22
+# Deleting all "kind" related stuff from the image.
 RUN rm -rf \
     /var/cache/debconf/* \
     /var/lib/apt/lists/* \
@@ -16,3 +27,10 @@ RUN rm -rf \
     /usr/share/local/* \
     /kind/bin/kubeadm /kind/bin/kubelet /kind/systemd /kind/images /kind/manifests
 RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"
+# for minikube ssh. to match VM using docker username
+RUN adduser --disabled-password --gecos '' docker
+RUN adduser docker sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER docker
+RUN mkdir /home/docker/.ssh
+USER root

pkg/drivers/kic/kic.go

@@ -18,16 +18,20 @@ package kic
 
 import (
 	"fmt"
+	"net"
 	"os/exec"
 	"strconv"
 	"strings"
 
 	"github.com/docker/machine/libmachine/drivers"
+	"github.com/docker/machine/libmachine/ssh"
 	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
 	"k8s.io/minikube/pkg/drivers/kic/node"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/constants"
 )
@@ -39,7 +43,7 @@ const DefaultPodCIDR = "10.244.0.0/16"
 const DefaultBindIPV4 = "127.0.0.1"
 
 // BaseImage is the base image is used to spin up kic containers created by kind.
-const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.1@sha256:c4ad2938877d2ae0d5b7248a5e7182ff58c0603165c3bedfe9d503e2d380a0db"
+const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.2@sha256:8f531b90901721a7bd4e67ceffbbc7ee6c4292b0e6d1a9d6eb59f117d57bc4e9"
 
 // OverlayImage is the cni plugin used for overlay image, created by kind.
 const OverlayImage = "kindest/kindnetd:0.5.3"
@@ -56,16 +60,16 @@ type Driver struct {
 
 // Config is configuration for the kic driver used by registry
 type Config struct {
-	MachineName  string            // maps to the container name being created
-	CPU          int               // Number of CPU cores assigned to the container
-	Memory       int               // max memory in MB
-	StorePath    string            // libmachine store path
-	OCIBinary    string            // oci tool to use (docker, podman,...)
-	ImageDigest  string            // image name with sha to use for the node
-	HostBindPort int               // port to connect to forward from container to user's machine
-	Mounts       []oci.Mount       // mounts
-	PortMappings []oci.PortMapping // container port mappings
-	Envs         map[string]string // key,value of environment variables passed to the node
+	MachineName   string            // maps to the container name being created
+	CPU           int               // Number of CPU cores assigned to the container
+	Memory        int               // max memory in MB
+	StorePath     string            // libmachine store path
+	OCIBinary     string            // oci tool to use (docker, podman,...)
+	ImageDigest   string            // image name with sha to use for the node
+	Mounts        []oci.Mount       // mounts
+	APIServerPort int               // kubernetes api server port inside the container
+	PortMappings  []oci.PortMapping // container port mappings
+	Envs          map[string]string // key,value of environment variables passed to the node
 }
 
 // NewDriver returns a fully configured Kic driver
@@ -85,27 +89,58 @@ func NewDriver(c Config) *Driver {
 // Create a host using the driver's config
 func (d *Driver) Create() error {
 	params := node.CreateConfig{
-		Name:         d.NodeConfig.MachineName,
-		Image:        d.NodeConfig.ImageDigest,
-		ClusterLabel: node.ClusterLabelKey + "=" + d.MachineName,
-		CPUs:         strconv.Itoa(d.NodeConfig.CPU),
-		Memory:       strconv.Itoa(d.NodeConfig.Memory) + "mb",
-		Envs:         d.NodeConfig.Envs,
-		ExtraArgs:    []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.HostBindPort)},
-		OCIBinary:    d.NodeConfig.OCIBinary,
+		Name:          d.NodeConfig.MachineName,
+		Image:         d.NodeConfig.ImageDigest,
+		ClusterLabel:  node.ClusterLabelKey + "=" + d.MachineName,
+		CPUs:          strconv.Itoa(d.NodeConfig.CPU),
+		Memory:        strconv.Itoa(d.NodeConfig.Memory) + "mb",
+		Envs:          d.NodeConfig.Envs,
+		ExtraArgs:     []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.APIServerPort)},
+		OCIBinary:     d.NodeConfig.OCIBinary,
+		APIServerPort: d.NodeConfig.APIServerPort,
 	}
 
 	// control plane specific options
 	params.PortMappings = append(params.PortMappings, oci.PortMapping{
-		ListenAddress: "127.0.0.1",
-		HostPort:      int32(d.NodeConfig.HostBindPort),
+		ListenAddress: DefaultBindIPV4,
 		ContainerPort: constants.APIServerPort,
-	})
-
+	},
+		oci.PortMapping{
+			ListenAddress: DefaultBindIPV4,
+			ContainerPort: constants.SSHPort,
+		},
+	)
 	_, err := node.CreateNode(params)
 	if err != nil {
 		return errors.Wrap(err, "create kic node")
 	}
+
+	if err := d.prepareSSH(); err != nil {
+		return errors.Wrap(err, "prepare kic ssh")
+	}
+	return nil
+}
+
+// prepareSSH will generate keys and copy to the container so minikube ssh works
+func (d *Driver) prepareSSH() error {
+	keyPath := d.GetSSHKeyPath()
+	glog.Infof("Creating ssh key for kic: %s...", keyPath)
+	if err := ssh.GenerateSSHKey(keyPath); err != nil {
+		return errors.Wrap(err, "generate ssh key")
+	}
+
+	cmder := command.NewKICRunner(d.NodeConfig.MachineName, d.NodeConfig.OCIBinary)
+	f, err := assets.NewFileAsset(d.GetSSHKeyPath()+".pub", "/home/docker/.ssh/", "authorized_keys", "0644")
+	if err != nil {
+		return errors.Wrap(err, "create pubkey assetfile ")
+	}
+	if err := cmder.Copy(f); err != nil {
+		return errors.Wrap(err, "copying pub key")
+	}
+	if rr, err := cmder.RunCmd(exec.Command("chown", "docker:docker", "/home/docker/.ssh/authorized_keys")); err != nil {
+		return errors.Wrapf(err, "apply authorized_keys file ownership, output %s", rr.Output())
+	}
+
 	return nil
 }
 
@@ -129,17 +164,39 @@ func (d *Driver) GetIP() (string, error) {
 
 // GetSSHHostname returns hostname for use with ssh
 func (d *Driver) GetSSHHostname() (string, error) {
-	return "", fmt.Errorf("driver does not have SSHHostName")
+	return DefaultBindIPV4, nil
 }
 
 // GetSSHPort returns port for use with ssh
 func (d *Driver) GetSSHPort() (int, error) {
-	return 0, fmt.Errorf("driver does not support GetSSHPort")
+	p, err := oci.HostPortBinding(d.OCIBinary, d.MachineName, constants.SSHPort)
+	if err != nil {
+		return p, errors.Wrap(err, "get ssh host-port")
+	}
+	return p, nil
+}
+
+// GetSSHUsername returns the ssh username
+func (d *Driver) GetSSHUsername() string {
+	return "docker"
+}
+
+// GetSSHKeyPath returns the ssh key path
+func (d *Driver) GetSSHKeyPath() string {
+	if d.SSHKeyPath == "" {
+		d.SSHKeyPath = d.ResolveStorePath("id_rsa")
+	}
+	return d.SSHKeyPath
 }
 
 // GetURL returns ip of the container running kic control-panel
 func (d *Driver) GetURL() (string, error) {
-	return d.GetIP()
+	p, err := oci.HostPortBinding(d.NodeConfig.OCIBinary, d.MachineName, d.NodeConfig.APIServerPort)
+	url := fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(p)))
+	if err != nil {
+		return url, errors.Wrap(err, "api host port binding")
+	}
+	return url, nil
 }
 
 // GetState returns the state that the host is in (running, stopped, etc)

pkg/drivers/kic/node/node.go

@@ -46,17 +46,18 @@ type Node struct {
 }
 
 type CreateConfig struct {
-	Name         string            // used for container name and hostname
-	Image        string            // container image to use to create the node.
-	ClusterLabel string            // label the containers we create using minikube so we can clean up
-	Role         string            // currently only role supported is control-plane
-	Mounts       []oci.Mount       // volume mounts
-	PortMappings []oci.PortMapping // ports to map to container from host
-	CPUs         string            // number of cpu cores assign to container
-	Memory       string            // memory (mbs) to assign to the container
-	Envs         map[string]string // environment variables to pass to the container
-	ExtraArgs    []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
-	OCIBinary    string            // docker or podman
+	Name          string            // used for container name and hostname
+	Image         string            // container image to use to create the node.
+	ClusterLabel  string            // label the containers we create using minikube so we can clean up
+	Role          string            // currently only role supported is control-plane
+	Mounts        []oci.Mount       // volume mounts
+	APIServerPort int               // kubernetes api server port
+	PortMappings  []oci.PortMapping // ports to map to container from host
+	CPUs          string            // number of cpu cores assign to container
+	Memory        string            // memory (mbs) to assign to the container
+	Envs          map[string]string // environment variables to pass to the container
+	ExtraArgs     []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
+	OCIBinary     string            // docker or podman
 }
 
 // CreateNode creates a new container node

pkg/drivers/kic/oci/oci.go

@@ -18,6 +18,7 @@ package oci
 
 import (
 	"os"
+	"strconv"
 
 	"github.com/docker/machine/libmachine/state"
 	"k8s.io/minikube/pkg/minikube/assets"
@@ -28,7 +29,6 @@ import (
 	"github.com/pkg/errors"
 
 	"fmt"
-	"net"
 	"os/exec"
 	"strings"
 	"time"
@@ -279,13 +279,9 @@ func UsernsRemap(ociBinary string) bool {
 func generatePortMappings(portMappings ...PortMapping) []string {
 	result := make([]string, 0, len(portMappings))
 	for _, pm := range portMappings {
-		var hostPortBinding string
-		if pm.ListenAddress != "" {
-			hostPortBinding = net.JoinHostPort(pm.ListenAddress, fmt.Sprintf("%d", pm.HostPort))
-		} else {
-			hostPortBinding = fmt.Sprintf("%d", pm.HostPort)
-		}
-		publish := fmt.Sprintf("--publish=%s:%d", hostPortBinding, pm.ContainerPort)
+		// let docker pick a host port by leaving it as ::
+		// example --publish=127.0.0.17::8443 will get a random host port for 8443
+		publish := fmt.Sprintf("--publish=%s::%d", pm.ListenAddress, pm.ContainerPort)
 		result = append(result, publish)
 	}
 	return result
@@ -394,3 +390,23 @@ func Copy(ociBinary string, ociID string, asset assets.CopyableFile) error {
 	}
 	return nil
 }
+
+// HostPortBinding will return port mapping for a container using cli.
+// example : HostPortBinding("docker", "minikube", "22")
+// will return the docker assigned port:
+// 32769, nil
+// only supports TCP ports
+func HostPortBinding(ociBinary string, ociID string, contPort int) (int, error) {
+	cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return 0, errors.Wrapf(err, "getting host-bind port %q for container ID %q", contPort, ociID)
+	}
+	o := strings.Trim(string(out), "\n")
+	o = strings.Trim(o, "'")
+	p, err := strconv.Atoi(o)
+	if err != nil {
+		return p, errors.Wrapf(err, "convert host-port %q to number", p)
+	}
+	return p, nil
+}

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -39,6 +39,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/minikube/pkg/drivers/kic"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
@@ -464,9 +465,12 @@ func (k *Bootstrapper) applyKicOverlay(cfg config.MachineConfig) error {
 
 // clientEndpointAddr returns ip and port accessible for the kubernetes clients to talk to the cluster
 func (k *Bootstrapper) clientEndpointAddr(cfg config.MachineConfig) (string, int) {
-	if driver.IsKIC(cfg.VMDriver) {
-		// because docker container ip on non-linux is not accesible
-		return kic.DefaultBindIPV4, cfg.KubernetesConfig.NodePort
+	if driver.IsKIC(cfg.VMDriver) { // for kic we ask docker/podman what port it assigned to node port
+		p, err := oci.HostPortBinding(cfg.VMDriver, cfg.Name, cfg.KubernetesConfig.NodePort)
+		if err != nil {
+			glog.Warningf("Error getting host bind port %q for api server for %q driver: %v ", p, cfg.VMDriver, err)
+		}
+		return kic.DefaultBindIPV4, p
 	}
 	return cfg.KubernetesConfig.NodeIP, cfg.KubernetesConfig.NodePort
 }

pkg/minikube/config/types.go

@@ -65,14 +65,13 @@ type MachineConfig struct {
 	HostOnlyNicType     string // Only used by virtualbox
 	NatNicType          string // Only used by virtualbox
 	Addons              map[string]bool
-	NodeBindPort        int32 // Only used by kic
 }
 
 // KubernetesConfig contains the parameters used to configure the VM Kubernetes.
 type KubernetesConfig struct {
 	KubernetesVersion string
 	NodeIP            string
-	NodePort          int
+	NodePort          int // kubernetes api server port
 	NodeName          string
 	APIServerName     string
 	APIServerNames    []string

pkg/minikube/constants/constants.go

@@ -27,6 +27,8 @@ import (
 )
 
 const (
+	// SSHPort is the SSH serviceport on the node vm and container
+	SSHPort = 22
 	// APIServerPort is the default API server port
 	APIServerPort = 8443
 	// APIServerName is the default API server name

pkg/minikube/registry/drvs/docker/docker.go

@@ -43,13 +43,13 @@ func init() {
 
 func configure(mc config.MachineConfig) interface{} {
 	return kic.NewDriver(kic.Config{
-		MachineName:  mc.Name,
-		StorePath:    localpath.MiniPath(),
-		ImageDigest:  kic.BaseImage,
-		CPU:          mc.CPUs,
-		Memory:       mc.Memory,
-		HostBindPort: mc.KubernetesConfig.NodePort,
-		OCIBinary:    oci.Docker,
+		MachineName:   mc.Name,
+		StorePath:     localpath.MiniPath(),
+		ImageDigest:   kic.BaseImage,
+		CPU:           mc.CPUs,
+		Memory:        mc.Memory,
+		APIServerPort: mc.KubernetesConfig.NodePort,
+		OCIBinary:     oci.Docker,
 	})
 
 }