diff --git a/Makefile b/Makefile index fb9e39a60042..22449c5a9d9e 100755 --- a/Makefile +++ b/Makefile @@ -485,8 +485,8 @@ endif .PHONY: kic-base-image kic-base-image: ## builds the base image used for kic. - docker rmi -f $(REGISTRY)/kicbase:v0.0.1-snapshot || true - docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.1-snapshot --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) . + docker rmi -f $(REGISTRY)/kicbase:v0.0.2-snapshot || true + docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.2-snapshot --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) . diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 6cc275a858c5..7de4225c66b5 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -445,15 +445,11 @@ func displayEnviron(env []string) { } func setupKubeconfig(h *host.Host, c *cfg.MachineConfig, clusterName string) (*kubeconfig.Settings, error) { - addr := "" - var err error - if driver.IsKIC(h.DriverName) { - addr = fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(c.KubernetesConfig.NodePort))) - } else { - addr, err = h.Driver.GetURL() - if err != nil { - exit.WithError("Failed to get driver URL", err) - } + addr, err := h.Driver.GetURL() + if err != nil { + exit.WithError("Failed to get driver URL", err) + } + if !driver.IsKIC(h.DriverName) { addr = strings.Replace(addr, "tcp://", "https://", -1) addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(c.KubernetesConfig.NodePort), -1) } diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile index 9d4aed5e885f..91e81bf89186 100644 --- a/hack/images/kicbase.Dockerfile +++ b/hack/images/kicbase.Dockerfile @@ -1,10 +1,21 @@ ARG COMMIT_SHA +# for now using node image created by kind https://github.com/kubernetes-sigs/kind +# could be changed to slim ubuntu with systemd. FROM kindest/node:v1.16.2 USER root RUN apt-get update && apt-get install -y \ sudo \ dnsutils \ + openssh-server \ && apt-get clean -y +# based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile +# making SSH work for docker container +RUN mkdir /var/run/sshd +RUN echo 'root:root' |chpasswd +RUN sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config +RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +EXPOSE 22 +# Deleting all "kind" related stuff from the image. RUN rm -rf \ /var/cache/debconf/* \ /var/lib/apt/lists/* \ @@ -16,3 +27,10 @@ RUN rm -rf \ /usr/share/local/* \ /kind/bin/kubeadm /kind/bin/kubelet /kind/systemd /kind/images /kind/manifests RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt" +# for minikube ssh. to match VM using docker username +RUN adduser --disabled-password --gecos '' docker +RUN adduser docker sudo +RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +USER docker +RUN mkdir /home/docker/.ssh +USER root diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index bcd08b436c48..38a5d9cdcd7b 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -18,16 +18,20 @@ package kic import ( "fmt" + "net" "os/exec" "strconv" "strings" "github.com/docker/machine/libmachine/drivers" + "github.com/docker/machine/libmachine/ssh" "github.com/docker/machine/libmachine/state" + "github.com/golang/glog" "github.com/pkg/errors" pkgdrivers "k8s.io/minikube/pkg/drivers" "k8s.io/minikube/pkg/drivers/kic/node" "k8s.io/minikube/pkg/drivers/kic/oci" + "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/constants" ) @@ -39,7 +43,7 @@ const DefaultPodCIDR = "10.244.0.0/16" const DefaultBindIPV4 = "127.0.0.1" // BaseImage is the base image is used to spin up kic containers created by kind. -const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.1@sha256:c4ad2938877d2ae0d5b7248a5e7182ff58c0603165c3bedfe9d503e2d380a0db" +const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.2@sha256:8f531b90901721a7bd4e67ceffbbc7ee6c4292b0e6d1a9d6eb59f117d57bc4e9" // OverlayImage is the cni plugin used for overlay image, created by kind. const OverlayImage = "kindest/kindnetd:0.5.3" @@ -56,16 +60,16 @@ type Driver struct { // Config is configuration for the kic driver used by registry type Config struct { - MachineName string // maps to the container name being created - CPU int // Number of CPU cores assigned to the container - Memory int // max memory in MB - StorePath string // libmachine store path - OCIBinary string // oci tool to use (docker, podman,...) - ImageDigest string // image name with sha to use for the node - HostBindPort int // port to connect to forward from container to user's machine - Mounts []oci.Mount // mounts - PortMappings []oci.PortMapping // container port mappings - Envs map[string]string // key,value of environment variables passed to the node + MachineName string // maps to the container name being created + CPU int // Number of CPU cores assigned to the container + Memory int // max memory in MB + StorePath string // libmachine store path + OCIBinary string // oci tool to use (docker, podman,...) + ImageDigest string // image name with sha to use for the node + Mounts []oci.Mount // mounts + APIServerPort int // kubernetes api server port inside the container + PortMappings []oci.PortMapping // container port mappings + Envs map[string]string // key,value of environment variables passed to the node } // NewDriver returns a fully configured Kic driver @@ -85,27 +89,58 @@ func NewDriver(c Config) *Driver { // Create a host using the driver's config func (d *Driver) Create() error { params := node.CreateConfig{ - Name: d.NodeConfig.MachineName, - Image: d.NodeConfig.ImageDigest, - ClusterLabel: node.ClusterLabelKey + "=" + d.MachineName, - CPUs: strconv.Itoa(d.NodeConfig.CPU), - Memory: strconv.Itoa(d.NodeConfig.Memory) + "mb", - Envs: d.NodeConfig.Envs, - ExtraArgs: []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.HostBindPort)}, - OCIBinary: d.NodeConfig.OCIBinary, + Name: d.NodeConfig.MachineName, + Image: d.NodeConfig.ImageDigest, + ClusterLabel: node.ClusterLabelKey + "=" + d.MachineName, + CPUs: strconv.Itoa(d.NodeConfig.CPU), + Memory: strconv.Itoa(d.NodeConfig.Memory) + "mb", + Envs: d.NodeConfig.Envs, + ExtraArgs: []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.APIServerPort)}, + OCIBinary: d.NodeConfig.OCIBinary, + APIServerPort: d.NodeConfig.APIServerPort, } // control plane specific options params.PortMappings = append(params.PortMappings, oci.PortMapping{ - ListenAddress: "127.0.0.1", - HostPort: int32(d.NodeConfig.HostBindPort), + ListenAddress: DefaultBindIPV4, ContainerPort: constants.APIServerPort, - }) - + }, + oci.PortMapping{ + ListenAddress: DefaultBindIPV4, + ContainerPort: constants.SSHPort, + }, + ) _, err := node.CreateNode(params) if err != nil { return errors.Wrap(err, "create kic node") } + + if err := d.prepareSSH(); err != nil { + return errors.Wrap(err, "prepare kic ssh") + } + return nil +} + +// prepareSSH will generate keys and copy to the container so minikube ssh works +func (d *Driver) prepareSSH() error { + keyPath := d.GetSSHKeyPath() + glog.Infof("Creating ssh key for kic: %s...", keyPath) + if err := ssh.GenerateSSHKey(keyPath); err != nil { + return errors.Wrap(err, "generate ssh key") + } + + cmder := command.NewKICRunner(d.NodeConfig.MachineName, d.NodeConfig.OCIBinary) + f, err := assets.NewFileAsset(d.GetSSHKeyPath()+".pub", "/home/docker/.ssh/", "authorized_keys", "0644") + if err != nil { + return errors.Wrap(err, "create pubkey assetfile ") + } + if err := cmder.Copy(f); err != nil { + return errors.Wrap(err, "copying pub key") + } + if rr, err := cmder.RunCmd(exec.Command("chown", "docker:docker", "/home/docker/.ssh/authorized_keys")); err != nil { + return errors.Wrapf(err, "apply authorized_keys file ownership, output %s", rr.Output()) + } + return nil } @@ -129,17 +164,39 @@ func (d *Driver) GetIP() (string, error) { // GetSSHHostname returns hostname for use with ssh func (d *Driver) GetSSHHostname() (string, error) { - return "", fmt.Errorf("driver does not have SSHHostName") + return DefaultBindIPV4, nil } // GetSSHPort returns port for use with ssh func (d *Driver) GetSSHPort() (int, error) { - return 0, fmt.Errorf("driver does not support GetSSHPort") + p, err := oci.HostPortBinding(d.OCIBinary, d.MachineName, constants.SSHPort) + if err != nil { + return p, errors.Wrap(err, "get ssh host-port") + } + return p, nil +} + +// GetSSHUsername returns the ssh username +func (d *Driver) GetSSHUsername() string { + return "docker" +} + +// GetSSHKeyPath returns the ssh key path +func (d *Driver) GetSSHKeyPath() string { + if d.SSHKeyPath == "" { + d.SSHKeyPath = d.ResolveStorePath("id_rsa") + } + return d.SSHKeyPath } // GetURL returns ip of the container running kic control-panel func (d *Driver) GetURL() (string, error) { - return d.GetIP() + p, err := oci.HostPortBinding(d.NodeConfig.OCIBinary, d.MachineName, d.NodeConfig.APIServerPort) + url := fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(p))) + if err != nil { + return url, errors.Wrap(err, "api host port binding") + } + return url, nil } // GetState returns the state that the host is in (running, stopped, etc) diff --git a/pkg/drivers/kic/node/node.go b/pkg/drivers/kic/node/node.go index 6f1aa60c56ed..aa3f0d308479 100644 --- a/pkg/drivers/kic/node/node.go +++ b/pkg/drivers/kic/node/node.go @@ -46,17 +46,18 @@ type Node struct { } type CreateConfig struct { - Name string // used for container name and hostname - Image string // container image to use to create the node. - ClusterLabel string // label the containers we create using minikube so we can clean up - Role string // currently only role supported is control-plane - Mounts []oci.Mount // volume mounts - PortMappings []oci.PortMapping // ports to map to container from host - CPUs string // number of cpu cores assign to container - Memory string // memory (mbs) to assign to the container - Envs map[string]string // environment variables to pass to the container - ExtraArgs []string // a list of any extra option to pass to oci binary during creation time, for example --expose 8080... - OCIBinary string // docker or podman + Name string // used for container name and hostname + Image string // container image to use to create the node. + ClusterLabel string // label the containers we create using minikube so we can clean up + Role string // currently only role supported is control-plane + Mounts []oci.Mount // volume mounts + APIServerPort int // kubernetes api server port + PortMappings []oci.PortMapping // ports to map to container from host + CPUs string // number of cpu cores assign to container + Memory string // memory (mbs) to assign to the container + Envs map[string]string // environment variables to pass to the container + ExtraArgs []string // a list of any extra option to pass to oci binary during creation time, for example --expose 8080... + OCIBinary string // docker or podman } // CreateNode creates a new container node diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 2c4e899055e8..50db161f58b4 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -18,6 +18,7 @@ package oci import ( "os" + "strconv" "github.com/docker/machine/libmachine/state" "k8s.io/minikube/pkg/minikube/assets" @@ -28,7 +29,6 @@ import ( "github.com/pkg/errors" "fmt" - "net" "os/exec" "strings" "time" @@ -279,13 +279,9 @@ func UsernsRemap(ociBinary string) bool { func generatePortMappings(portMappings ...PortMapping) []string { result := make([]string, 0, len(portMappings)) for _, pm := range portMappings { - var hostPortBinding string - if pm.ListenAddress != "" { - hostPortBinding = net.JoinHostPort(pm.ListenAddress, fmt.Sprintf("%d", pm.HostPort)) - } else { - hostPortBinding = fmt.Sprintf("%d", pm.HostPort) - } - publish := fmt.Sprintf("--publish=%s:%d", hostPortBinding, pm.ContainerPort) + // let docker pick a host port by leaving it as :: + // example --publish=127.0.0.17::8443 will get a random host port for 8443 + publish := fmt.Sprintf("--publish=%s::%d", pm.ListenAddress, pm.ContainerPort) result = append(result, publish) } return result @@ -394,3 +390,23 @@ func Copy(ociBinary string, ociID string, asset assets.CopyableFile) error { } return nil } + +// HostPortBinding will return port mapping for a container using cli. +// example : HostPortBinding("docker", "minikube", "22") +// will return the docker assigned port: +// 32769, nil +// only supports TCP ports +func HostPortBinding(ociBinary string, ociID string, contPort int) (int, error) { + cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID) + out, err := cmd.CombinedOutput() + if err != nil { + return 0, errors.Wrapf(err, "getting host-bind port %q for container ID %q", contPort, ociID) + } + o := strings.Trim(string(out), "\n") + o = strings.Trim(o, "'") + p, err := strconv.Atoi(o) + if err != nil { + return p, errors.Wrapf(err, "convert host-port %q to number", p) + } + return p, nil +} diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 1f27f156cddf..aa7c571b1c6b 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -39,6 +39,7 @@ import ( "k8s.io/client-go/kubernetes" kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/minikube/pkg/drivers/kic" + "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper" @@ -464,9 +465,12 @@ func (k *Bootstrapper) applyKicOverlay(cfg config.MachineConfig) error { // clientEndpointAddr returns ip and port accessible for the kubernetes clients to talk to the cluster func (k *Bootstrapper) clientEndpointAddr(cfg config.MachineConfig) (string, int) { - if driver.IsKIC(cfg.VMDriver) { - // because docker container ip on non-linux is not accesible - return kic.DefaultBindIPV4, cfg.KubernetesConfig.NodePort + if driver.IsKIC(cfg.VMDriver) { // for kic we ask docker/podman what port it assigned to node port + p, err := oci.HostPortBinding(cfg.VMDriver, cfg.Name, cfg.KubernetesConfig.NodePort) + if err != nil { + glog.Warningf("Error getting host bind port %q for api server for %q driver: %v ", p, cfg.VMDriver, err) + } + return kic.DefaultBindIPV4, p } return cfg.KubernetesConfig.NodeIP, cfg.KubernetesConfig.NodePort } diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 18bd9a75b4f3..80dd0bf81a4a 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -65,14 +65,13 @@ type MachineConfig struct { HostOnlyNicType string // Only used by virtualbox NatNicType string // Only used by virtualbox Addons map[string]bool - NodeBindPort int32 // Only used by kic } // KubernetesConfig contains the parameters used to configure the VM Kubernetes. type KubernetesConfig struct { KubernetesVersion string NodeIP string - NodePort int + NodePort int // kubernetes api server port NodeName string APIServerName string APIServerNames []string diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index ad95e39a164a..2d6276a0993c 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -27,6 +27,8 @@ import ( ) const ( + // SSHPort is the SSH serviceport on the node vm and container + SSHPort = 22 // APIServerPort is the default API server port APIServerPort = 8443 // APIServerName is the default API server name diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index bafcf1b09361..f99143db1eb9 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -43,13 +43,13 @@ func init() { func configure(mc config.MachineConfig) interface{} { return kic.NewDriver(kic.Config{ - MachineName: mc.Name, - StorePath: localpath.MiniPath(), - ImageDigest: kic.BaseImage, - CPU: mc.CPUs, - Memory: mc.Memory, - HostBindPort: mc.KubernetesConfig.NodePort, - OCIBinary: oci.Docker, + MachineName: mc.Name, + StorePath: localpath.MiniPath(), + ImageDigest: kic.BaseImage, + CPU: mc.CPUs, + Memory: mc.Memory, + APIServerPort: mc.KubernetesConfig.NodePort, + OCIBinary: oci.Docker, }) }