Implement multi-node cluster capabilities

cmd/minikube/cmd/delete.go

@@ -221,14 +221,20 @@ func deleteProfile(profile *pkg_config.Profile) error {
 	}
 
 	if err == nil && driver.BareMetal(cc.Driver) {
-		if err := uninstallKubernetes(api, profile.Name, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper)); err != nil {
-			deletionError, ok := err.(DeletionError)
-			if ok {
-				delErr := profileDeletionErr(profile.Name, fmt.Sprintf("%v", err))
-				deletionError.Err = delErr
-				return deletionError
+		var e error
+		for _, n := range cc.Nodes {
+			if err := uninstallKubernetes(api, profile.Name, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper), n.Name); err != nil {
+				deletionError, ok := err.(DeletionError)
+				if ok {
+					delErr := profileDeletionErr(profile.Name, fmt.Sprintf("%v", err))
+					deletionError.Err = delErr
+					e = deletionError
+				}
+				e = err
 			}
-			return err
+		}
+		if e != nil {
+			return e
 		}
 	}
 
@@ -302,9 +308,9 @@ func profileDeletionErr(profileName string, additionalInfo string) error {
 	return fmt.Errorf("error deleting profile \"%s\": %s", profileName, additionalInfo)
 }
 
-func uninstallKubernetes(api libmachine.API, profile string, kc pkg_config.KubernetesConfig, bsName string) error {
+func uninstallKubernetes(api libmachine.API, profile string, kc pkg_config.KubernetesConfig, bsName string, nodeName string) error {
 	out.T(out.Resetting, "Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...", out.V{"kubernetes_version": kc.KubernetesVersion, "bootstrapper_name": bsName})
-	clusterBootstrapper, err := cluster.Bootstrapper(api, bsName)
+	clusterBootstrapper, err := cluster.Bootstrapper(api, bsName, nodeName)
 	if err != nil {
 		return DeletionError{Err: fmt.Errorf("unable to get bootstrapper: %v", err), Errtype: Fatal}
 	}

cmd/minikube/cmd/logs.go

@@ -67,7 +67,7 @@ var logsCmd = &cobra.Command{
 		if err != nil {
 			exit.WithError("command runner", err)
 		}
-		bs, err := cluster.Bootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
+		bs, err := cluster.Bootstrapper(api, viper.GetString(cmdcfg.Bootstrapper), viper.GetString(config.MachineProfile))
 		if err != nil {
 			exit.WithError("Error getting cluster bootstrapper", err)
 		}

cmd/minikube/cmd/node.go

@@ -23,10 +23,9 @@ import (
 
 // nodeCmd represents the set of node subcommands
 var nodeCmd = &cobra.Command{
-	Use:    "node",
-	Short:  "Node operations",
-	Long:   "Operations on nodes",
-	Hidden: true, // This won't be fully functional and thus should not be documented yet
+	Use:   "node",
+	Short: "Node operations",
+	Long:  "Operations on nodes",
 	Run: func(cmd *cobra.Command, args []string) {
 		exit.UsageT("Usage: minikube node [add|start|stop|delete]")
 	},

cmd/minikube/cmd/node_add.go

@@ -47,18 +47,17 @@ var nodeAddCmd = &cobra.Command{
 		if nodeName == "" {
 			name = profile + strconv.Itoa(len(mc.Nodes)+1)
 		}
+		_, _, err = node.Retrieve(mc, name)
+		if err == nil {
+			exit.WithCodeT(100, "{{.nodeName}} already exists in cluster {{.cluster}}. Choose a different name.", out.V{"nodeName": name, "cluster": mc.Name})
+		}
 		out.T(out.Happy, "Adding node {{.name}} to cluster {{.cluster}}", out.V{"name": name, "cluster": profile})
 
-		n, err := node.Add(mc, name, cp, worker, "", profile)
+		err = node.Add(mc, name, cp, worker, "", profile)
 		if err != nil {
 			exit.WithError("Error adding node to cluster", err)
 		}
 
-		_, err = node.Start(*mc, *n, false, nil)
-		if err != nil {
-			exit.WithError("Error starting node", err)
-		}
-
 		out.T(out.Ready, "Successfully added {{.name}} to {{.cluster}}!", out.V{"name": name, "cluster": profile})
 	},
 }

cmd/minikube/cmd/node_delete.go

@@ -46,7 +46,7 @@ var nodeDeleteCmd = &cobra.Command{
 
 		err = node.Delete(*cc, name)
 		if err != nil {
-			out.FatalT("Failed to delete node {{.name}}", out.V{"name": name})
+			exit.WithError("deleting node", err)
 		}
 
 		out.T(out.Deleted, "Node {{.name}} was successfully deleted.", out.V{"name": name})

cmd/minikube/cmd/node_start.go

@@ -61,7 +61,7 @@ var nodeStartCmd = &cobra.Command{
 		}
 
 		// Start it up baby
-		_, err = node.Start(*cc, *n, false, nil)
+		err = node.Start(*cc, *n, nil)
 		if err != nil {
 			out.FatalT("Failed to start node {{.name}}", out.V{"name": name})
 		}

cmd/minikube/cmd/start.go

@@ -118,6 +118,7 @@ const (
 	autoUpdate              = "auto-update-drivers"
 	hostOnlyNicType         = "host-only-nic-type"
 	natNicType              = "nat-nic-type"
+	nodes                   = "nodes"
 )
 
 var (
@@ -160,7 +161,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().String(containerRuntime, "docker", "The container runtime to be used (docker, crio, containerd).")
 	startCmd.Flags().Bool(createMount, false, "This will start the mount daemon and automatically mount files into minikube.")
 	startCmd.Flags().String(mountString, constants.DefaultMountDir+":/minikube-host", "The argument to pass the minikube mount command on start.")
-	startCmd.Flags().StringArrayVar(&node.AddonList, "addons", nil, "Enable addons. see `minikube addons list` for a list of valid addon names.")
+	startCmd.Flags().StringArrayVar(&config.AddonList, "addons", nil, "Enable addons. see `minikube addons list` for a list of valid addon names.")
 	startCmd.Flags().String(criSocket, "", "The cri socket path to be used.")
 	startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.")
 	startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".")
@@ -169,12 +170,13 @@ func initMinikubeFlags() {
 	startCmd.Flags().Bool(nativeSSH, true, "Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.")
 	startCmd.Flags().Bool(autoUpdate, true, "If set, automatically updates drivers to the latest version. Defaults to true.")
 	startCmd.Flags().Bool(installAddons, true, "If set, install addons. Defaults to true.")
+	startCmd.Flags().IntP(nodes, "n", 1, "The number of nodes to spin up. Defaults to 1.")
 }
 
 // initKubernetesFlags inits the commandline flags for kubernetes related options
 func initKubernetesFlags() {
 	startCmd.Flags().String(kubernetesVersion, "", "The kubernetes version that the minikube VM will use (ex: v1.2.3)")
-	startCmd.Flags().Var(&node.ExtraOptions, "extra-config",
+	startCmd.Flags().Var(&config.ExtraOptions, "extra-config",
 		`A set of key=value pairs that describe configuration that may be passed to different components.
 		The key should be '.' separated, and the first part before the dot is the component to apply the configuration to.
 		Valid components are: kubelet, kubeadm, apiserver, controller-manager, etcd, proxy, scheduler
@@ -226,8 +228,8 @@ func initNetworkingFlags() {
 	startCmd.Flags().String(imageRepository, "", "Alternative image repository to pull docker images from. This can be used when you have limited access to gcr.io. Set it to \"auto\" to let minikube decide one for you. For Chinese mainland users, you may use local gcr.io mirrors such as registry.cn-hangzhou.aliyuncs.com/google_containers")
 	startCmd.Flags().String(imageMirrorCountry, "", "Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.")
 	startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.")
-	startCmd.Flags().StringArrayVar(&node.DockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)")
-	startCmd.Flags().StringArrayVar(&node.DockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)")
+	startCmd.Flags().StringArrayVar(&config.DockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)")
+	startCmd.Flags().StringArrayVar(&config.DockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)")
 }
 
 // startCmd represents the start command
@@ -335,14 +337,29 @@ func runStart(cmd *cobra.Command, args []string) {
 			existingAddons = existing.Addons
 		}
 	}
-	kubeconfig, err := node.Start(mc, n, true, existingAddons)
+
+	// Abstraction leakage alert: startHost requires the config to be saved, to satistfy pkg/provision/buildroot.
+	// Hence, saveConfig must be called before startHost, and again afterwards when we know the IP.
+	if err := config.SaveProfile(viper.GetString(config.MachineProfile), &mc); err != nil {
+		exit.WithError("Failed to save config", err)
+	}
+
+	kubeconfig, err := cluster.InitialSetup(mc, n, existingAddons)
 	if err != nil {
 		exit.WithError("Starting node", err)
 	}
 
 	if err := showKubectlInfo(kubeconfig, k8sVersion, mc.Name); err != nil {
 		glog.Errorf("kubectl info: %v", err)
 	}
+
+	numNodes := viper.GetInt(nodes)
+	if numNodes > 1 {
+		for i := 0; i < numNodes-1; i++ {
+			nodeName := fmt.Sprintf("%s%d", n.Name, i+1)
+			node.Add(&mc, nodeName, false, true, "", "")
+		}
+	}
 }
 
 func updateDriver(driverName string) {
@@ -691,7 +708,7 @@ func validateFlags(cmd *cobra.Command, drvName string) {
 	validateCPUCount(driver.BareMetal(drvName))
 
 	// check that kubeadm extra args contain only whitelisted parameters
-	for param := range node.ExtraOptions.AsMap().Get(bsutil.Kubeadm) {
+	for param := range config.ExtraOptions.AsMap().Get(bsutil.Kubeadm) {
 		if !config.ContainsParam(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmCmdParam], param) &&
 			!config.ContainsParam(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmConfigParam], param) {
 			exit.UsageT("Sorry, the kubeadm.{{.parameter_name}} parameter is currently not supported by --extra-config", out.V{"parameter_name": param})
@@ -791,8 +808,8 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 		HyperkitVSockPorts:      viper.GetStringSlice(vsockPorts),
 		NFSShare:                viper.GetStringSlice(nfsShare),
 		NFSSharesRoot:           viper.GetString(nfsSharesRoot),
-		DockerEnv:               node.DockerEnv,
-		DockerOpt:               node.DockerOpt,
+		DockerEnv:               config.DockerEnv,
+		DockerOpt:               config.DockerOpt,
 		InsecureRegistry:        insecureRegistry,
 		RegistryMirror:          registryMirror,
 		HostOnlyCIDR:            viper.GetString(hostOnlyCIDR),
@@ -824,7 +841,7 @@ func generateCfgFromFlags(cmd *cobra.Command, k8sVersion string, drvName string)
 			NetworkPlugin:          selectedNetworkPlugin,
 			ServiceCIDR:            viper.GetString(serviceCIDR),
 			ImageRepository:        repository,
-			ExtraOptions:           node.ExtraOptions,
+			ExtraOptions:           config.ExtraOptions,
 			ShouldLoadCachedImages: viper.GetBool(cacheImages),
 			EnableDefaultCNI:       selectedEnableDefaultCNI,
 		},
@@ -846,7 +863,7 @@ func setDockerProxy() {
 					continue
 				}
 			}
-			node.DockerEnv = append(node.DockerEnv, fmt.Sprintf("%s=%s", k, v))
+			config.DockerEnv = append(config.DockerEnv, fmt.Sprintf("%s=%s", k, v))
 		}
 	}
 }
@@ -858,7 +875,7 @@ func autoSetDriverOptions(cmd *cobra.Command, drvName string) (err error) {
 	if !cmd.Flags().Changed("extra-config") && len(hints.ExtraOptions) > 0 {
 		for _, eo := range hints.ExtraOptions {
 			glog.Infof("auto setting extra-config to %q.", eo)
-			err = node.ExtraOptions.Set(eo)
+			err = config.ExtraOptions.Set(eo)
 			if err != nil {
 				err = errors.Wrapf(err, "setting extra option %s", eo)
 			}

pkg/drivers/hyperkit/driver.go

@@ -65,6 +65,7 @@ type Driver struct {
 	UUID           string
 	VpnKitSock     string
 	VSockPorts     []string
+	ClusterName    string
 }
 
 // NewDriver creates a new driver for a host
@@ -199,7 +200,7 @@ func (d *Driver) Restart() error {
 }
 
 func (d *Driver) createHost() (*hyperkit.HyperKit, error) {
-	stateDir := filepath.Join(d.StorePath, "machines", d.MachineName)
+	stateDir := filepath.Join(d.StorePath, "machines", d.ClusterName, d.MachineName)
 	h, err := hyperkit.New("", d.VpnKitSock, stateDir)
 	if err != nil {
 		return nil, errors.Wrap(err, "new-ing Hyperkit")
@@ -519,6 +520,7 @@ func (d *Driver) sendSignal(s os.Signal) error {
 func (d *Driver) getPid() int {
 	pidPath := d.ResolveStorePath(machineFileName)
 
+	log.Debugf("PIDPATH=%s", pidPath)
 	f, err := os.Open(pidPath)
 	if err != nil {
 		log.Warnf("Error reading pid file: %v", err)

pkg/minikube/bootstrapper/bootstrapper.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/cruntime"
 )
 
 // LogOptions are options to be passed to LogCommands
@@ -39,6 +40,9 @@ type Bootstrapper interface {
 	UpdateCluster(config.ClusterConfig) error
 	DeleteCluster(config.KubernetesConfig) error
 	WaitForCluster(config.ClusterConfig, time.Duration) error
+	JoinCluster(config.ClusterConfig, config.Node, string) error
+	UpdateNode(config.ClusterConfig, config.Node, cruntime.Manager) error
+	GenerateToken(config.KubernetesConfig) (string, error)
 	// LogCommands returns a map of log type to a command which will display that log.
 	LogCommands(LogOptions) map[string]string
 	SetupCerts(config.KubernetesConfig, config.Node) error

pkg/minikube/bootstrapper/bsutil/ops.go

@@ -47,7 +47,7 @@ func AdjustResourceLimits(c command.Runner) error {
 	return nil
 }
 
-// ExistingConfig checks if there are config files from possible previous kubernets cluster
+// ExistingConfig checks if there are config files from possible previous kubernetes cluster
 func ExistingConfig(c command.Runner) error {
 	args := append([]string{"ls"}, expectedRemoteArtifacts...)
 	_, err := c.RunCmd(exec.Command("sudo", args...))

pkg/minikube/bootstrapper/certs.go

@@ -121,8 +121,10 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig, n config.Node)
 		return errors.Wrap(err, "encoding kubeconfig")
 	}
 
-	kubeCfgFile := assets.NewMemoryAsset(data, vmpath.GuestPersistentDir, "kubeconfig", "0644")
-	copyableFiles = append(copyableFiles, kubeCfgFile)
+	if n.ControlPlane {
+		kubeCfgFile := assets.NewMemoryAsset(data, vmpath.GuestPersistentDir, "kubeconfig", "0644")
+		copyableFiles = append(copyableFiles, kubeCfgFile)
+	}
 
 	for _, f := range copyableFiles {
 		if err := cmd.Copy(f); err != nil {

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -36,7 +36,6 @@ import (
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
-	"github.com/spf13/viper"
 	"k8s.io/client-go/kubernetes"
 	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/minikube/pkg/drivers/kic"
@@ -65,8 +64,7 @@ type Bootstrapper struct {
 }
 
 // NewBootstrapper creates a new kubeadm.Bootstrapper
-func NewBootstrapper(api libmachine.API) (*Bootstrapper, error) {
-	name := viper.GetString(config.MachineProfile)
+func NewBootstrapper(api libmachine.API, name string) (*Bootstrapper, error) {
 	h, err := api.Load(name)
 	if err != nil {
 		return nil, errors.Wrap(err, "getting api client")
@@ -360,6 +358,43 @@ func (k *Bootstrapper) restartCluster(cfg config.ClusterConfig) error {
 	return nil
 }
 
+// JoinCluster adds a node to an existing cluster
+func (k *Bootstrapper) JoinCluster(cc config.ClusterConfig, n config.Node, joinCmd string) error {
+	start := time.Now()
+	glog.Infof("JoinCluster: %+v", cc)
+	defer func() {
+		glog.Infof("JoinCluster complete in %s", time.Since(start))
+	}()
+
+	// Join the master by specifying its token
+	joinCmd = fmt.Sprintf("%s --v=10 --node-name=%s", joinCmd, n.Name)
+	fmt.Println(joinCmd)
+	out, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", joinCmd))
+	if err != nil {
+		return errors.Wrapf(err, "cmd failed: %s\n%s\n", joinCmd, out)
+	}
+
+	if _, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", "sudo systemctl daemon-reload && sudo systemctl enable kubelet && sudo systemctl start kubelet")); err != nil {
+		return errors.Wrap(err, "starting kubelet")
+	}
+
+	return nil
+}
+
+// GenerateToken creates a token and returns the appropriate kubeadm join command to run
+func (k *Bootstrapper) GenerateToken(k8s config.KubernetesConfig) (string, error) {
+	tokenCmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s token create --print-join-command --ttl=0", bsutil.InvokeKubeadm(k8s.KubernetesVersion)))
+	r, err := k.c.RunCmd(tokenCmd)
+	if err != nil {
+		return "", errors.Wrap(err, "generating bootstrap token")
+	}
+	joinCmd := r.Stdout.String()
+	joinCmd = strings.Replace(joinCmd, "kubeadm", bsutil.InvokeKubeadm(k8s.KubernetesVersion), 1)
+	joinCmd = fmt.Sprintf("%s --ignore-preflight-errors=all", strings.TrimSpace(joinCmd))
+
+	return joinCmd, nil
+}
+
 // DeleteCluster removes the components that were started earlier
 func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error {
 	version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion)
@@ -384,7 +419,7 @@ func (k *Bootstrapper) SetupCerts(k8s config.KubernetesConfig, n config.Node) er
 	return bootstrapper.SetupCerts(k.c, k8s, n)
 }
 
-// UpdateCluster updates the cluster
+// UpdateCluster updates the cluster.
 func (k *Bootstrapper) UpdateCluster(cfg config.ClusterConfig) error {
 	images, err := images.Kubeadm(cfg.KubernetesConfig.ImageRepository, cfg.KubernetesConfig.KubernetesVersion)
 	if err != nil {
@@ -402,14 +437,24 @@ func (k *Bootstrapper) UpdateCluster(cfg config.ClusterConfig) error {
 		return errors.Wrap(err, "runtime")
 	}
 
-	// TODO: multiple nodes
-	kubeadmCfg, err := bsutil.GenerateKubeadmYAML(cfg, r, cfg.Nodes[0])
+	for _, n := range cfg.Nodes {
+		err := k.UpdateNode(cfg, n, r)
+		if err != nil {
+			return errors.Wrap(err, "updating node")
+		}
+	}
+
+	return nil
+}
+
+// UpdateNode updates a node.
+func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cruntime.Manager) error {
+	kubeadmCfg, err := bsutil.GenerateKubeadmYAML(cfg, r, n)
 	if err != nil {
 		return errors.Wrap(err, "generating kubeadm cfg")
 	}
 
-	// TODO: multiple nodes
-	kubeletCfg, err := bsutil.NewKubeletConfig(cfg, cfg.Nodes[0], r)
+	kubeletCfg, err := bsutil.NewKubeletConfig(cfg, n, r)
 	if err != nil {
 		return errors.Wrap(err, "generating kubelet config")
 	}