Implement multi-node cluster capabilities

cmd/minikube/cmd/node.go

@@ -23,10 +23,9 @@ import (
 
 // nodeCmd represents the set of node subcommands
 var nodeCmd = &cobra.Command{
-	Use:    "node",
-	Short:  "Node operations",
-	Long:   "Operations on nodes",
-	Hidden: true, // This won't be fully functional and thus should not be documented yet
+	Use:   "node",
+	Short: "Node operations",
+	Long:  "Operations on nodes",
 	Run: func(cmd *cobra.Command, args []string) {
 		exit.UsageT("Usage: minikube node [add|start|stop|delete]")
 	},

cmd/minikube/cmd/node_add.go

@@ -23,6 +23,7 @@ import (
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/node"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -43,19 +44,25 @@ var nodeAddCmd = &cobra.Command{
 			exit.WithError("Error getting config", err)
 		}
 
-		//name := profile + strconv.Itoa(len(mc.Nodes)+1)
-		name := fmt.Sprintf("m%d", len(cc.Nodes)+1)
+		if driver.BareMetal(cc.Driver) {
+			out.ErrT(out.FailureType, "none driver does not support multi-node clusters")
+		}
+
+		name := fmt.Sprintf("m%02d", len(cc.Nodes)+1)
 
 		out.T(out.Happy, "Adding node {{.name}} to cluster {{.cluster}}", out.V{"name": name, "cluster": profile})
 
-		n, err := node.Add(cc, name, cp, worker, "", profile)
-		if err != nil {
-			exit.WithError("Error adding node to cluster", err)
+		// TODO: Deal with parameters better. Ideally we should be able to acceot any node-specific minikube start params here.
+		n := config.Node{
+			Name:              name,
+			Worker:            worker,
+			ControlPlane:      cp,
+			KubernetesVersion: cc.KubernetesConfig.KubernetesVersion,
 		}
 
-		_, err = node.Start(*cc, *n, false, nil)
+		err = node.Add(cc, n)
 		if err != nil {
-			exit.WithError("Error starting node", err)
+			exit.WithError("Error adding node to cluster", err)
 		}
 
 		out.T(out.Ready, "Successfully added {{.name}} to {{.cluster}}!", out.V{"name": name, "cluster": profile})

cmd/minikube/cmd/node_delete.go

@@ -46,7 +46,7 @@ var nodeDeleteCmd = &cobra.Command{
 
 		err = node.Delete(*cc, name)
 		if err != nil {
-			out.FatalT("Failed to delete node {{.name}}", out.V{"name": name})
+			exit.WithError("deleting node", err)
 		}
 
 		out.T(out.Deleted, "Node {{.name}} was successfully deleted.", out.V{"name": name})

cmd/minikube/cmd/node_start.go

@@ -61,10 +61,7 @@ var nodeStartCmd = &cobra.Command{
 		}
 
 		// Start it up baby
-		_, err = node.Start(*cc, *n, false, nil)
-		if err != nil {
-			out.FatalT("Failed to start node {{.name}}", out.V{"name": name})
-		}
+		node.Start(*cc, *n, nil, false)
 	},
 }
 

cmd/minikube/cmd/ssh.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
+	"k8s.io/minikube/pkg/minikube/node"
 	"k8s.io/minikube/pkg/minikube/out"
 )
 
@@ -49,12 +50,21 @@ var sshCmd = &cobra.Command{
 		if err != nil {
 			exit.WithError("Error getting config", err)
 		}
-		// TODO: allow choice of node to ssh into
-		cp, err := config.PrimaryControlPlane(cc)
-		if err != nil {
-			exit.WithError("Error getting primary control plane", err)
+		var n *config.Node
+		if nodeName == "" {
+			cp, err := config.PrimaryControlPlane(cc)
+			if err != nil {
+				exit.WithError("Getting primary control plane", err)
+			}
+			n = &cp
+		} else {
+			n, _, err = node.Retrieve(cc, nodeName)
+			if err != nil {
+				out.FailureT("Node {{.nodeName}} does not exist.", out.V{"nodeName": nodeName})
+				exit.WithError("", err)
+			}
 		}
-		host, err := machine.LoadHost(api, driver.MachineName(*cc, cp))
+		host, err := machine.LoadHost(api, driver.MachineName(*cc, *n))
 		if err != nil {
 			exit.WithError("Error getting host", err)
 		}
@@ -67,7 +77,7 @@ var sshCmd = &cobra.Command{
 			ssh.SetDefaultClient(ssh.External)
 		}
 
-		err = machine.CreateSSHShell(api, *cc, cp, args)
+		err = machine.CreateSSHShell(api, *cc, *n, args)
 		if err != nil {
 			// This is typically due to a non-zero exit code, so no need for flourish.
 			out.ErrLn("ssh: %v", err)
@@ -78,5 +88,6 @@ var sshCmd = &cobra.Command{
 }
 
 func init() {
-	sshCmd.Flags().BoolVar(&nativeSSHClient, nativeSSH, true, "Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.")
+	sshCmd.Flags().Bool(nativeSSH, true, "Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.")
+	sshCmd.Flags().StringVarP(&nodeName, "node", "n", "", "The node to ssh into. Defaults to the primary control plane.")
 }

cmd/minikube/cmd/start.go

@@ -120,6 +120,7 @@ const (
 	autoUpdate              = "auto-update-drivers"
 	hostOnlyNicType         = "host-only-nic-type"
 	natNicType              = "nat-nic-type"
+	nodes                   = "nodes"
 )
 
 var (
@@ -162,7 +163,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().String(containerRuntime, "docker", "The container runtime to be used (docker, crio, containerd).")
 	startCmd.Flags().Bool(createMount, false, "This will start the mount daemon and automatically mount files into minikube.")
 	startCmd.Flags().String(mountString, constants.DefaultMountDir+":/minikube-host", "The argument to pass the minikube mount command on start.")
-	startCmd.Flags().StringArrayVar(&node.AddonList, "addons", nil, "Enable addons. see `minikube addons list` for a list of valid addon names.")
+	startCmd.Flags().StringArrayVar(&config.AddonList, "addons", nil, "Enable addons. see `minikube addons list` for a list of valid addon names.")
 	startCmd.Flags().String(criSocket, "", "The cri socket path to be used.")
 	startCmd.Flags().String(networkPlugin, "", "The name of the network plugin.")
 	startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\".")
@@ -171,12 +172,13 @@ func initMinikubeFlags() {
 	startCmd.Flags().Bool(nativeSSH, true, "Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'.")
 	startCmd.Flags().Bool(autoUpdate, true, "If set, automatically updates drivers to the latest version. Defaults to true.")
 	startCmd.Flags().Bool(installAddons, true, "If set, install addons. Defaults to true.")
+	startCmd.Flags().IntP(nodes, "n", 1, "The number of nodes to spin up. Defaults to 1.")
 }
 
 // initKubernetesFlags inits the commandline flags for kubernetes related options
 func initKubernetesFlags() {
 	startCmd.Flags().String(kubernetesVersion, "", "The kubernetes version that the minikube VM will use (ex: v1.2.3)")
-	startCmd.Flags().Var(&node.ExtraOptions, "extra-config",
+	startCmd.Flags().Var(&config.ExtraOptions, "extra-config",
 		`A set of key=value pairs that describe configuration that may be passed to different components.
 		The key should be '.' separated, and the first part before the dot is the component to apply the configuration to.
 		Valid components are: kubelet, kubeadm, apiserver, controller-manager, etcd, proxy, scheduler
@@ -229,8 +231,8 @@ func initNetworkingFlags() {
 	startCmd.Flags().String(imageRepository, "", "Alternative image repository to pull docker images from. This can be used when you have limited access to gcr.io. Set it to \"auto\" to let minikube decide one for you. For Chinese mainland users, you may use local gcr.io mirrors such as registry.cn-hangzhou.aliyuncs.com/google_containers")
 	startCmd.Flags().String(imageMirrorCountry, "", "Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.")
 	startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.")
-	startCmd.Flags().StringArrayVar(&node.DockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)")
-	startCmd.Flags().StringArrayVar(&node.DockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)")
+	startCmd.Flags().StringArrayVar(&config.DockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)")
+	startCmd.Flags().StringArrayVar(&config.DockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)")
 }
 
 // startCmd represents the start command
@@ -313,7 +315,7 @@ func runStart(cmd *cobra.Command, args []string) {
 	}
 
 	k8sVersion := getKubernetesVersion(existing)
-	mc, n, err := generateCfgFromFlags(cmd, k8sVersion, driverName)
+	cc, n, err := generateCfgFromFlags(cmd, k8sVersion, driverName)
 	if err != nil {
 		exit.WithError("Failed to generate config", err)
 	}
@@ -324,12 +326,12 @@ func runStart(cmd *cobra.Command, args []string) {
 		return
 	}
 
-	if !driver.BareMetal(driverName) && !driver.IsKIC(driverName) {
+	if driver.IsVM(driverName) {
 		url, err := download.ISO(viper.GetStringSlice(isoURL), cmd.Flags().Changed(isoURL))
 		if err != nil {
 			exit.WithError("Failed to cache ISO", err)
 		}
-		mc.MinikubeISO = url
+		cc.MinikubeISO = url
 	}
 
 	if viper.GetBool(nativeSSH) {
@@ -338,12 +340,41 @@ func runStart(cmd *cobra.Command, args []string) {
 		ssh.SetDefaultClient(ssh.External)
 	}
 
-	kubeconfig, err := startNode(existing, mc, n)
-	if err != nil {
-		exit.WithError("Starting node", err)
+	var existingAddons map[string]bool
+	if viper.GetBool(installAddons) {
+		existingAddons = map[string]bool{}
+		if existing != nil && existing.Addons != nil {
+			existingAddons = existing.Addons
+		}
 	}
 
-	if err := showKubectlInfo(kubeconfig, k8sVersion, mc.Name); err != nil {
+	kubeconfig := node.Start(cc, n, existingAddons, true)
+
+	numNodes := viper.GetInt(nodes)
+	if numNodes == 1 && existing != nil {
+		numNodes = len(existing.Nodes)
+	}
+	if numNodes > 1 {
+		if driver.BareMetal(driverName) {
+			out.T(out.Meh, "The none driver is not compatible with multi-node clusters.")
+		} else {
+			for i := 1; i < numNodes; i++ {
+				nodeName := fmt.Sprintf("m%02d", i+1)
+				n := config.Node{
+					Name:              nodeName,
+					Worker:            true,
+					ControlPlane:      false,
+					KubernetesVersion: cc.KubernetesConfig.KubernetesVersion,
+				}
+				err := node.Add(&cc, n)
+				if err != nil {
+					exit.WithError("adding node", err)
+				}
+			}
+		}
+	}
+
+	if err := showKubectlInfo(kubeconfig, k8sVersion, cc.Name); err != nil {
 		glog.Errorf("kubectl info: %v", err)
 	}
 }
@@ -383,17 +414,6 @@ func displayEnviron(env []string) {
 	}
 }
 
-func startNode(existing *config.ClusterConfig, mc config.ClusterConfig, n config.Node) (*kubeconfig.Settings, error) {
-	var existingAddons map[string]bool
-	if viper.GetBool(installAddons) {
-		existingAddons = map[string]bool{}
-		if existing != nil && existing.Addons != nil {
-			existingAddons = existing.Addons
-		}
-	}
-	return node.Start(mc, n, true, existingAddons)
-}
-
 func showKubectlInfo(kcs *kubeconfig.Settings, k8sVersion string, machineName string) error {
 	if kcs.KeepContext {
 		out.T(out.Kubectl, "To connect to this cluster, use: kubectl --context={{.name}}", out.V{"name": kcs.ClusterName})
@@ -796,7 +816,7 @@ func validateFlags(cmd *cobra.Command, drvName string) {
 	}
 
 	// check that kubeadm extra args contain only whitelisted parameters
-	for param := range node.ExtraOptions.AsMap().Get(bsutil.Kubeadm) {
+	for param := range config.ExtraOptions.AsMap().Get(bsutil.Kubeadm) {
 		if !config.ContainsParam(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmCmdParam], param) &&
 			!config.ContainsParam(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmConfigParam], param) {
 			exit.UsageT("Sorry, the kubeadm.{{.parameter_name}} parameter is currently not supported by --extra-config", out.V{"parameter_name": param})
@@ -924,8 +944,8 @@ func createNode(cmd *cobra.Command, k8sVersion, kubeNodeName, drvName, repositor
 		HyperkitVSockPorts:      viper.GetStringSlice(vsockPorts),
 		NFSShare:                viper.GetStringSlice(nfsShare),
 		NFSSharesRoot:           viper.GetString(nfsSharesRoot),
-		DockerEnv:               node.DockerEnv,
-		DockerOpt:               node.DockerOpt,
+		DockerEnv:               config.DockerEnv,
+		DockerOpt:               config.DockerOpt,
 		InsecureRegistry:        insecureRegistry,
 		RegistryMirror:          registryMirror,
 		HostOnlyCIDR:            viper.GetString(hostOnlyCIDR),
@@ -956,7 +976,7 @@ func createNode(cmd *cobra.Command, k8sVersion, kubeNodeName, drvName, repositor
 			NetworkPlugin:          selectedNetworkPlugin,
 			ServiceCIDR:            viper.GetString(serviceCIDR),
 			ImageRepository:        repository,
-			ExtraOptions:           node.ExtraOptions,
+			ExtraOptions:           config.ExtraOptions,
 			ShouldLoadCachedImages: viper.GetBool(cacheImages),
 			EnableDefaultCNI:       selectedEnableDefaultCNI,
 		},
@@ -978,7 +998,7 @@ func setDockerProxy() {
 					continue
 				}
 			}
-			node.DockerEnv = append(node.DockerEnv, fmt.Sprintf("%s=%s", k, v))
+			config.DockerEnv = append(config.DockerEnv, fmt.Sprintf("%s=%s", k, v))
 		}
 	}
 }
@@ -990,7 +1010,7 @@ func autoSetDriverOptions(cmd *cobra.Command, drvName string) (err error) {
 	if !cmd.Flags().Changed("extra-config") && len(hints.ExtraOptions) > 0 {
 		for _, eo := range hints.ExtraOptions {
 			glog.Infof("auto setting extra-config to %q.", eo)
-			err = node.ExtraOptions.Set(eo)
+			err = config.ExtraOptions.Set(eo)
 			if err != nil {
 				err = errors.Wrapf(err, "setting extra option %s", eo)
 			}