From a023de61a792630d47fe7f69cfa042d309a12abd Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Thu, 16 Apr 2020 13:38:17 -0700 Subject: [PATCH 01/18] defer printing enabling addons --- pkg/addons/addons.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index b39ee3f60174..0cd0a450d017 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -332,7 +332,9 @@ func Start(wg *sync.WaitGroup, cc *config.ClusterConfig, toEnable map[string]boo var awg sync.WaitGroup - out.T(out.AddonEnable, "Enabling addons: {{.addons}}", out.V{"addons": strings.Join(toEnableList, ", ")}) + defer func() { // making it show after verifications + out.T(out.AddonEnable, "Enabling addons: {{.addons}}", out.V{"addons": strings.Join(toEnableList, ", ")}) + }() for _, a := range toEnableList { awg.Add(1) go func(name string) { From 0687e2adc0f71524f2a2e042653d9f8ea9afa39c Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Thu, 16 Apr 2020 13:39:59 -0700 Subject: [PATCH 02/18] defer print addon --- pkg/addons/addons.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index 0cd0a450d017..3fce931671b4 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -332,7 +332,7 @@ func Start(wg *sync.WaitGroup, cc *config.ClusterConfig, toEnable map[string]boo var awg sync.WaitGroup - defer func() { // making it show after verifications + defer func() { // making it show after verifications( not perfect till #7613 is closed) out.T(out.AddonEnable, "Enabling addons: {{.addons}}", out.V{"addons": strings.Join(toEnableList, ", ")}) }() for _, a := range toEnableList { From 44ff50f144a58e1ec3b76df7ef38eae61af25dba Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 17 Apr 2020 11:32:41 -0700 Subject: [PATCH 03/18] add WaitForNodeReady --- .../bootstrapper/bsutil/kverify/kverify.go | 10 +-- .../bootstrapper/bsutil/kverify/node_ready.go | 64 +++++++++++++++++++ 2 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 pkg/minikube/bootstrapper/bsutil/kverify/node_ready.go diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go index f4486196c349..0bc234c7011f 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go @@ -32,7 +32,9 @@ const ( // DefaultSAWaitKey is the name used in the flags for default service account DefaultSAWaitKey = "default_sa" // AppsRunning is the name used in the flags for waiting for k8s-apps to be running - AppsRunning = "apps_running" + AppsRunningKey = "apps_running" + // NodeReadyKey is the name used in the flags for waiting for the node status to be ready + NodeReadyKey = "node_ready" ) // vars related to the --wait flag @@ -40,13 +42,13 @@ var ( // DefaultComponents is map of the the default components to wait for DefaultComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true} // NoWaitComponents is map of componets to wait for if specified 'none' or 'false' - NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunning: false} + NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunningKey: false, NodeReadyKey: false} // AllComponents is map for waiting for all components. - AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunning: true} + AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunningKey: true} // DefaultWaitList is list of all default components to wait for. only names to be used for start flags. DefaultWaitList = []string{APIServerWaitKey, SystemPodsWaitKey} // AllComponentsList list of all valid components keys to wait for. only names to be used used for start flags. - AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey, AppsRunning} + AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey, AppsRunningKey, NodeReadyKey} // AppsRunningList running list are valid k8s-app components to wait for them to be running AppsRunningList = []string{ "kube-dns", // coredns diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/node_ready.go b/pkg/minikube/bootstrapper/bsutil/kverify/node_ready.go new file mode 100644 index 000000000000..69e21ba162e5 --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/kverify/node_ready.go @@ -0,0 +1,64 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kverify verifies a running kubernetes cluster is healthy +package kverify + +import ( + "fmt" + "time" + + "github.com/golang/glog" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" +) + +// WaitForNodeReady waits for a node to be ready +func WaitForNodeReady(cs *kubernetes.Clientset, timeout time.Duration) error { + glog.Info("waiting for node to be ready ...") + start := time.Now() + defer func() { + glog.Infof("duration metric: took %s to wait for WaitForNodeReady...", time.Since(start)) + }() + checkReady := func() (bool, error) { + if time.Since(start) > timeout { + return false, fmt.Errorf("wait for node to be ready timed out") + } + ns, err := cs.CoreV1().Nodes().List(meta.ListOptions{}) + if err != nil { + glog.Infof("error listing nodes will retry: %v", err) + return false, nil + } + + for _, n := range ns.Items { + for _, c := range n.Status.Conditions { + if c.Type == v1.NodeReady && c.Status != v1.ConditionTrue { + glog.Infof("node %q has unwanted condition %q : Reason %q Message: %q. will try. ", n.Name, c.Type, c.Reason, c.Message) + return false, nil + } + } + } + return true, nil + } + if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, checkReady); err != nil { + return errors.Wrapf(err, "wait node ready") + } + return nil +} From 991c97a7389f69e71ed102a9ebd80ae9acf7d85b Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 17 Apr 2020 11:33:08 -0700 Subject: [PATCH 04/18] rename appsrunning key to match others --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 3ae20295e467..cf477560a633 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -381,7 +381,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time } } - if cfg.VerifyComponents[kverify.AppsRunning] { + if cfg.VerifyComponents[kverify.AppsRunningKey] { client, err := k.client(hostname, port) if err != nil { return errors.Wrap(err, "get k8s client") From 76ca5e0f20ee79b995925aa3fb57d5e3737c5f46 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 17 Apr 2020 11:42:20 -0700 Subject: [PATCH 05/18] add node conditions and error types --- .../bsutil/kverify/node_conditions.go | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 pkg/minikube/bootstrapper/bsutil/kverify/node_conditions.go diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/node_conditions.go b/pkg/minikube/bootstrapper/bsutil/kverify/node_conditions.go new file mode 100644 index 000000000000..6d7d0d0f71da --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/kverify/node_conditions.go @@ -0,0 +1,144 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kverify verifies a running kubernetes cluster is healthy +package kverify + +import ( + "fmt" + "time" + + "github.com/golang/glog" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// NodeCondition represents a favorable or unfavorable node condition. +type NodeCondition struct { + Type v1.NodeConditionType + Status v1.ConditionStatus + Reason string + Message string +} + +// DiskPressure detects if the condition is disk pressure +func (pc *NodeCondition) DiskPressure() bool { + return pc.Type == v1.NodeDiskPressure && pc.Status == v1.ConditionTrue +} + +// MemoryPressure detects if the condition is memory pressure +func (pc *NodeCondition) MemoryPressure() bool { + return pc.Type == v1.NodeMemoryPressure && pc.Status == v1.ConditionTrue +} + +// PIDPressure detects if the condition is PID pressure +func (pc *NodeCondition) PIDPressure() bool { + return pc.Type == v1.NodePIDPressure && pc.Status == v1.ConditionTrue +} + +// NetworkUnavailable detects if the condition is PID pressure +func (pc *NodeCondition) NetworkUnavailable() bool { + return pc.Type == v1.NodeNetworkUnavailable && pc.Status == v1.ConditionTrue +} + +const errTextFormat = "node has unwanted condition %q : Reason %q Message: %q" + +// ErrMemoryPressure is thrown when there is node memory pressure condition +type ErrMemoryPressure struct { + NodeCondition +} + +func (e *ErrMemoryPressure) Error() string { + return fmt.Sprintf(errTextFormat, e.Type, e.Reason, e.Message) +} + +// ErrDiskPressure is thrown when there is node disk pressure condition +type ErrDiskPressure struct { + NodeCondition +} + +func (e *ErrDiskPressure) Error() string { + return fmt.Sprintf(errTextFormat, e.Type, e.Reason, e.Message) +} + +// ErrPIDPressure is thrown when there is node PID pressure condition +type ErrPIDPressure struct { + NodeCondition +} + +func (e *ErrPIDPressure) Error() string { + return fmt.Sprintf(errTextFormat, e.Type, e.Reason, e.Message) +} + +// ErrNetworkNotReady is thrown when there is node condition is network not ready +type ErrNetworkNotReady struct { + NodeCondition +} + +func (e *ErrNetworkNotReady) Error() string { + return fmt.Sprintf(errTextFormat, e.Type, e.Reason, e.Message) +} + + +// NodePressure verfies that node is not under disk, memory, pid or network pressure. +func NodePressure(cs *kubernetes.Clientset) error { + glog.Info("verifying NodePressure condition ...") + start := time.Now() + defer func() { + glog.Infof("duration metric: took %s to wait for NodePressure...", time.Since(start)) + }() + + ns, err := cs.CoreV1().Nodes().List(meta.ListOptions{}) + if err != nil { + return errors.Wrap(err, "list nodes") + } + + for _, n := range ns.Items { + glog.Infof("node storage ephemeral capacity is %s", n.Status.Capacity.StorageEphemeral()) + glog.Infof("node cpu capacity is %s", n.Status.Capacity.Cpu().AsDec()) + for _, c := range n.Status.Conditions { + pc := NodeCondition{Type: c.Type, Status: c.Status, Reason: c.Reason, Message: c.Message} + if pc.DiskPressure() { + return &ErrDiskPressure{ + NodeCondition: pc, + } + } + + if pc.MemoryPressure() { + return &ErrDiskPressure{ + NodeCondition: pc, + } + } + + if pc.PIDPressure() { + return &ErrPIDPressure{ + NodeCondition: pc, + } + } + + if pc.NetworkUnavailable() { + return &ErrNetworkNotReady{ + NodeCondition: pc, + } + } + + } + } + return nil +} + From ffebe51e000c82cec297c85dd1bbb717d89ca31b Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 17 Apr 2020 12:06:46 -0700 Subject: [PATCH 06/18] add styles for verifying --- pkg/minikube/bootstrapper/bsutil/kverify/node_conditions.go | 2 -- pkg/minikube/out/style.go | 2 ++ pkg/minikube/out/style_enum.go | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/node_conditions.go b/pkg/minikube/bootstrapper/bsutil/kverify/node_conditions.go index 6d7d0d0f71da..0fb8e5c3e5d5 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/node_conditions.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/node_conditions.go @@ -94,7 +94,6 @@ func (e *ErrNetworkNotReady) Error() string { return fmt.Sprintf(errTextFormat, e.Type, e.Reason, e.Message) } - // NodePressure verfies that node is not under disk, memory, pid or network pressure. func NodePressure(cs *kubernetes.Clientset) error { glog.Info("verifying NodePressure condition ...") @@ -141,4 +140,3 @@ func NodePressure(cs *kubernetes.Clientset) error { } return nil } - diff --git a/pkg/minikube/out/style.go b/pkg/minikube/out/style.go index 1cd400d2ae24..a70e9bf489a0 100644 --- a/pkg/minikube/out/style.go +++ b/pkg/minikube/out/style.go @@ -70,6 +70,7 @@ var styles = map[StyleEnum]style{ ThumbsUp: {Prefix: "👍 "}, ThumbsDown: {Prefix: "👎 "}, Option: {Prefix: " ▪ ", LowPrefix: lowIndent}, // Indented bullet + OptionVerify: {Prefix: " 🔎 ", LowPrefix: lowIndent}, // Indented verifying icon Command: {Prefix: " ▪ ", LowPrefix: lowIndent}, // Indented bullet LogEntry: {Prefix: " "}, // Indent Deleted: {Prefix: "💀 "}, @@ -108,6 +109,7 @@ var styles = map[StyleEnum]style{ Enabling: {Prefix: "🔌 "}, Shutdown: {Prefix: "🛑 "}, Pulling: {Prefix: "🚜 "}, + HealthCheck: {Prefix: "🕵️ "}, // mac needed extra space for right tabbing Verifying: {Prefix: "🤔 "}, VerifyingNoLine: {Prefix: "🤔 ", OmitNewline: true}, Kubectl: {Prefix: "💗 "}, diff --git a/pkg/minikube/out/style_enum.go b/pkg/minikube/out/style_enum.go index 1437b26823a9..40122a48db60 100644 --- a/pkg/minikube/out/style_enum.go +++ b/pkg/minikube/out/style_enum.go @@ -43,6 +43,7 @@ const ( ThumbsUp ThumbsDown Option + OptionVerify Command LogEntry Deleted @@ -73,6 +74,7 @@ const ( Enabling Shutdown Pulling + HealthCheck Verifying VerifyingNoLine Kubectl From ad00988d50bc202b197e07922dda79c87be7a00f Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 17 Apr 2020 12:14:09 -0700 Subject: [PATCH 07/18] add node_ready for WaitForNode --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index cf477560a633..36b78f3df9ba 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -337,6 +337,8 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time return nil } + out.T(out.HealthCheck, "Verifying Kubernetes Components:") + cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: k.c}) if err != nil { return errors.Wrapf(err, "create runtme-manager %s", cfg.KubernetesConfig.ContainerRuntime) @@ -348,6 +350,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time } if cfg.VerifyComponents[kverify.APIServerWaitKey] { + out.T(out.OptionVerify, "verifying api server ...") client, err := k.client(hostname, port) if err != nil { return errors.Wrap(err, "get k8s client") @@ -362,6 +365,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time } if cfg.VerifyComponents[kverify.SystemPodsWaitKey] { + out.T(out.OptionVerify, "verifying system pods ...") client, err := k.client(hostname, port) if err != nil { return errors.Wrap(err, "get k8s client") @@ -372,6 +376,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time } if cfg.VerifyComponents[kverify.DefaultSAWaitKey] { + out.T(out.OptionVerify, "verifying default service account ...") client, err := k.client(hostname, port) if err != nil { return errors.Wrap(err, "get k8s client") @@ -382,6 +387,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time } if cfg.VerifyComponents[kverify.AppsRunningKey] { + out.T(out.OptionVerify, "verifying apps running ...") client, err := k.client(hostname, port) if err != nil { return errors.Wrap(err, "get k8s client") @@ -391,6 +397,17 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time } } + if cfg.VerifyComponents[kverify.NodeReadyKey] { + out.T(out.OptionVerify, "verifying node ready") + client, err := k.client(hostname, port) + if err != nil { + return errors.Wrap(err, "get k8s client") + } + if err := kverify.WaitForNodeReady(client, timeout); err != nil { + return errors.Wrap(err, "waiting for node to be ready") + } + } + glog.Infof("duration metric: took %s to wait for : %+v ...", time.Since(start), cfg.VerifyComponents) return nil } From 710b2899935740cac2decb52f36bb3dc07eaf8b4 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 17 Apr 2020 12:18:30 -0700 Subject: [PATCH 08/18] remove duplicate checking ShouldWait --- pkg/minikube/node/start.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 689adef274b5..4970ebdd2fa4 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -37,7 +37,6 @@ import ( cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config" "k8s.io/minikube/pkg/addons" "k8s.io/minikube/pkg/minikube/bootstrapper" - "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/cluster" "k8s.io/minikube/pkg/minikube/command" @@ -146,7 +145,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { } // Skip pre-existing, because we already waited for health - if kverify.ShouldWait(starter.Cfg.VerifyComponents) && !starter.PreExists { + if !starter.PreExists { if err := bs.WaitForNode(*starter.Cfg, *starter.Node, viper.GetDuration(waitTimeout)); err != nil { return nil, errors.Wrap(err, "Wait failed") } From 786801dd43aacd8778af91f2906e1137beff9525 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 17 Apr 2020 12:40:00 -0700 Subject: [PATCH 09/18] add required verifying node conditions --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 28 +++++-- pkg/minikube/node/advise.go | 83 ++++++++++++++++++++ 2 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 pkg/minikube/node/advise.go diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 36b78f3df9ba..367099353fec 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -332,21 +332,33 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time glog.Infof("%s is not a control plane, nothing to wait for", n.Name) return nil } - if !kverify.ShouldWait(cfg.VerifyComponents) { - glog.Infof("skip waiting for components based on config.") - return nil - } + // starting from minikube v1.10 checking for node pressure is a reuqired check out.T(out.HealthCheck, "Verifying Kubernetes Components:") + out.T(out.OptionVerify, "verifying node conditions ...") - cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: k.c}) + // TODO: #7706: for better perforamnce we could use k.client inside minikube to avoid asking for external IP:PORT + hostname, _, port, err := driver.ControlPaneEndpoint(&cfg, &n, cfg.Driver) if err != nil { - return errors.Wrapf(err, "create runtme-manager %s", cfg.KubernetesConfig.ContainerRuntime) + return errors.Wrap(err, "get control plane endpoint") } - hostname, _, port, err := driver.ControlPaneEndpoint(&cfg, &n, cfg.Driver) + client, err := k.client(hostname, port) if err != nil { - return errors.Wrap(err, "get control plane endpoint") + return errors.Wrap(err, "get k8s client") + } + if err := kverify.NodePressure(client); err != nil { + return errors.Wrap(err, "verifying node conditions") + } + + if !kverify.ShouldWait(cfg.VerifyComponents) { + glog.Infof("skip waiting for the reset of components based on config.") + return nil + } + + cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: k.c}) + if err != nil { + return errors.Wrapf(err, "create runtme-manager %s", cfg.KubernetesConfig.ContainerRuntime) } if cfg.VerifyComponents[kverify.APIServerWaitKey] { diff --git a/pkg/minikube/node/advise.go b/pkg/minikube/node/advise.go new file mode 100644 index 000000000000..ef6d8bb3fdae --- /dev/null +++ b/pkg/minikube/node/advise.go @@ -0,0 +1,83 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package node + +import ( + "runtime" + + "github.com/golang/glog" + "k8s.io/minikube/pkg/drivers/kic/oci" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" + "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/out" +) + +// adviseNodePressure will advise the user what to do with difference pressure errors based on their environment +func adviseNodePressure(err error, name string, drv string) { + if diskErr, ok := err.(*kverify.ErrDiskPressure); ok { + out.ErrLn("") + glog.Warning(diskErr) + out.WarningT("The node {{.name}} has ran out of disk space.", out.V{"name": name}) + // generic advice for all drivers + out.T(out.Tip, "Please free up disk or prune images.") + if driver.IsVM(drv) { + out.T(out.Stopped, "Consider creating a cluster with bigger disk size: `minikube start --disk SIZE_MB` ") + } else if drv == oci.Docker && runtime.GOOS != "linux" { + out.T(out.Stopped, "Consider increasing Docker Desktop's disk size.") + if runtime.GOOS == "darwin" { + out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-mac/space/"}) + } + if runtime.GOOS == "windows" { + out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-windows/"}) + } + } + out.ErrLn("") + } + + if memErr, ok := err.(*kverify.ErrMemoryPressure); ok { + out.ErrLn("") + glog.Warning(memErr) + out.WarningT("The node {{.name}} has ran out of memory.", out.V{"name": name}) + out.T(out.Tip, "Please free up memory on the cluster.") + if driver.IsVM(drv) { + out.T(out.Stopped, "Consider creating a cluster with larger memory size using `minikube start --memory SIZE_MB` ") + } else if drv == oci.Docker && runtime.GOOS != "linux" { + out.T(out.Stopped, "Consider increasing Docker Desktop's memory size.") + if runtime.GOOS == "darwin" { + out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-mac/space/"}) + } + if runtime.GOOS == "windows" { + out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-windows/"}) + } + } + out.ErrLn("") + } + + if pidErr, ok := err.(*kverify.ErrPIDPressure); ok { + glog.Warning(pidErr) + out.ErrLn("") + out.WarningT("The node {{.name}} has ran out of available PIDs.", out.V{"name": name}) + out.ErrLn("") + } + + if netErr, ok := err.(*kverify.ErrNetworkNotReady); ok { + glog.Warning(netErr) + out.ErrLn("") + out.WarningT("The node {{.name}} network is not available. Please verify network settings.", out.V{"name": name}) + out.ErrLn("") + } +} \ No newline at end of file From 100bc08dd1ceff1a9ab3d269de08fb35354b6b94 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 17 Apr 2020 12:45:31 -0700 Subject: [PATCH 10/18] add advise --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 68 +++++++++++++++- pkg/minikube/node/advise.go | 83 -------------------- 2 files changed, 67 insertions(+), 84 deletions(-) delete mode 100644 pkg/minikube/node/advise.go diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 367099353fec..e25327a456b8 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -21,6 +21,7 @@ import ( "context" "os/exec" "path" + "runtime" "sync" "fmt" @@ -41,6 +42,7 @@ import ( "k8s.io/client-go/kubernetes" kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" "k8s.io/minikube/pkg/drivers/kic" + "k8s.io/minikube/pkg/drivers/kic/oci" "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper" @@ -348,7 +350,9 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time return errors.Wrap(err, "get k8s client") } if err := kverify.NodePressure(client); err != nil { - return errors.Wrap(err, "verifying node conditions") + adviseNodePressure(err,cfg.Name,cfg.Driver) + return errors.Wrap(err,"node pressure") + } if !kverify.ShouldWait(cfg.VerifyComponents) { @@ -503,6 +507,11 @@ func (k *Bootstrapper) restartCluster(cfg config.ClusterConfig) error { return nil } + if err := kverify.NodePressure(client); err != nil { + adviseNodePressure(err,cfg.Name,cfg.Driver) + } + + if err := k.clearStaleConfigs(cfg); err != nil { return errors.Wrap(err, "clearing stale configs") } @@ -889,3 +898,60 @@ func (k *Bootstrapper) elevateKubeSystemPrivileges(cfg config.ClusterConfig) err } return nil } + + +// adviseNodePressure will advise the user what to do with difference pressure errors based on their environment +func adviseNodePressure(err error, name string, drv string) { + if diskErr, ok := err.(*kverify.ErrDiskPressure); ok { + out.ErrLn("") + glog.Warning(diskErr) + out.WarningT("The node {{.name}} has ran out of disk space.", out.V{"name": name}) + // generic advice for all drivers + out.T(out.Tip, "Please free up disk or prune images.") + if driver.IsVM(drv) { + out.T(out.Stopped, "Consider creating a cluster with bigger disk size: `minikube start --disk SIZE_MB` ") + } else if drv == oci.Docker && runtime.GOOS != "linux" { + out.T(out.Stopped, "Consider increasing Docker Desktop's disk size.") + if runtime.GOOS == "darwin" { + out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-mac/space/"}) + } + if runtime.GOOS == "windows" { + out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-windows/"}) + } + } + out.ErrLn("") + } + + if memErr, ok := err.(*kverify.ErrMemoryPressure); ok { + out.ErrLn("") + glog.Warning(memErr) + out.WarningT("The node {{.name}} has ran out of memory.", out.V{"name": name}) + out.T(out.Tip, "Please free up memory on the cluster.") + if driver.IsVM(drv) { + out.T(out.Stopped, "Consider creating a cluster with larger memory size using `minikube start --memory SIZE_MB` ") + } else if drv == oci.Docker && runtime.GOOS != "linux" { + out.T(out.Stopped, "Consider increasing Docker Desktop's memory size.") + if runtime.GOOS == "darwin" { + out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-mac/space/"}) + } + if runtime.GOOS == "windows" { + out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-windows/"}) + } + } + out.ErrLn("") + } + + if pidErr, ok := err.(*kverify.ErrPIDPressure); ok { + glog.Warning(pidErr) + out.ErrLn("") + out.WarningT("The node {{.name}} has ran out of available PIDs.", out.V{"name": name}) + out.ErrLn("") + } + + if netErr, ok := err.(*kverify.ErrNetworkNotReady); ok { + glog.Warning(netErr) + out.ErrLn("") + out.WarningT("The node {{.name}} network is not available. Please verify network settings.", out.V{"name": name}) + out.ErrLn("") + } +} \ No newline at end of file diff --git a/pkg/minikube/node/advise.go b/pkg/minikube/node/advise.go deleted file mode 100644 index ef6d8bb3fdae..000000000000 --- a/pkg/minikube/node/advise.go +++ /dev/null @@ -1,83 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package node - -import ( - "runtime" - - "github.com/golang/glog" - "k8s.io/minikube/pkg/drivers/kic/oci" - "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify" - "k8s.io/minikube/pkg/minikube/driver" - "k8s.io/minikube/pkg/minikube/out" -) - -// adviseNodePressure will advise the user what to do with difference pressure errors based on their environment -func adviseNodePressure(err error, name string, drv string) { - if diskErr, ok := err.(*kverify.ErrDiskPressure); ok { - out.ErrLn("") - glog.Warning(diskErr) - out.WarningT("The node {{.name}} has ran out of disk space.", out.V{"name": name}) - // generic advice for all drivers - out.T(out.Tip, "Please free up disk or prune images.") - if driver.IsVM(drv) { - out.T(out.Stopped, "Consider creating a cluster with bigger disk size: `minikube start --disk SIZE_MB` ") - } else if drv == oci.Docker && runtime.GOOS != "linux" { - out.T(out.Stopped, "Consider increasing Docker Desktop's disk size.") - if runtime.GOOS == "darwin" { - out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-mac/space/"}) - } - if runtime.GOOS == "windows" { - out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-windows/"}) - } - } - out.ErrLn("") - } - - if memErr, ok := err.(*kverify.ErrMemoryPressure); ok { - out.ErrLn("") - glog.Warning(memErr) - out.WarningT("The node {{.name}} has ran out of memory.", out.V{"name": name}) - out.T(out.Tip, "Please free up memory on the cluster.") - if driver.IsVM(drv) { - out.T(out.Stopped, "Consider creating a cluster with larger memory size using `minikube start --memory SIZE_MB` ") - } else if drv == oci.Docker && runtime.GOOS != "linux" { - out.T(out.Stopped, "Consider increasing Docker Desktop's memory size.") - if runtime.GOOS == "darwin" { - out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-mac/space/"}) - } - if runtime.GOOS == "windows" { - out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-windows/"}) - } - } - out.ErrLn("") - } - - if pidErr, ok := err.(*kverify.ErrPIDPressure); ok { - glog.Warning(pidErr) - out.ErrLn("") - out.WarningT("The node {{.name}} has ran out of available PIDs.", out.V{"name": name}) - out.ErrLn("") - } - - if netErr, ok := err.(*kverify.ErrNetworkNotReady); ok { - glog.Warning(netErr) - out.ErrLn("") - out.WarningT("The node {{.name}} network is not available. Please verify network settings.", out.V{"name": name}) - out.ErrLn("") - } -} \ No newline at end of file From a18e27d807b3ffdb1bf9807ff1da508dde786db8 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 17 Apr 2020 13:12:51 -0700 Subject: [PATCH 11/18] lint --- .../bootstrapper/bsutil/kverify/node_ready.go | 4 ++-- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 14 ++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/node_ready.go b/pkg/minikube/bootstrapper/bsutil/kverify/node_ready.go index 69e21ba162e5..a9c1879b6a01 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/node_ready.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/node_ready.go @@ -30,9 +30,9 @@ import ( kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) -// WaitForNodeReady waits for a node to be ready +// WaitForNodeReady waits till kube client reports node status as "ready" func WaitForNodeReady(cs *kubernetes.Clientset, timeout time.Duration) error { - glog.Info("waiting for node to be ready ...") + glog.Info("waiting for node status to be ready ...") start := time.Now() defer func() { glog.Infof("duration metric: took %s to wait for WaitForNodeReady...", time.Since(start)) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index e25327a456b8..d33c07726914 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -339,7 +339,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time out.T(out.HealthCheck, "Verifying Kubernetes Components:") out.T(out.OptionVerify, "verifying node conditions ...") - // TODO: #7706: for better perforamnce we could use k.client inside minikube to avoid asking for external IP:PORT + // TODO: #7706: for better performance we could use k.client inside minikube to avoid asking for external IP:PORT hostname, _, port, err := driver.ControlPaneEndpoint(&cfg, &n, cfg.Driver) if err != nil { return errors.Wrap(err, "get control plane endpoint") @@ -350,9 +350,9 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time return errors.Wrap(err, "get k8s client") } if err := kverify.NodePressure(client); err != nil { - adviseNodePressure(err,cfg.Name,cfg.Driver) - return errors.Wrap(err,"node pressure") - + adviseNodePressure(err, cfg.Name, cfg.Driver) + return errors.Wrap(err, "node pressure") + } if !kverify.ShouldWait(cfg.VerifyComponents) { @@ -508,10 +508,9 @@ func (k *Bootstrapper) restartCluster(cfg config.ClusterConfig) error { } if err := kverify.NodePressure(client); err != nil { - adviseNodePressure(err,cfg.Name,cfg.Driver) + adviseNodePressure(err, cfg.Name, cfg.Driver) } - if err := k.clearStaleConfigs(cfg); err != nil { return errors.Wrap(err, "clearing stale configs") } @@ -899,7 +898,6 @@ func (k *Bootstrapper) elevateKubeSystemPrivileges(cfg config.ClusterConfig) err return nil } - // adviseNodePressure will advise the user what to do with difference pressure errors based on their environment func adviseNodePressure(err error, name string, drv string) { if diskErr, ok := err.(*kverify.ErrDiskPressure); ok { @@ -954,4 +952,4 @@ func adviseNodePressure(err error, name string, drv string) { out.WarningT("The node {{.name}} network is not available. Please verify network settings.", out.V{"name": name}) out.ErrLn("") } -} \ No newline at end of file +} From 4f044273267b979d10f74f8d7a31bf70a981c58d Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 17 Apr 2020 13:50:33 -0700 Subject: [PATCH 12/18] rename icon --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 12 ++++++------ pkg/minikube/out/style.go | 2 +- pkg/minikube/out/style_enum.go | 2 +- site/content/en/docs/commands/start.md | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index d33c07726914..7c3ae9388842 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -337,7 +337,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time // starting from minikube v1.10 checking for node pressure is a reuqired check out.T(out.HealthCheck, "Verifying Kubernetes Components:") - out.T(out.OptionVerify, "verifying node conditions ...") + out.T(out.IndentVerify, "verifying node conditions ...") // TODO: #7706: for better performance we could use k.client inside minikube to avoid asking for external IP:PORT hostname, _, port, err := driver.ControlPaneEndpoint(&cfg, &n, cfg.Driver) @@ -366,7 +366,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time } if cfg.VerifyComponents[kverify.APIServerWaitKey] { - out.T(out.OptionVerify, "verifying api server ...") + out.T(out.IndentVerify, "verifying api server ...") client, err := k.client(hostname, port) if err != nil { return errors.Wrap(err, "get k8s client") @@ -381,7 +381,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time } if cfg.VerifyComponents[kverify.SystemPodsWaitKey] { - out.T(out.OptionVerify, "verifying system pods ...") + out.T(out.IndentVerify, "verifying system pods ...") client, err := k.client(hostname, port) if err != nil { return errors.Wrap(err, "get k8s client") @@ -392,7 +392,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time } if cfg.VerifyComponents[kverify.DefaultSAWaitKey] { - out.T(out.OptionVerify, "verifying default service account ...") + out.T(out.IndentVerify, "verifying default service account ...") client, err := k.client(hostname, port) if err != nil { return errors.Wrap(err, "get k8s client") @@ -403,7 +403,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time } if cfg.VerifyComponents[kverify.AppsRunningKey] { - out.T(out.OptionVerify, "verifying apps running ...") + out.T(out.IndentVerify, "verifying apps running ...") client, err := k.client(hostname, port) if err != nil { return errors.Wrap(err, "get k8s client") @@ -414,7 +414,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time } if cfg.VerifyComponents[kverify.NodeReadyKey] { - out.T(out.OptionVerify, "verifying node ready") + out.T(out.IndentVerify, "verifying node ready") client, err := k.client(hostname, port) if err != nil { return errors.Wrap(err, "get k8s client") diff --git a/pkg/minikube/out/style.go b/pkg/minikube/out/style.go index a70e9bf489a0..29629db19138 100644 --- a/pkg/minikube/out/style.go +++ b/pkg/minikube/out/style.go @@ -70,7 +70,7 @@ var styles = map[StyleEnum]style{ ThumbsUp: {Prefix: "👍 "}, ThumbsDown: {Prefix: "👎 "}, Option: {Prefix: " ▪ ", LowPrefix: lowIndent}, // Indented bullet - OptionVerify: {Prefix: " 🔎 ", LowPrefix: lowIndent}, // Indented verifying icon + IndentVerify: {Prefix: " 🔎 ", LowPrefix: lowIndent}, // Indented verifying icon, it needs extra space to make it work Command: {Prefix: " ▪ ", LowPrefix: lowIndent}, // Indented bullet LogEntry: {Prefix: " "}, // Indent Deleted: {Prefix: "💀 "}, diff --git a/pkg/minikube/out/style_enum.go b/pkg/minikube/out/style_enum.go index 40122a48db60..e47ec2572254 100644 --- a/pkg/minikube/out/style_enum.go +++ b/pkg/minikube/out/style_enum.go @@ -43,7 +43,7 @@ const ( ThumbsUp ThumbsDown Option - OptionVerify + IndentVerify Command LogEntry Deleted diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index 839be1e1673d..60128f2f653c 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -86,7 +86,7 @@ minikube start [flags] --uuid string Provide VM UUID to restore MAC address (hyperkit driver only) --vm Filter to use only VM Drivers --vm-driver driver DEPRECATED, use driver instead. - --wait strings comma separated list of kubernetes components to verify and wait for after starting a cluster. defaults to "apiserver,system_pods", available options: "apiserver,system_pods,default_sa,apps_running" . other acceptable values are 'all' or 'none', 'true' and 'false' (default [apiserver,system_pods]) + --wait strings comma separated list of kubernetes components to verify and wait for after starting a cluster. defaults to "apiserver,system_pods", available options: "apiserver,system_pods,default_sa,apps_running,node_ready" . other acceptable values are 'all' or 'none', 'true' and 'false' (default [apiserver,system_pods]) --wait-timeout duration max time to wait per Kubernetes core services to be healthy. (default 6m0s) ``` From 4e7429695de66d97e805ea55d678dcb1d69fb81e Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 17 Apr 2020 14:07:41 -0700 Subject: [PATCH 13/18] address review comments --- pkg/addons/addons.go | 2 +- .../bsutil/kverify/node_conditions.go | 4 +-- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 28 +++++++++---------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index 3fce931671b4..f394fc0681b4 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -333,7 +333,7 @@ func Start(wg *sync.WaitGroup, cc *config.ClusterConfig, toEnable map[string]boo var awg sync.WaitGroup defer func() { // making it show after verifications( not perfect till #7613 is closed) - out.T(out.AddonEnable, "Enabling addons: {{.addons}}", out.V{"addons": strings.Join(toEnableList, ", ")}) + out.T(out.AddonEnable, "Enabled addons: {{.addons}}", out.V{"addons": strings.Join(toEnableList, ", ")}) }() for _, a := range toEnableList { awg.Add(1) diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/node_conditions.go b/pkg/minikube/bootstrapper/bsutil/kverify/node_conditions.go index 0fb8e5c3e5d5..7e8e9a40a5c8 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/node_conditions.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/node_conditions.go @@ -99,7 +99,7 @@ func NodePressure(cs *kubernetes.Clientset) error { glog.Info("verifying NodePressure condition ...") start := time.Now() defer func() { - glog.Infof("duration metric: took %s to wait for NodePressure...", time.Since(start)) + glog.Infof("duration metric: took %s to run NodePressure ...", time.Since(start)) }() ns, err := cs.CoreV1().Nodes().List(meta.ListOptions{}) @@ -119,7 +119,7 @@ func NodePressure(cs *kubernetes.Clientset) error { } if pc.MemoryPressure() { - return &ErrDiskPressure{ + return &ErrMemoryPressure{ NodeCondition: pc, } } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 7c3ae9388842..e96cf0b22e43 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -335,7 +335,6 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time return nil } - // starting from minikube v1.10 checking for node pressure is a reuqired check out.T(out.HealthCheck, "Verifying Kubernetes Components:") out.T(out.IndentVerify, "verifying node conditions ...") @@ -345,18 +344,19 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time return errors.Wrap(err, "get control plane endpoint") } - client, err := k.client(hostname, port) - if err != nil { - return errors.Wrap(err, "get k8s client") - } - if err := kverify.NodePressure(client); err != nil { - adviseNodePressure(err, cfg.Name, cfg.Driver) - return errors.Wrap(err, "node pressure") - - } + defer func() { // run pressure verification after all other checks, so there be an api server to talk to. + client, err := k.client(hostname, port) + if err != nil { + return errors.Wrap(err, "get k8s client") + } + if err := kverify.NodePressure(client); err != nil { + adviseNodePressure(err, cfg.Name, cfg.Driver) + return errors.Wrap(err, "node pressure") + } + }() if !kverify.ShouldWait(cfg.VerifyComponents) { - glog.Infof("skip waiting for the reset of components based on config.") + glog.Infof("skip waiting for components based on config.") return nil } @@ -907,9 +907,9 @@ func adviseNodePressure(err error, name string, drv string) { // generic advice for all drivers out.T(out.Tip, "Please free up disk or prune images.") if driver.IsVM(drv) { - out.T(out.Stopped, "Consider creating a cluster with bigger disk size: `minikube start --disk SIZE_MB` ") + out.T(out.Stopped, "Please create a cluster with bigger disk size: `minikube start --disk SIZE_MB` ") } else if drv == oci.Docker && runtime.GOOS != "linux" { - out.T(out.Stopped, "Consider increasing Docker Desktop's disk size.") + out.T(out.Stopped, "Please increse Desktop's disk size.") if runtime.GOOS == "darwin" { out.T(out.Documentation, "Documentation: {{.url}}", out.V{"url": "https://docs.docker.com/docker-for-mac/space/"}) } @@ -924,7 +924,7 @@ func adviseNodePressure(err error, name string, drv string) { out.ErrLn("") glog.Warning(memErr) out.WarningT("The node {{.name}} has ran out of memory.", out.V{"name": name}) - out.T(out.Tip, "Please free up memory on the cluster.") + out.T(out.Tip, "Check if you have unneccessary pods running by running 'kubectl get po -A") if driver.IsVM(drv) { out.T(out.Stopped, "Consider creating a cluster with larger memory size using `minikube start --memory SIZE_MB` ") } else if drv == oci.Docker && runtime.GOOS != "linux" { From cd99a47ad251bd97b115597cf1618582bb8206fa Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 17 Apr 2020 14:10:13 -0700 Subject: [PATCH 14/18] address review comments --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index e96cf0b22e43..8c526e4b0610 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -344,7 +344,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time return errors.Wrap(err, "get control plane endpoint") } - defer func() { // run pressure verification after all other checks, so there be an api server to talk to. + defer func() error { // run pressure verification after all other checks, so there be an api server to talk to. client, err := k.client(hostname, port) if err != nil { return errors.Wrap(err, "get k8s client") @@ -353,6 +353,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time adviseNodePressure(err, cfg.Name, cfg.Driver) return errors.Wrap(err, "node pressure") } + return nil }() if !kverify.ShouldWait(cfg.VerifyComponents) { From 775a91cc1a28d90098d8fe5f999b49726968b821 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 17 Apr 2020 14:35:36 -0700 Subject: [PATCH 15/18] make node pressure happen after all other waits --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 8c526e4b0610..26735a792289 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -327,7 +327,7 @@ func (k *Bootstrapper) client(ip string, port int) (*kubernetes.Clientset, error } // WaitForNode blocks until the node appears to be healthy -func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, timeout time.Duration) error { +func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, timeout time.Duration) (waitErr error) { start := time.Now() if !n.ControlPlane { @@ -344,21 +344,20 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time return errors.Wrap(err, "get control plane endpoint") } - defer func() error { // run pressure verification after all other checks, so there be an api server to talk to. + defer func() { // run pressure verification after all other checks, so there be an api server to talk to. client, err := k.client(hostname, port) if err != nil { - return errors.Wrap(err, "get k8s client") + waitErr = errors.Wrap(err, "get k8s client") } if err := kverify.NodePressure(client); err != nil { adviseNodePressure(err, cfg.Name, cfg.Driver) - return errors.Wrap(err, "node pressure") + waitErr = errors.Wrap(err, "node pressure") } - return nil }() if !kverify.ShouldWait(cfg.VerifyComponents) { glog.Infof("skip waiting for components based on config.") - return nil + return waitErr } cr, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: k.c}) @@ -426,7 +425,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time } glog.Infof("duration metric: took %s to wait for : %+v ...", time.Since(start), cfg.VerifyComponents) - return nil + return waitErr } // needsReset returns whether or not the cluster needs to be reconfigured @@ -919,13 +918,14 @@ func adviseNodePressure(err error, name string, drv string) { } } out.ErrLn("") + return } if memErr, ok := err.(*kverify.ErrMemoryPressure); ok { out.ErrLn("") glog.Warning(memErr) out.WarningT("The node {{.name}} has ran out of memory.", out.V{"name": name}) - out.T(out.Tip, "Check if you have unneccessary pods running by running 'kubectl get po -A") + out.T(out.Tip, "Check if you have unnecessary pods running by running 'kubectl get po -A") if driver.IsVM(drv) { out.T(out.Stopped, "Consider creating a cluster with larger memory size using `minikube start --memory SIZE_MB` ") } else if drv == oci.Docker && runtime.GOOS != "linux" { @@ -938,6 +938,7 @@ func adviseNodePressure(err error, name string, drv string) { } } out.ErrLn("") + return } if pidErr, ok := err.(*kverify.ErrPIDPressure); ok { @@ -945,6 +946,7 @@ func adviseNodePressure(err error, name string, drv string) { out.ErrLn("") out.WarningT("The node {{.name}} has ran out of available PIDs.", out.V{"name": name}) out.ErrLn("") + return } if netErr, ok := err.(*kverify.ErrNetworkNotReady); ok { @@ -952,5 +954,6 @@ func adviseNodePressure(err error, name string, drv string) { out.ErrLn("") out.WarningT("The node {{.name}} network is not available. Please verify network settings.", out.V{"name": name}) out.ErrLn("") + return } } From 70cbc679836a55177546d83f7c1bfeb6b08a131f Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 17 Apr 2020 15:21:34 -0700 Subject: [PATCH 16/18] add todo comment --- pkg/minikube/node/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 4970ebdd2fa4..004ce8923448 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -144,7 +144,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { prepareNone() } - // Skip pre-existing, because we already waited for health + // TODO: existing cluster should wait for health #7597 if !starter.PreExists { if err := bs.WaitForNode(*starter.Cfg, *starter.Node, viper.GetDuration(waitTimeout)); err != nil { return nil, errors.Wrap(err, "Wait failed") From 5e36969ab96a603563effca10da57327e1f14a43 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 17 Apr 2020 16:15:12 -0700 Subject: [PATCH 17/18] run NodePressure after api server is up --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 26735a792289..29f219eeac3a 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -507,9 +507,6 @@ func (k *Bootstrapper) restartCluster(cfg config.ClusterConfig) error { return nil } - if err := kverify.NodePressure(client); err != nil { - adviseNodePressure(err, cfg.Name, cfg.Driver) - } if err := k.clearStaleConfigs(cfg); err != nil { return errors.Wrap(err, "clearing stale configs") @@ -554,6 +551,10 @@ func (k *Bootstrapper) restartCluster(cfg config.ClusterConfig) error { return errors.Wrap(err, "system pods") } + if err := kverify.NodePressure(client); err != nil { + adviseNodePressure(err, cfg.Name, cfg.Driver) + } + // This can fail during upgrades if the old pods have not shut down yet addonPhase := func() error { _, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s phase addon all --config %s", baseCmd, conf))) From 21ad12fb448640c524fb99c8fe972da7f14297fa Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Fri, 17 Apr 2020 16:50:12 -0700 Subject: [PATCH 18/18] lint --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 29f219eeac3a..625e580fbae9 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -507,7 +507,6 @@ func (k *Bootstrapper) restartCluster(cfg config.ClusterConfig) error { return nil } - if err := k.clearStaleConfigs(cfg); err != nil { return errors.Wrap(err, "clearing stale configs") }