From c037a97392b5b7c2ce625e6f6c2216ed903edf15 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 7 May 2020 12:59:55 -0700 Subject: [PATCH 01/18] Restart kubeadm on soft `minikube start` This should resolve a race condition in which the expected node IP and the ip passed into kubelet are different. This should fix the embed-certs integration test which has been flaky for a while. --- pkg/drivers/kic/kic.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 041e7449b3a6..db5b8451d5f2 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -294,6 +294,10 @@ func (d *Driver) Restart() error { if err = d.Start(); err != nil { return fmt.Errorf("start during restart %v", err) } + cr := command.NewExecRunner() // using exec runner for interacting with docker/podman daemon + if _, err := cr.RunCmd(exec.Command("sudo", "systemctl", "restart", "kubelet")); err != nil { + return errors.Wrap(err, "restarting kubelet") + } return nil } From a353d564dd187833b3f0085067473ec78961fc1f Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 7 May 2020 13:03:07 -0700 Subject: [PATCH 02/18] Add comments --- pkg/drivers/kic/kic.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index db5b8451d5f2..648e6a57f33e 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -294,6 +294,10 @@ func (d *Driver) Restart() error { if err = d.Start(); err != nil { return fmt.Errorf("start during restart %v", err) } + // we need to restart kubelet to ensure that it picks up the correct node ip + // there seems to be a race condition between when the ip is updated + // on node restarts and kubelet starting up + glog.Infof("Restarting kubelet...") cr := command.NewExecRunner() // using exec runner for interacting with docker/podman daemon if _, err := cr.RunCmd(exec.Command("sudo", "systemctl", "restart", "kubelet")); err != nil { return errors.Wrap(err, "restarting kubelet") From 163ab8840888214bc98f1b4359ac54fd1d6122a8 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 7 May 2020 13:22:09 -0700 Subject: [PATCH 03/18] Restart kubelet in a function that is called on soft start --- pkg/drivers/kic/kic.go | 9 --------- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 7 +++++-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 648e6a57f33e..257e78cba067 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -294,16 +294,7 @@ func (d *Driver) Restart() error { if err = d.Start(); err != nil { return fmt.Errorf("start during restart %v", err) } - // we need to restart kubelet to ensure that it picks up the correct node ip - // there seems to be a race condition between when the ip is updated - // on node restarts and kubelet starting up - glog.Infof("Restarting kubelet...") - cr := command.NewExecRunner() // using exec runner for interacting with docker/podman daemon - if _, err := cr.RunCmd(exec.Command("sudo", "systemctl", "restart", "kubelet")); err != nil { - return errors.Wrap(err, "restarting kubelet") - } return nil - } // Start an already created kic container diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 5408744837fd..e3dad31b1e24 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -812,8 +812,11 @@ func startKubeletIfRequired(runner command.Runner, sm sysinit.Manager) error { checkCmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("pgrep kubelet && diff -u %s %s.new && diff -u %s %s.new", svc, svc, conf, conf)) if _, err := runner.RunCmd(checkCmd); err == nil { - glog.Infof("kubelet is already running with the right configs") - return nil + // we need to restart kubelet to ensure that it picks up the correct node ip + // there seems to be a race condition between when the ip is updated + // on node restarts and kubelet starting up + glog.Infof("Restarting kubelet...") + return sysinit.New(runner).Restart("kubelet") } startCmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo cp %s.new %s && sudo cp %s.new %s", svc, svc, conf, conf)) From 8ac42815aa4e6855b51b9b74a473788b3c5b8f9c Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 7 May 2020 13:40:26 -0700 Subject: [PATCH 04/18] Restart kubelet if soft start --- pkg/minikube/node/start.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 689530fda5c4..1faaf489df64 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -53,6 +53,7 @@ import ( "k8s.io/minikube/pkg/minikube/mustload" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/proxy" + "k8s.io/minikube/pkg/minikube/sysinit" "k8s.io/minikube/pkg/util" "k8s.io/minikube/pkg/util/retry" ) @@ -180,6 +181,16 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { } } + if starter.PreExists { + // we need to restart kubelet to ensure that it picks up the correct node ip + // there seems to be a race condition between when the ip is updated + // on node restarts and kubelet starting up + glog.Infof("Restarting kubelet...") + if err := sysinit.New(starter.Runner).Restart("kubelet"); err != nil { + return nil, errors.Wrap(err, "restarting kubelet") + } + } + wg.Wait() // Write enabled addons to the config before completion From 69a39c100c08fa6662d92d94bc00f4a8c57337f7 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 7 May 2020 13:41:37 -0700 Subject: [PATCH 05/18] Add kic --- pkg/minikube/node/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 1faaf489df64..0ba96c84d094 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -181,7 +181,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { } } - if starter.PreExists { + if starter.PreExists && driver.IsKIC(starter.Host.DriverName) { // we need to restart kubelet to ensure that it picks up the correct node ip // there seems to be a race condition between when the ip is updated // on node restarts and kubelet starting up From ced1914feddcad42eed35b2b3d1e5a96258646cf Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 7 May 2020 13:43:03 -0700 Subject: [PATCH 06/18] remove unnecessary code --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index e3dad31b1e24..5408744837fd 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -812,11 +812,8 @@ func startKubeletIfRequired(runner command.Runner, sm sysinit.Manager) error { checkCmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("pgrep kubelet && diff -u %s %s.new && diff -u %s %s.new", svc, svc, conf, conf)) if _, err := runner.RunCmd(checkCmd); err == nil { - // we need to restart kubelet to ensure that it picks up the correct node ip - // there seems to be a race condition between when the ip is updated - // on node restarts and kubelet starting up - glog.Infof("Restarting kubelet...") - return sysinit.New(runner).Restart("kubelet") + glog.Infof("kubelet is already running with the right configs") + return nil } startCmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo cp %s.new %s && sudo cp %s.new %s", svc, svc, conf, conf)) From 80960945af4f1e5ba703621f60646f9234fa3fc8 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 7 May 2020 13:45:34 -0700 Subject: [PATCH 07/18] move into separate function --- pkg/minikube/node/start.go | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 0ba96c84d094..f4031029b370 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -181,14 +181,8 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { } } - if starter.PreExists && driver.IsKIC(starter.Host.DriverName) { - // we need to restart kubelet to ensure that it picks up the correct node ip - // there seems to be a race condition between when the ip is updated - // on node restarts and kubelet starting up - glog.Infof("Restarting kubelet...") - if err := sysinit.New(starter.Runner).Restart("kubelet"); err != nil { - return nil, errors.Wrap(err, "restarting kubelet") - } + if err := restartKubeletIfNeeded(starter); err != nil { + return nil, errors.Wrap(err, "restarting kubelet") } wg.Wait() @@ -197,6 +191,23 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { return kcs, config.Write(viper.GetString(config.ProfileName), starter.Cfg) } +func restartKubeletIfNeeded(starter Starter) error { + if !starter.PreExists { + return nil + } + if !driver.IsKIC(starter.Host.DriverName) { + return nil + } + // on soft start and kic, we need to restart kubelet to ensure that it picks up the correct node ip + // there seems to be a race condition between when the ip is updated + // on node restarts and kubelet starting up + glog.Infof("Restarting kubelet...") + if err := sysinit.New(starter.Runner).Restart("kubelet"); err != nil { + return errors.Wrap(err, "restarting kubelet") + } + return nil +} + // Provision provisions the machine/container for the node func Provision(cc *config.ClusterConfig, n *config.Node, apiServer bool) (command.Runner, bool, libmachine.API, *host.Host, error) { From fce75622cf2c89200c84414f95aaaad0f96fc18c Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 7 May 2020 14:18:20 -0700 Subject: [PATCH 08/18] Add start/stop test for plain old docker --- test/integration/start_stop_delete_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index b6764ea967b5..88191cce24d2 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -71,6 +71,9 @@ func TestStartStop(t *testing.T) { "--disable-driver-mounts", "--extra-config=kubeadm.ignore-preflight-errors=SystemVerification", }}, + {"docker", constants.DefaultKubernetesVersion, []string{ + "--container-runtime=docker", + }}, {"embed-certs", constants.DefaultKubernetesVersion, []string{ "--embed-certs", }}, From 934866735125d43fb2881ec4a408678e6b038e27 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 7 May 2020 17:00:37 -0700 Subject: [PATCH 09/18] Add host alias back after restarting kubelet --- pkg/minikube/node/start.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index f4031029b370..94494b6b6e5c 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -205,6 +205,13 @@ func restartKubeletIfNeeded(starter Starter) error { if err := sysinit.New(starter.Runner).Restart("kubelet"); err != nil { return errors.Wrap(err, "restarting kubelet") } + cp, err := config.PrimaryControlPlane(starter.Cfg) + if err != nil { + return errors.Wrap(err, "control plane") + } + if err := machine.AddHostAlias(starter.Runner, constants.ControlPlaneAlias, net.ParseIP(cp.IP)); err != nil { + return errors.Wrap(err, "adding host alias") + } return nil } From 73d15fc2753dd68bd93649fccaac69883eaffca3 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 8 May 2020 12:30:06 -0700 Subject: [PATCH 10/18] Start kubelet after host aliases have been set in /etc/hosts kubelet manages the value of `/etc/hosts` within running containers (e.g. kube-controller-manager). It takes this file from the host, but this causes a race condition between: 1. Us updating /etc/hosts with aliases on IP changes 1. kubelet starting k8s with current value of /etc/hosts This change removes automatically starting kubelet when systemd comes up. Instead, we start it immediately after making our updates to `/etc/hosts` --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 13 ++------ pkg/minikube/node/start.go | 35 ++------------------ 2 files changed, 5 insertions(+), 43 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 5408744837fd..c0cc0b24a31c 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -775,7 +775,7 @@ func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cru return errors.Wrap(err, "host alias") } - if err := startKubeletIfRequired(k.c, sm); err != nil { + if err := startKubelet(k.c, sm); err != nil { return errors.Wrap(err, "reload") } @@ -801,7 +801,7 @@ func copyFiles(runner command.Runner, files []assets.CopyableFile) error { return nil } -func startKubeletIfRequired(runner command.Runner, sm sysinit.Manager) error { +func startKubelet(runner command.Runner, sm sysinit.Manager) error { now := time.Now() defer func() { glog.Infof("reloadKubelet took %s", time.Since(now)) @@ -810,20 +810,11 @@ func startKubeletIfRequired(runner command.Runner, sm sysinit.Manager) error { svc := bsutil.KubeletServiceFile conf := bsutil.KubeletSystemdConfFile - checkCmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("pgrep kubelet && diff -u %s %s.new && diff -u %s %s.new", svc, svc, conf, conf)) - if _, err := runner.RunCmd(checkCmd); err == nil { - glog.Infof("kubelet is already running with the right configs") - return nil - } - startCmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo cp %s.new %s && sudo cp %s.new %s", svc, svc, conf, conf)) if _, err := runner.RunCmd(startCmd); err != nil { return errors.Wrap(err, "starting kubelet") } - if err := sm.Enable("kubelet"); err != nil { - return err - } return sm.Start("kubelet") } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 94494b6b6e5c..3f3a63318ea3 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -53,7 +53,6 @@ import ( "k8s.io/minikube/pkg/minikube/mustload" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/proxy" - "k8s.io/minikube/pkg/minikube/sysinit" "k8s.io/minikube/pkg/util" "k8s.io/minikube/pkg/util/retry" ) @@ -76,14 +75,14 @@ type Starter struct { ExistingAddons map[string]bool } -// Start spins up a guest and starts the kubernetes node. +// Start spins up a guest and starts the Kubernetes node. func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { // wait for preloaded tarball to finish downloading before configuring runtimes waitCacheRequiredImages(&cacheGroup) sv, err := util.ParseKubernetesVersion(starter.Node.KubernetesVersion) if err != nil { - return nil, errors.Wrap(err, "Failed to parse kubernetes version") + return nil, errors.Wrap(err, "Failed to parse Kubernetes version") } // configure the runtime (docker, containerd, crio) @@ -181,40 +180,12 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { } } - if err := restartKubeletIfNeeded(starter); err != nil { - return nil, errors.Wrap(err, "restarting kubelet") - } - wg.Wait() // Write enabled addons to the config before completion return kcs, config.Write(viper.GetString(config.ProfileName), starter.Cfg) } -func restartKubeletIfNeeded(starter Starter) error { - if !starter.PreExists { - return nil - } - if !driver.IsKIC(starter.Host.DriverName) { - return nil - } - // on soft start and kic, we need to restart kubelet to ensure that it picks up the correct node ip - // there seems to be a race condition between when the ip is updated - // on node restarts and kubelet starting up - glog.Infof("Restarting kubelet...") - if err := sysinit.New(starter.Runner).Restart("kubelet"); err != nil { - return errors.Wrap(err, "restarting kubelet") - } - cp, err := config.PrimaryControlPlane(starter.Cfg) - if err != nil { - return errors.Wrap(err, "control plane") - } - if err := machine.AddHostAlias(starter.Runner, constants.ControlPlaneAlias, net.ParseIP(cp.IP)); err != nil { - return errors.Wrap(err, "adding host alias") - } - return nil -} - // Provision provisions the machine/container for the node func Provision(cc *config.ClusterConfig, n *config.Node, apiServer bool) (command.Runner, bool, libmachine.API, *host.Host, error) { @@ -467,7 +438,7 @@ func trySSH(h *host.Host, ip string) error { d := net.Dialer{Timeout: 3 * time.Second} conn, err := d.Dial("tcp", sshAddr) if err != nil { - out.WarningT("Unable to verify SSH connectivity: {{.error}}. Will retry...", out.V{"error": err}) + glog.Warningf("dial failed (will retry): %v", err) return err } _ = conn.Close() From 826fb2bbf31bf25ee192d9e9b2f33e878527ab8b Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 8 May 2020 12:32:27 -0700 Subject: [PATCH 11/18] Restart kubelet after updating config files This makes sure the kubelet picks up the correct IP --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index c0cc0b24a31c..aee406d4bc99 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -542,6 +542,7 @@ func (k *Bootstrapper) restartCluster(cfg config.ClusterConfig) error { cmds := []string{ fmt.Sprintf("%s phase certs all --config %s", baseCmd, conf), fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, conf), + fmt.Sprintf("sudo systemctl restart kubelet"), fmt.Sprintf("%s phase %s all --config %s", baseCmd, controlPlane, conf), fmt.Sprintf("%s phase etcd local --config %s", baseCmd, conf), } From 0fa24990c9cfa0bb06a5d9de5fecefc26557fb1f Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 8 May 2020 12:33:45 -0700 Subject: [PATCH 12/18] Remove Enable(kubelet) from unpause This way, we won't ever enable kubelet to start when systemd starts, preventing a race condition with `/etc/hosts` --- pkg/minikube/cluster/pause.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/minikube/cluster/pause.go b/pkg/minikube/cluster/pause.go index 9f59005f436d..67651b2443f9 100644 --- a/pkg/minikube/cluster/pause.go +++ b/pkg/minikube/cluster/pause.go @@ -96,9 +96,6 @@ func unpause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]stri } sm := sysinit.New(r) - if err := sm.Enable("kubelet"); err != nil { - return ids, errors.Wrap(err, "kubelet enable") - } if err := sm.Start("kubelet"); err != nil { return ids, errors.Wrap(err, "kubelet start") From f574852125c79d0dc8e92669e044c26ac42a5bd5 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 8 May 2020 12:37:28 -0700 Subject: [PATCH 13/18] make it impossible to enable the kubelet service --- pkg/minikube/sysinit/systemd.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/minikube/sysinit/systemd.go b/pkg/minikube/sysinit/systemd.go index 4bb53a186b09..9c6ad84b1c2d 100644 --- a/pkg/minikube/sysinit/systemd.go +++ b/pkg/minikube/sysinit/systemd.go @@ -18,6 +18,7 @@ limitations under the License. package sysinit import ( + "errors" "os/exec" "k8s.io/minikube/pkg/minikube/assets" @@ -53,6 +54,9 @@ func (s *Systemd) Disable(svc string) error { // Enable enables a service func (s *Systemd) Enable(svc string) error { + if svc == "kubelet" { + return errors.New("please don't enable kubelet as it creates a race condition; if it starts on systemd boot it will pick up /etc/hosts before we have time to configure /etc/hosts") + } _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "enable", svc)) return err } From cb4dc30babc7eb8f0a6e37657864297911fdcbc2 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 8 May 2020 12:49:54 -0700 Subject: [PATCH 14/18] Add unit test --- pkg/minikube/sysinit/systemd_test.go | 57 ++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 pkg/minikube/sysinit/systemd_test.go diff --git a/pkg/minikube/sysinit/systemd_test.go b/pkg/minikube/sysinit/systemd_test.go new file mode 100644 index 000000000000..e3d01c70de08 --- /dev/null +++ b/pkg/minikube/sysinit/systemd_test.go @@ -0,0 +1,57 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package sysinit provides an abstraction over init systems like systemctl +package sysinit + +import ( + "testing" + + "k8s.io/minikube/pkg/minikube/command" +) + +func TestEnable(t *testing.T) { + + tests := []struct { + service string + shouldErr bool + }{ + { + service: "docker", + }, { + service: "kubelet", + shouldErr: true, + }, + } + cr := command.NewFakeCommandRunner() + cr.SetCommandToOutput(map[string]string{ + "sudo systemctl enable docker": "", + }) + sd := &Systemd{ + r: cr, + } + for _, test := range tests { + t.Run(test.service, func(t *testing.T) { + err := sd.Enable(test.service) + if err == nil && test.shouldErr { + t.Fatalf("expected %s service to error, but it did not", test.service) + } + if err != nil && !test.shouldErr { + t.Fatalf("expected %s service to pass, but it did not", test.service) + } + }) + } +} From 6d1d07c8575a25b78c041cc3294499a38c20f357 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 8 May 2020 13:55:30 -0700 Subject: [PATCH 15/18] Remove unnecessary restart --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index ed4462fe7f9a..7e75e09680f1 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -542,7 +542,6 @@ func (k *Bootstrapper) restartCluster(cfg config.ClusterConfig) error { cmds := []string{ fmt.Sprintf("%s phase certs all --config %s", baseCmd, conf), fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, conf), - fmt.Sprintf("sudo systemctl restart kubelet"), fmt.Sprintf("%s phase %s all --config %s", baseCmd, controlPlane, conf), fmt.Sprintf("%s phase etcd local --config %s", baseCmd, conf), } From 753dc447061bfc27d9aaf24ad4ccc051e9d74397 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 8 May 2020 13:57:11 -0700 Subject: [PATCH 16/18] Remove unnecessary function --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 23 +------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 7e75e09680f1..234901b00eb2 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -775,11 +775,7 @@ func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cru return errors.Wrap(err, "host alias") } - if err := startKubelet(k.c, sm); err != nil { - return errors.Wrap(err, "reload") - } - - return nil + return sm.Start("kubelet") } func copyFiles(runner command.Runner, files []assets.CopyableFile) error { @@ -801,23 +797,6 @@ func copyFiles(runner command.Runner, files []assets.CopyableFile) error { return nil } -func startKubelet(runner command.Runner, sm sysinit.Manager) error { - now := time.Now() - defer func() { - glog.Infof("reloadKubelet took %s", time.Since(now)) - }() - - svc := bsutil.KubeletServiceFile - conf := bsutil.KubeletSystemdConfFile - - startCmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo cp %s.new %s && sudo cp %s.new %s", svc, svc, conf, conf)) - if _, err := runner.RunCmd(startCmd); err != nil { - return errors.Wrap(err, "starting kubelet") - } - - return sm.Start("kubelet") -} - // kubectlPath returns the path to the kubelet func kubectlPath(cfg config.ClusterConfig) string { return path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl") From bfef99b63d952b417d31768bdbc66de6acf6015e Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 8 May 2020 14:12:13 -0700 Subject: [PATCH 17/18] Copy files directly now that we control kubeadm --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 234901b00eb2..15e338b948f3 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -744,8 +744,8 @@ func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cru files := []assets.CopyableFile{ assets.NewMemoryAssetTarget(kubeadmCfg, bsutil.KubeadmYamlPath+".new", "0640"), - assets.NewMemoryAssetTarget(kubeletCfg, bsutil.KubeletSystemdConfFile+".new", "0644"), - assets.NewMemoryAssetTarget(kubeletService, bsutil.KubeletServiceFile+".new", "0644"), + assets.NewMemoryAssetTarget(kubeletCfg, bsutil.KubeletSystemdConfFile, "0644"), + assets.NewMemoryAssetTarget(kubeletService, bsutil.KubeletServiceFile, "0644"), } // Copy the default CNI config (k8s.conf), so that kubelet can successfully // start a Pod in the case a user hasn't manually installed any CNI plugin From 2915aaf597be72dc06516ebee6f2daf166ef7b92 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 8 May 2020 14:12:47 -0700 Subject: [PATCH 18/18] Remove test --- test/integration/start_stop_delete_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index 88191cce24d2..b6764ea967b5 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -71,9 +71,6 @@ func TestStartStop(t *testing.T) { "--disable-driver-mounts", "--extra-config=kubeadm.ignore-preflight-errors=SystemVerification", }}, - {"docker", constants.DefaultKubernetesVersion, []string{ - "--container-runtime=docker", - }}, {"embed-certs", constants.DefaultKubernetesVersion, []string{ "--embed-certs", }},