diff --git a/cmd/minikube/cmd/docker-env.go b/cmd/minikube/cmd/docker-env.go index 76ae9d4f5ad0..c4432c6b5825 100644 --- a/cmd/minikube/cmd/docker-env.go +++ b/cmd/minikube/cmd/docker-env.go @@ -24,7 +24,6 @@ import ( "io" "net" "os" - "os/exec" "strconv" "strings" @@ -38,6 +37,7 @@ import ( "k8s.io/minikube/pkg/minikube/mustload" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/shell" + "k8s.io/minikube/pkg/minikube/sysinit" ) var dockerEnvTmpl = fmt.Sprintf("{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerTLSVerify }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerHost }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerCertPath }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubeDockerdProfile }}{{ .Suffix }}{{ if .NoProxyVar }}{{ .Prefix }}{{ .NoProxyVar }}{{ .Delimiter }}{{ .NoProxyValue }}{{ .Suffix }}{{end}}{{ .UsageHint }}", constants.DockerTLSVerifyEnv, constants.DockerHostEnv, constants.DockerCertPathEnv, constants.MinikubeActiveDockerdEnv) @@ -116,9 +116,7 @@ func (EnvNoProxyGetter) GetNoProxyVar() (string, string) { // isDockerActive checks if Docker is active func isDockerActive(r command.Runner) bool { - c := exec.Command("sudo", "systemctl", "is-active", "--quiet", "service", "docker") - _, err := r.RunCmd(c) - return err == nil + return sysinit.New(r).Active("docker") } // dockerEnvCmd represents the docker-env command diff --git a/cmd/minikube/cmd/status.go b/cmd/minikube/cmd/status.go index 95d21e30790b..a95c7978cb26 100644 --- a/cmd/minikube/cmd/status.go +++ b/cmd/minikube/cmd/status.go @@ -208,15 +208,9 @@ func status(api libmachine.API, cc config.ClusterConfig, n config.Node) (*Status return st, err } - stk, err := kverify.KubeletStatus(cr) - glog.Infof("%s kubelet status = %s (err=%v)", name, stk, err) - - if err != nil { - glog.Warningf("kubelet err: %v", err) - st.Kubelet = state.Error.String() - } else { - st.Kubelet = stk.String() - } + stk := kverify.KubeletStatus(cr) + glog.Infof("%s kubelet status = %s", name, stk) + st.Kubelet = stk.String() // Early exit for regular nodes if !controlPlane { diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index f178a6592bb0..1a22e7404de4 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -31,6 +31,7 @@ import ( "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/localpath" + "k8s.io/minikube/pkg/minikube/sysinit" ) func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string) error { @@ -86,7 +87,9 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string KubernetesVersion: kubernetesVersion, } runner := command.NewKICRunner(profile, driver.OCIBinary) - if err := bsutil.TransferBinaries(kcfg, runner); err != nil { + sm := sysinit.New(runner) + + if err := bsutil.TransferBinaries(kcfg, runner, sm); err != nil { return errors.Wrap(err, "transferring k8s binaries") } // Create image tarball diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 692bcdf485c0..7a342afcbc92 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -37,7 +37,7 @@ import ( "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/download" - "k8s.io/minikube/pkg/minikube/kubelet" + "k8s.io/minikube/pkg/minikube/sysinit" ) // Driver represents a kic driver https://minikube.sigs.k8s.io/docs/reference/drivers/docker @@ -245,7 +245,7 @@ func (d *Driver) GetState() (state.State, error) { func (d *Driver) Kill() error { // on init this doesn't get filled when called from cmd d.exec = command.NewKICRunner(d.MachineName, d.OCIBinary) - if err := kubelet.ForceStop(d.exec); err != nil { + if err := sysinit.New(d.exec).ForceStop("kubelet"); err != nil { glog.Warningf("couldn't force stop kubelet. will continue with kill anyways: %v", err) } cmd := exec.Command(d.NodeConfig.OCIBinary, "kill", d.MachineName) @@ -318,9 +318,9 @@ func (d *Driver) Stop() error { d.exec = command.NewKICRunner(d.MachineName, d.OCIBinary) // docker does not send right SIG for systemd to know to stop the systemd. // to avoid bind address be taken on an upgrade. more info https://github.com/kubernetes/minikube/issues/7171 - if err := kubelet.Stop(d.exec); err != nil { + if err := sysinit.New(d.exec).Stop("kubelet"); err != nil { glog.Warningf("couldn't stop kubelet. will continue with stop anyways: %v", err) - if err := kubelet.ForceStop(d.exec); err != nil { + if err := sysinit.New(d.exec).ForceStop("kubelet"); err != nil { glog.Warningf("couldn't force stop kubelet. will continue with stop anyways: %v", err) } } diff --git a/pkg/drivers/none/none.go b/pkg/drivers/none/none.go index 91ab30546463..1049f713fe38 100644 --- a/pkg/drivers/none/none.go +++ b/pkg/drivers/none/none.go @@ -31,7 +31,7 @@ import ( "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/kubeconfig" - "k8s.io/minikube/pkg/minikube/kubelet" + "k8s.io/minikube/pkg/minikube/sysinit" "k8s.io/minikube/pkg/minikube/vmpath" ) @@ -142,12 +142,12 @@ func (d *Driver) GetState() (state.State, error) { return state.Running, nil } - return kverify.KubeletStatus(d.exec) + return kverify.KubeletStatus(d.exec), nil } // Kill stops a host forcefully, including any containers that we are managing. func (d *Driver) Kill() error { - if err := kubelet.ForceStop(d.exec); err != nil { + if err := sysinit.New(d.exec).ForceStop("kubelet"); err != nil { glog.Warningf("couldn't force stop kubelet. will continue with kill anyways: %v", err) } @@ -211,9 +211,9 @@ func (d *Driver) Start() error { // Stop a host gracefully, including any containers that we are managing. func (d *Driver) Stop() error { - if err := kubelet.Stop(d.exec); err != nil { + if err := sysinit.New(d.exec).Stop("kubelet"); err != nil { glog.Warningf("couldn't stop kubelet. will continue with stop anyways: %v", err) - if err := kubelet.ForceStop(d.exec); err != nil { + if err := sysinit.New(d.exec).ForceStop("kubelet"); err != nil { glog.Warningf("couldn't force stop kubelet. will continue with stop anyways: %v", err) } } diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index c724b2607fa3..0019b1750d5a 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -45,7 +45,6 @@ type Bootstrapper interface { // LogCommands returns a map of log type to a command which will display that log. LogCommands(config.ClusterConfig, LogOptions) map[string]string SetupCerts(config.KubernetesConfig, config.Node) error - GetKubeletStatus() (string, error) GetAPIServerStatus(string, int) (string, error) } diff --git a/pkg/minikube/bootstrapper/bsutil/binaries.go b/pkg/minikube/bootstrapper/bsutil/binaries.go index 0ffcaa05ad3e..341a5b295b7e 100644 --- a/pkg/minikube/bootstrapper/bsutil/binaries.go +++ b/pkg/minikube/bootstrapper/bsutil/binaries.go @@ -32,11 +32,12 @@ import ( "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/machine" + "k8s.io/minikube/pkg/minikube/sysinit" "k8s.io/minikube/pkg/minikube/vmpath" ) // TransferBinaries transfers all required Kubernetes binaries -func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error { +func TransferBinaries(cfg config.KubernetesConfig, c command.Runner, sm sysinit.Manager) error { ok, err := binariesExist(cfg, c) if err == nil && ok { glog.Info("Found k8s binaries, skipping transfer") @@ -50,11 +51,6 @@ func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error { return err } - // stop kubelet to avoid "Text File Busy" error - if _, err := c.RunCmd(exec.Command("/bin/bash", "-c", "pgrep kubelet && sudo systemctl stop kubelet")); err != nil { - glog.Warningf("unable to stop kubelet: %s", err) - } - var g errgroup.Group for _, name := range constants.KubernetesReleaseBinaries { name := name @@ -64,6 +60,12 @@ func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error { return errors.Wrapf(err, "downloading %s", name) } + if name == "kubelet" { + if err := sm.ForceStop("kubelet"); err != nil { + glog.Errorf("unable to stop kubelet: %v", err) + } + } + dst := path.Join(dir, name) if err := machine.CopyBinary(c, src, dst); err != nil { return errors.Wrapf(err, "copybinary %s -> %s", src, dst) diff --git a/pkg/minikube/bootstrapper/bsutil/files.go b/pkg/minikube/bootstrapper/bsutil/files.go index cec6d690856d..8d6bc05bc36d 100644 --- a/pkg/minikube/bootstrapper/bsutil/files.go +++ b/pkg/minikube/bootstrapper/bsutil/files.go @@ -20,8 +20,6 @@ package bsutil import ( "path" - "k8s.io/minikube/pkg/minikube/assets" - "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/vmpath" ) @@ -35,20 +33,8 @@ const ( KubeletServiceFile = "/lib/systemd/system/kubelet.service" // KubeletSystemdConfFile is config for the systemd kubelet.service KubeletSystemdConfFile = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" + // InitRestartWrapper is ... + InitRestartWrapper = "/etc/init.d/.restart_wrapper.sh" + // KubeletInitPath is where Sys-V style init script is installed + KubeletInitPath = "/etc/init.d/kubelet" ) - -// ConfigFileAssets returns configuration file assets -func ConfigFileAssets(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byte, kubeletSvc []byte, defaultCNIConfig []byte) []assets.CopyableFile { - fs := []assets.CopyableFile{ - assets.NewMemoryAssetTarget(kubeadm, KubeadmYamlPath+".new", "0640"), - assets.NewMemoryAssetTarget(kubelet, KubeletSystemdConfFile+".new", "0644"), - assets.NewMemoryAssetTarget(kubeletSvc, KubeletServiceFile+".new", "0644"), - } - // Copy the default CNI config (k8s.conf), so that kubelet can successfully - // start a Pod in the case a user hasn't manually installed any CNI plugin - // and minikube was started with "--extra-config=kubelet.network-plugin=cni". - if defaultCNIConfig != nil { - fs = append(fs, assets.NewMemoryAssetTarget(defaultCNIConfig, DefaultCNIConfigPath, "0644")) - } - return fs -} diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go index d13f3b3ea34a..35dd40a858b6 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go @@ -19,7 +19,6 @@ package kverify import ( "fmt" - "os/exec" "strings" "time" @@ -36,6 +35,7 @@ import ( "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/logs" + "k8s.io/minikube/pkg/minikube/sysinit" ) // WaitForSystemPods verifies essential pods for running kurnetes is running @@ -156,22 +156,11 @@ func announceProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg conf } // KubeletStatus checks the kubelet status -func KubeletStatus(cr command.Runner) (state.State, error) { +func KubeletStatus(cr command.Runner) state.State { glog.Infof("Checking kubelet status ...") - rr, err := cr.RunCmd(exec.Command("sudo", "systemctl", "is-active", "kubelet")) - if err != nil { - // Do not return now, as we still have parsing to do! - glog.Warningf("%s returned error: %v", rr.Command(), err) - } - s := strings.TrimSpace(rr.Stdout.String()) - glog.Infof("kubelet is-active: %s", s) - switch s { - case "active": - return state.Running, nil - case "inactive": - return state.Stopped, nil - case "activating": - return state.Starting, nil + active := sysinit.New(cr).Active("kubelet") + if active { + return state.Running } - return state.Error, nil + return state.Stopped } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 10bac2d72623..b97707a5cf98 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -51,9 +51,9 @@ import ( "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/driver" - "k8s.io/minikube/pkg/minikube/kubelet" "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/sysinit" "k8s.io/minikube/pkg/minikube/vmpath" "k8s.io/minikube/pkg/util" "k8s.io/minikube/pkg/util/retry" @@ -82,26 +82,6 @@ func NewBootstrapper(api libmachine.API, cc config.ClusterConfig, n config.Node) return &Bootstrapper{c: runner, contextName: cc.Name, k8sClient: nil}, nil } -// GetKubeletStatus returns the kubelet status -func (k *Bootstrapper) GetKubeletStatus() (string, error) { - rr, err := k.c.RunCmd(exec.Command("sudo", "systemctl", "is-active", "kubelet")) - if err != nil { - // Do not return now, as we still have parsing to do! - glog.Warningf("%s returned error: %v", rr.Command(), err) - } - s := strings.TrimSpace(rr.Stdout.String()) - glog.Infof("kubelet is-active: %s", s) - switch s { - case "active": - return state.Running.String(), nil - case "inactive": - return state.Stopped.String(), nil - case "activating": - return state.Starting.String(), nil - } - return state.Error.String(), nil -} - // GetAPIServerStatus returns the api-server status func (k *Bootstrapper) GetAPIServerStatus(hostname string, port int) (string, error) { s, err := kverify.APIServerStatus(k.c, hostname, port) @@ -190,7 +170,7 @@ func (k *Bootstrapper) init(cfg config.ClusterConfig) error { } extraFlags := bsutil.CreateFlagsFromExtraArgs(cfg.KubernetesConfig.ExtraOptions) - r, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime}) + r, err := cruntime.New(cruntime.Config{Type: cfg.KubernetesConfig.ContainerRuntime, Runner: k.c}) if err != nil { return err } @@ -615,7 +595,7 @@ func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error { glog.Warningf("%s: %v", rr.Command(), err) } - if err := kubelet.ForceStop(k.c); err != nil { + if err := sysinit.New(k.c).ForceStop("kubelet"); err != nil { glog.Warningf("stop kubelet: %v", err) } @@ -679,6 +659,11 @@ func (k *Bootstrapper) UpdateCluster(cfg config.ClusterConfig) error { // UpdateNode updates a node. func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cruntime.Manager) error { + now := time.Now() + defer func() { + glog.Infof("reloadKubelet took %s", time.Since(now)) + }() + kubeadmCfg, err := bsutil.GenerateKubeadmYAML(cfg, n, r) if err != nil { return errors.Wrap(err, "generating kubeadm cfg") @@ -696,24 +681,40 @@ func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cru glog.Infof("kubelet %s config:\n%+v", kubeletCfg, cfg.KubernetesConfig) - if err := bsutil.TransferBinaries(cfg.KubernetesConfig, k.c); err != nil { + sm := sysinit.New(k.c) + + if err := bsutil.TransferBinaries(cfg.KubernetesConfig, k.c, sm); err != nil { return errors.Wrap(err, "downloading binaries") } - var cniFile []byte + files := []assets.CopyableFile{ + assets.NewMemoryAssetTarget(kubeadmCfg, bsutil.KubeadmYamlPath+".new", "0640"), + assets.NewMemoryAssetTarget(kubeletCfg, bsutil.KubeletSystemdConfFile+".new", "0644"), + assets.NewMemoryAssetTarget(kubeletService, bsutil.KubeletServiceFile+".new", "0644"), + } + // Copy the default CNI config (k8s.conf), so that kubelet can successfully + // start a Pod in the case a user hasn't manually installed any CNI plugin + // and minikube was started with "--extra-config=kubelet.network-plugin=cni". if cfg.KubernetesConfig.EnableDefaultCNI { - cniFile = []byte(defaultCNIConfig) + files = append(files, assets.NewMemoryAssetTarget([]byte(defaultCNIConfig), bsutil.DefaultCNIConfigPath, "0644")) } - // Install assets into temporary files - files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile) + // Installs compatibility shims for non-systemd environments + kubeletPath := path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl") + shims, err := sm.GenerateInitShim("kubelet", kubeletPath, bsutil.KubeletSystemdConfFile) + if err != nil { + return errors.Wrap(err, "shim") + } + files = append(files, shims...) + if err := copyFiles(k.c, files); err != nil { - return err + return errors.Wrap(err, "copy") } - if err := reloadKubelet(k.c); err != nil { - return err + if err := startKubeletIfRequired(k.c, sm); err != nil { + return errors.Wrap(err, "reload") } + return nil } @@ -736,7 +737,12 @@ func copyFiles(runner command.Runner, files []assets.CopyableFile) error { return nil } -func reloadKubelet(runner command.Runner) error { +func startKubeletIfRequired(runner command.Runner, sm sysinit.Manager) error { + now := time.Now() + defer func() { + glog.Infof("reloadKubelet took %s", time.Since(now)) + }() + svc := bsutil.KubeletServiceFile conf := bsutil.KubeletSystemdConfFile @@ -746,11 +752,12 @@ func reloadKubelet(runner command.Runner) error { return nil } - startCmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo cp %s.new %s && sudo cp %s.new %s && sudo systemctl daemon-reload && sudo systemctl restart kubelet", svc, svc, conf, conf)) + startCmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo cp %s.new %s && sudo cp %s.new %s", svc, svc, conf, conf)) if _, err := runner.RunCmd(startCmd); err != nil { return errors.Wrap(err, "starting kubelet") } - return nil + + return sm.Start("kubelet") } // applyKicOverlay applies the CNI plugin needed to make kic work diff --git a/pkg/minikube/cluster/pause.go b/pkg/minikube/cluster/pause.go index d7661f1274c3..09b1a10c2449 100644 --- a/pkg/minikube/cluster/pause.go +++ b/pkg/minikube/cluster/pause.go @@ -21,27 +21,33 @@ import ( "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/cruntime" - "k8s.io/minikube/pkg/minikube/kubelet" + "k8s.io/minikube/pkg/minikube/sysinit" ) // Pause pauses a Kubernetes cluster func Pause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) { ids := []string{} + // Disable the kubelet so it does not attempt to restart paused pods - if err := kubelet.Disable(r); err != nil { + sm := sysinit.New(r) + if err := sm.Disable("kubelet"); err != nil { return ids, errors.Wrap(err, "kubelet disable") } - if err := kubelet.Stop(r); err != nil { + + if err := sm.Stop("kubelet"); err != nil { return ids, errors.Wrap(err, "kubelet stop") } + ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Running, Namespaces: namespaces}) if err != nil { return ids, errors.Wrap(err, "list running") } + if len(ids) == 0 { glog.Warningf("no running containers to pause") return ids, nil } + return ids, cr.PauseContainers(ids) } @@ -59,11 +65,14 @@ func Unpause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]stri return ids, errors.Wrap(err, "unpause") } - if err := kubelet.Enable(r); err != nil { + sm := sysinit.New(r) + if err := sm.Enable("kubelet"); err != nil { return ids, errors.Wrap(err, "kubelet enable") } - if err := kubelet.Start(r); err != nil { + + if err := sm.Start("kubelet"); err != nil { return ids, errors.Wrap(err, "kubelet start") } + return ids, nil } diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 89a2e3912a06..8eadb57392e2 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -32,6 +32,7 @@ import ( "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/sysinit" ) const ( @@ -116,6 +117,7 @@ type Containerd struct { Runner CommandRunner ImageRepository string KubernetesVersion semver.Version + Init sysinit.Manager } // Name is a human readable name for containerd @@ -158,9 +160,7 @@ func (r *Containerd) DefaultCNI() bool { // Active returns if containerd is active on the host func (r *Containerd) Active() bool { - c := exec.Command("sudo", "systemctl", "is-active", "--quiet", "service", "containerd") - _, err := r.Runner.RunCmd(c) - return err == nil + return r.Init.Active("containerd") } // Available returns an error if it is not possible to use this runtime on a host @@ -208,21 +208,14 @@ func (r *Containerd) Enable(disOthers bool) error { if err := enableIPForwarding(r.Runner); err != nil { return err } + // Otherwise, containerd will fail API requests with 'Unimplemented' - c := exec.Command("sudo", "systemctl", "restart", "containerd") - if _, err := r.Runner.RunCmd(c); err != nil { - return errors.Wrap(err, "restart containerd") - } - return nil + return r.Init.Restart("containerd") } // Disable idempotently disables containerd on a host func (r *Containerd) Disable() error { - c := exec.Command("sudo", "systemctl", "stop", "-f", "containerd") - if _, err := r.Runner.RunCmd(c); err != nil { - return errors.Wrapf(err, "stop containerd") - } - return nil + return r.Init.ForceStop("containerd") } // ImageExists checks if an image exists, expected input format diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 5678a9d6a4db..250d765df4fc 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -28,6 +28,7 @@ import ( "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/sysinit" ) const ( @@ -41,6 +42,7 @@ type CRIO struct { Runner CommandRunner ImageRepository string KubernetesVersion semver.Version + Init sysinit.Manager } // generateCRIOConfig sets up /etc/crio/crio.conf @@ -104,9 +106,7 @@ func (r *CRIO) Available() error { // Active returns if CRIO is active on the host func (r *CRIO) Active() bool { - c := exec.Command("sudo", "systemctl", "is-active", "--quiet", "service", "crio") - _, err := r.Runner.RunCmd(c) - return err == nil + return r.Init.Active("crio") } // Enable idempotently enables CRIO on a host @@ -125,19 +125,12 @@ func (r *CRIO) Enable(disOthers bool) error { if err := enableIPForwarding(r.Runner); err != nil { return err } - - if _, err := r.Runner.RunCmd(exec.Command("sudo", "systemctl", "restart", "crio")); err != nil { - return errors.Wrapf(err, "enable crio.") - } - return nil + return r.Init.Start("crio") } // Disable idempotently disables CRIO on a host func (r *CRIO) Disable() error { - if _, err := r.Runner.RunCmd(exec.Command("sudo", "systemctl", "stop", "-f", "crio")); err != nil { - return errors.Wrapf(err, "disable crio.") - } - return nil + return r.Init.ForceStop("crio") } // ImageExists checks if an image exists diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index d7153d083094..9bde921993e1 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -28,6 +28,7 @@ import ( "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/sysinit" ) // ContainerState is the run state of a container @@ -131,13 +132,27 @@ type ListOptions struct { // New returns an appropriately configured runtime func New(c Config) (Manager, error) { + sm := sysinit.New(c.Runner) + switch c.Type { case "", "docker": - return &Docker{Socket: c.Socket, Runner: c.Runner}, nil + return &Docker{Socket: c.Socket, Runner: c.Runner, Init: sm}, nil case "crio", "cri-o": - return &CRIO{Socket: c.Socket, Runner: c.Runner, ImageRepository: c.ImageRepository, KubernetesVersion: c.KubernetesVersion}, nil + return &CRIO{ + Socket: c.Socket, + Runner: c.Runner, + ImageRepository: c.ImageRepository, + KubernetesVersion: c.KubernetesVersion, + Init: sm, + }, nil case "containerd": - return &Containerd{Socket: c.Socket, Runner: c.Runner, ImageRepository: c.ImageRepository, KubernetesVersion: c.KubernetesVersion}, nil + return &Containerd{ + Socket: c.Socket, + Runner: c.Runner, + ImageRepository: c.ImageRepository, + KubernetesVersion: c.KubernetesVersion, + Init: sm, + }, nil default: return nil, fmt.Errorf("unknown runtime type: %q", c.Type) } diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go index 9523f1126a14..aef420cf8496 100644 --- a/pkg/minikube/cruntime/cruntime_test.go +++ b/pkg/minikube/cruntime/cruntime_test.go @@ -23,6 +23,7 @@ import ( "strings" "testing" + "github.com/golang/glog" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/pkg/errors" @@ -406,12 +407,27 @@ func (f *FakeRunner) crictl(args []string, _ bool) (string, error) { // systemctl is a fake implementation of systemctl func (f *FakeRunner) systemctl(args []string, root bool) (string, error) { // nolint result 0 (string) is always "" + glog.Infof("fake systemctl: %v", args) action := args[0] - svcs := args[1:] + + if action == "--version" { + return "systemd 123 (321.2-1)", nil + } + + if action == "daemon-reload" { + return "ok", nil + } + + var svcs []string + if len(args) > 0 { + svcs = args[1:] + } + // force if svcs[0] == "-f" { svcs = svcs[1:] } + out := "" for i, arg := range args { @@ -496,7 +512,6 @@ func TestVersion(t *testing.T) { // defaultServices reflects the default boot state for the minikube VM var defaultServices = map[string]serviceState{ "docker": SvcRunning, - "docker.socket": SvcRunning, "crio": SvcExited, "crio-shutdown": SvcExited, "containerd": SvcExited, @@ -507,7 +522,7 @@ func TestDisable(t *testing.T) { runtime string want []string }{ - {"docker", []string{"sudo", "systemctl", "stop", "-f", "docker", "docker.socket"}}, + {"docker", []string{"sudo", "systemctl", "stop", "-f", "docker"}}, {"crio", []string{"sudo", "systemctl", "stop", "-f", "crio"}}, {"containerd", []string{"sudo", "systemctl", "stop", "-f", "containerd"}}, } @@ -539,23 +554,20 @@ func TestEnable(t *testing.T) { }{ {"docker", map[string]serviceState{ "docker": SvcRunning, - "docker.socket": SvcRunning, "containerd": SvcExited, "crio": SvcExited, "crio-shutdown": SvcExited, }}, {"containerd", map[string]serviceState{ "docker": SvcExited, - "docker.socket": SvcExited, "containerd": SvcRestarted, "crio": SvcExited, "crio-shutdown": SvcExited, }}, {"crio", map[string]serviceState{ "docker": SvcExited, - "docker.socket": SvcExited, "containerd": SvcExited, - "crio": SvcRestarted, + "crio": SvcRunning, "crio-shutdown": SvcExited, }}, } diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index df9577f47c42..236ae3510ac9 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -32,6 +32,7 @@ import ( "k8s.io/minikube/pkg/minikube/docker" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/out" + "k8s.io/minikube/pkg/minikube/sysinit" ) // KubernetesContainerPrefix is the prefix of each kubernetes container @@ -56,6 +57,7 @@ func (e *ErrISOFeature) Error() string { type Docker struct { Socket string Runner CommandRunner + Init sysinit.Manager } // Name is a human readable name for Docker @@ -97,9 +99,7 @@ func (r *Docker) Available() error { // Active returns if docker is active on the host func (r *Docker) Active() bool { - c := exec.Command("sudo", "systemctl", "is-active", "--quiet", "service", "docker") - _, err := r.Runner.RunCmd(c) - return err == nil + return r.Init.Active("docker") } // Enable idempotently enables Docker on a host @@ -109,29 +109,18 @@ func (r *Docker) Enable(disOthers bool) error { glog.Warningf("disableOthers: %v", err) } } - c := exec.Command("sudo", "systemctl", "start", "docker") - if _, err := r.Runner.RunCmd(c); err != nil { - return errors.Wrap(err, "enable docker.") - } - return nil + + return r.Init.Start("docker") } // Restart restarts Docker on a host func (r *Docker) Restart() error { - c := exec.Command("sudo", "systemctl", "restart", "docker") - if _, err := r.Runner.RunCmd(c); err != nil { - return errors.Wrap(err, "restarting docker.") - } - return nil + return r.Init.Restart("docker") } // Disable idempotently disables Docker on a host func (r *Docker) Disable() error { - c := exec.Command("sudo", "systemctl", "stop", "-f", "docker", "docker.socket") - if _, err := r.Runner.RunCmd(c); err != nil { - return errors.Wrap(err, "disable docker") - } - return nil + return r.Init.ForceStop("docker") } // ImageExists checks if an image exists diff --git a/pkg/minikube/kubelet/kubelet.go b/pkg/minikube/kubelet/kubelet.go deleted file mode 100644 index 8f15e6aca02a..000000000000 --- a/pkg/minikube/kubelet/kubelet.go +++ /dev/null @@ -1,117 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubelet - -import ( - "fmt" - "os/exec" - "strings" - "time" - - "github.com/golang/glog" - "github.com/pkg/errors" - "k8s.io/minikube/pkg/minikube/command" - "k8s.io/minikube/pkg/util/retry" -) - -// Stop idempotently stops the kubelet -func Stop(cr command.Runner) error { - return stop(cr, false) -} - -// ForceStop idempotently force stops the kubelet -func ForceStop(cr command.Runner) error { - return stop(cr, true) -} - -// stop dempotently stops the kubelet -func stop(cr command.Runner, force bool) error { - glog.Infof("stopping kubelet ...") - stp := func() error { - cmd := exec.Command("sudo", "systemctl", "stop", "kubelet.service") - if force { - cmd = exec.Command("sudo", "systemctl", "stop", "-f", "kubelet.service") - } - if rr, err := cr.RunCmd(cmd); err != nil { - return fmt.Errorf("temporary error for %q : %v", rr.Command(), err) - } - cmd = exec.Command("sudo", "systemctl", "show", "-p", "SubState", "kubelet") - rr, err := cr.RunCmd(cmd) - if err != nil { - return fmt.Errorf("temporary error: for %q : %v", rr.Command(), err) - } - if !strings.Contains(rr.Stdout.String(), "dead") && !strings.Contains(rr.Stdout.String(), "failed") { - return fmt.Errorf("unexpected kubelet state: %q", rr.Stdout.String()) - } - return nil - } - - if err := retry.Expo(stp, 1*time.Second, time.Minute, 2); err != nil { - return errors.Wrapf(err, "error stopping kubelet") - } - return nil -} - -// Start starts the kubelet -func Start(cr command.Runner) error { - glog.Infof("restarting kubelet.service ...") - c := exec.Command("sudo", "systemctl", "start", "kubelet") - if _, err := cr.RunCmd(c); err != nil { - return err - } - return nil -} - -// Restart restarts the kubelet -func Restart(cr command.Runner) error { - glog.Infof("restarting kubelet.service ...") - c := exec.Command("sudo", "systemctl", "restart", "kubelet.service") - if _, err := cr.RunCmd(c); err != nil { - return err - } - return nil -} - -// Check checks on the status of the kubelet -func Check(cr command.Runner) error { - glog.Infof("checking for running kubelet ...") - c := exec.Command("sudo", "systemctl", "is-active", "--quiet", "service", "kubelet") - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "check kubelet") - } - return nil -} - -// Disable disables the Kubelet -func Disable(cr command.Runner) error { - glog.Infof("disabling kubelet ...") - c := exec.Command("sudo", "systemctl", "disable", "kubelet") - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "disable") - } - return nil -} - -// Enable enables the Kubelet -func Enable(cr command.Runner) error { - glog.Infof("enabling kubelet ...") - c := exec.Command("sudo", "systemctl", "enable", "kubelet") - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "enable") - } - return nil -} diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index b2129662cd40..46af9f8bd8e0 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -125,6 +125,7 @@ func Start(cc config.ClusterConfig, n config.Node, existingAddons map[string]boo // setup kubeadm (must come after setupKubeconfig) bs = setupKubeAdm(machineAPI, cc, n) + err = bs.StartCluster(cc) if err != nil { exit.WithLogEntries("Error starting cluster", err, logs.FindProblems(cr, bs, cc, mRunner)) diff --git a/pkg/minikube/registry/drvs/none/none.go b/pkg/minikube/registry/drvs/none/none.go index fd7e42a6de78..ec8cea71d8a6 100644 --- a/pkg/minikube/registry/drvs/none/none.go +++ b/pkg/minikube/registry/drvs/none/none.go @@ -52,8 +52,9 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { } func status() registry.State { - if _, err := exec.LookPath("systemctl"); err != nil { - return registry.State{Error: err, Fix: "Use a systemd based Linux distribution", Doc: "https://minikube.sigs.k8s.io/docs/reference/drivers/none/"} + _, err := exec.LookPath("iptables") + if err != nil { + return registry.State{Error: err, Fix: "iptables must be installed", Doc: "https://minikube.sigs.k8s.io/docs/reference/drivers/none/"} } if _, err := exec.LookPath("docker"); err != nil { diff --git a/pkg/minikube/sysinit/openrc.go b/pkg/minikube/sysinit/openrc.go new file mode 100644 index 000000000000..822c4157d083 --- /dev/null +++ b/pkg/minikube/sysinit/openrc.go @@ -0,0 +1,170 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package sysinit provides an abstraction over init systems like systemctl +package sysinit + +import ( + "bytes" + "context" + "html/template" + "os/exec" + "path" + "time" + + "github.com/golang/glog" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/vmpath" +) + +var restartWrapper = `#!/bin/bash +# Wrapper script to emulate systemd restart on non-systemd systems +readonly UNIT_PATH=$1 + +while true; do + if [[ -f "${UNIT_PATH}" ]]; then + eval $(egrep "^ExecStart=" "${UNIT_PATH}" | cut -d"=" -f2-) + fi + sleep 1 +done +` + +var initScriptTmpl = template.Must(template.New("initScript").Parse(`#!/bin/bash +# OpenRC init script shim for systemd units +readonly NAME="{{.Name}}" +readonly RESTART_WRAPPER="{{.Wrapper}}" +readonly UNIT_PATH="{{.Unit}}" +readonly PID_PATH="/var/run/${NAME}.pid" + +function start() { + start-stop-daemon --oknodo --pidfile "${PID_PATH}" --background --start --make-pid --exec "${RESTART_WRAPPER}" "${UNIT_PATH}" +} + +function stop() { + if [[ -f "${PID_PATH}" ]]; then + pkill -P "$(cat ${PID_PATH})" + fi + start-stop-daemon --oknodo --pidfile "${PID_PATH}" --stop +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + restart) + stop + start + ;; + status) + start-stop-daemon --pidfile "${PID_PATH}" --status + ;; + *) + echo "Usage: {{.Name}} {start|stop|restart|status}" + exit 1 + ;; +esac +`)) + +// OpenRC is a service manager for OpenRC-like init systems +type OpenRC struct { + r Runner +} + +// Name returns the name of the init system +func (s *OpenRC) Name() string { + return "OpenRC" +} + +// Active checks if a service is running +func (s *OpenRC) Active(svc string) bool { + _, err := s.r.RunCmd(exec.Command("sudo", "service", svc, "status")) + return err == nil +} + +// Start starts a service idempotently +func (s *OpenRC) Start(svc string) error { + if s.Active(svc) { + return nil + } + ctx, cb := context.WithTimeout(context.Background(), 5*time.Second) + defer cb() + + rr, err := s.r.RunCmd(exec.CommandContext(ctx, "sudo", "service", svc, "start")) + glog.Infof("start output: %s", rr.Output()) + return err +} + +// Disable does nothing +func (s *OpenRC) Disable(svc string) error { + return nil +} + +// Enable does nothing +func (s *OpenRC) Enable(svc string) error { + return nil +} + +// Restart restarts a service +func (s *OpenRC) Restart(svc string) error { + rr, err := s.r.RunCmd(exec.Command("sudo", "service", svc, "restart")) + glog.Infof("restart output: %s", rr.Output()) + return err +} + +// Stop stops a service +func (s *OpenRC) Stop(svc string) error { + rr, err := s.r.RunCmd(exec.Command("sudo", "service", svc, "stop")) + glog.Infof("stop output: %s", rr.Output()) + return err +} + +// ForceStop stops a service with prejuidice +func (s *OpenRC) ForceStop(svc string) error { + return s.Stop(svc) +} + +// GenerateInitShim generates any additional init files required for this service +func (s *OpenRC) GenerateInitShim(svc string, binary string, unit string) ([]assets.CopyableFile, error) { + restartWrapperPath := path.Join(vmpath.GuestPersistentDir, "openrc-restart-wrapper.sh") + + opts := struct { + Binary string + Wrapper string + Name string + Unit string + }{ + Name: svc, + Binary: binary, + Wrapper: restartWrapperPath, + Unit: unit, + } + + var b bytes.Buffer + if err := initScriptTmpl.Execute(&b, opts); err != nil { + return nil, errors.Wrap(err, "template execute") + } + + files := []assets.CopyableFile{ + assets.NewMemoryAssetTarget([]byte(restartWrapper), restartWrapperPath, "0755"), + assets.NewMemoryAssetTarget(b.Bytes(), path.Join("/etc/init.d/", svc), "0755"), + } + + return files, nil +} diff --git a/pkg/minikube/sysinit/sysinit.go b/pkg/minikube/sysinit/sysinit.go new file mode 100644 index 000000000000..fc20479ded5e --- /dev/null +++ b/pkg/minikube/sysinit/sysinit.go @@ -0,0 +1,85 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sysinit + +import ( + "os/exec" + + "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/command" +) + +var cachedSystemdCheck *bool + +// Runner is the subset of command.Runner this package consumes +type Runner interface { + RunCmd(cmd *exec.Cmd) (*command.RunResult, error) +} + +// Manager is a common interface for init systems +type Manager interface { + // Name returns the name of the init manager + Name() string + + // Active returns if a service is active + Active(string) bool + + // Disable disables a service + Disable(string) error + + // Enable enables a service + Enable(string) error + + // Start starts a service idempotently + Start(string) error + + // Restart restarts a service + Restart(string) error + + // Stop stops a service + Stop(string) error + + // ForceStop stops a service with prejudice + ForceStop(string) error + + // GenerateInitShim generates any additional init files required for this service + GenerateInitShim(svc string, binary string, unit string) ([]assets.CopyableFile, error) +} + +// New returns an appropriately configured service manager +func New(r Runner) Manager { + // If we are not provided a runner, we can't do anything anyways + if r == nil { + return nil + } + + var systemd bool + + // Caching the result is important, as this manager may be created in many places, + // and ssh calls are expensive on some drivers, such as Docker. + if cachedSystemdCheck != nil { + systemd = *cachedSystemdCheck + } else { + systemd = usesSystemd(r) + cachedSystemdCheck = &systemd + } + + if systemd { + return &Systemd{r: r} + } + return &OpenRC{r: r} +} diff --git a/pkg/minikube/sysinit/systemd.go b/pkg/minikube/sysinit/systemd.go new file mode 100644 index 000000000000..4bb53a186b09 --- /dev/null +++ b/pkg/minikube/sysinit/systemd.go @@ -0,0 +1,98 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package sysinit provides an abstraction over init systems like systemctl +package sysinit + +import ( + "os/exec" + + "k8s.io/minikube/pkg/minikube/assets" +) + +// Systemd is a service manager for systemd distributions +type Systemd struct { + r Runner +} + +// Name returns the name of the init system +func (s *Systemd) Name() string { + return "systemd" +} + +// reload reloads systemd configuration +func (s *Systemd) reload() error { + _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "daemon-reload")) + return err +} + +// Active checks if a service is running +func (s *Systemd) Active(svc string) bool { + _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "is-active", "--quiet", "service", svc)) + return err == nil +} + +// Disable disables a service +func (s *Systemd) Disable(svc string) error { + _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "disable", svc)) + return err +} + +// Enable enables a service +func (s *Systemd) Enable(svc string) error { + _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "enable", svc)) + return err +} + +// Start starts a service +func (s *Systemd) Start(svc string) error { + if err := s.reload(); err != nil { + return err + } + _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "start", svc)) + return err +} + +// Restart restarts a service +func (s *Systemd) Restart(svc string) error { + if err := s.reload(); err != nil { + return err + } + _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "restart", svc)) + return err +} + +// Stop stops a service +func (s *Systemd) Stop(svc string) error { + _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "stop", svc)) + return err +} + +// ForceStop terminates a service with prejudice +func (s *Systemd) ForceStop(svc string) error { + _, err := s.r.RunCmd(exec.Command("sudo", "systemctl", "stop", "-f", svc)) + return err +} + +// GenerateInitShim does nothing for systemd +func (s *Systemd) GenerateInitShim(svc string, binary string, unit string) ([]assets.CopyableFile, error) { + return nil, nil +} + +func usesSystemd(r Runner) bool { + _, err := r.RunCmd(exec.Command("systemctl", "--version")) + return err == nil +}