diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kubelet.go index 5c21698deba2..5046b67bb142 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubelet.go +++ b/pkg/minikube/bootstrapper/bsutil/kubelet.go @@ -41,6 +41,11 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, return nil, errors.Wrap(err, "generating extra configuration for kubelet") } + cgroupDriver, err := r.CGroupDriver() + if err == nil { + extraOpts["cgroup-driver"] = cgroupDriver + } + for k, v := range r.KubeletOptions() { extraOpts[k] = v } diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go index 3c93aa97c2d3..226f4869964d 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go +++ b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/pmezard/go-difflib/difflib" + "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" @@ -135,7 +136,8 @@ ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhoo for _, tc := range tests { t.Run(tc.description, func(t *testing.T) { - runtime, err := cruntime.New(cruntime.Config{Type: tc.cfg.ContainerRuntime}) + runtime, err := cruntime.New(cruntime.Config{Type: tc.cfg.ContainerRuntime, + Runner: command.NewFakeCommandRunner()}) if err != nil { t.Fatalf("runtime: %v", err) } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index f5cfe92dc367..323e5e7609ab 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -368,7 +368,8 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { out.FailureT("Unable to load cached images: {{.error}}", out.V{"error": err}) } } - r, err := cruntime.New(cruntime.Config{Type: cfg.ContainerRuntime, Socket: cfg.KubernetesConfig.CRISocket}) + r, err := cruntime.New(cruntime.Config{Type: cfg.ContainerRuntime, + Runner: k.c, Socket: cfg.KubernetesConfig.CRISocket}) if err != nil { return errors.Wrap(err, "runtime") } diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 3e91a45edc55..fe10692464e3 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -240,6 +240,29 @@ func (r *Containerd) LoadImage(path string) error { return nil } +// CGroupDriver returns cgroup driver ("cgroupfs" or "systemd") +func (r *Containerd) CGroupDriver() (string, error) { + info, err := getCRIInfo(r.Runner) + if err != nil { + return "", err + } + if info["config"] == nil { + return "", errors.Wrapf(err, "missing config") + } + config, ok := info["config"].(map[string]interface{}) + if !ok { + return "", errors.Wrapf(err, "config not map") + } + cgroupManager := "cgroupfs" // default + switch config["systemdCgroup"] { + case false: + cgroupManager = "cgroupfs" + case true: + cgroupManager = "systemd" + } + return cgroupManager, nil +} + // KubeletOptions returns kubelet options for a containerd func (r *Containerd) KubeletOptions() map[string]string { return map[string]string{ diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go index 1ac6d3d3b363..881167de99da 100644 --- a/pkg/minikube/cruntime/cri.go +++ b/pkg/minikube/cruntime/cri.go @@ -19,6 +19,7 @@ package cruntime import ( "bytes" "encoding/base64" + "encoding/json" "fmt" "html/template" "os/exec" @@ -420,6 +421,23 @@ image-endpoint: unix://{{.Socket}} return nil } +// getCRIInfo returns current information +func getCRIInfo(cr CommandRunner) (map[string]interface{}, error) { + args := []string{"crictl", "info"} + c := exec.Command("sudo", args...) + rr, err := cr.RunCmd(c) + if err != nil { + return nil, errors.Wrap(err, "get cri info") + } + info := rr.Stdout.String() + jsonMap := make(map[string]interface{}) + err = json.Unmarshal([]byte(info), &jsonMap) + if err != nil { + return nil, err + } + return jsonMap, nil +} + // generateCRIOConfig sets up /etc/crio/crio.conf func generateCRIOConfig(cr CommandRunner, imageRepository string) error { cPath := crioConfigFile diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 2449733e1298..e455c61309c1 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -143,6 +143,26 @@ func (r *CRIO) LoadImage(path string) error { return nil } +// CGroupDriver returns cgroup driver ("cgroupfs" or "systemd") +func (r *CRIO) CGroupDriver() (string, error) { + c := exec.Command("crio", "config") + rr, err := r.Runner.RunCmd(c) + if err != nil { + return "", err + } + cgroupManager := "cgroupfs" // default + for _, line := range strings.Split(rr.Stdout.String(), "\n") { + if strings.HasPrefix(line, "cgroup_manager") { + // cgroup_manager = "cgroupfs" + f := strings.Split(strings.TrimSpace(line), " = ") + if len(f) == 2 { + cgroupManager = strings.Trim(f[1], "\"") + } + } + } + return cgroupManager, nil +} + // KubeletOptions returns kubelet options for a runtime. func (r *CRIO) KubeletOptions() map[string]string { return map[string]string{ diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 4c0a01186656..11ce642d4409 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -49,6 +49,8 @@ type Manager interface { // Style is an associated StyleEnum for Name() Style() out.StyleEnum + // CGroupDriver returns cgroup driver ("cgroupfs" or "systemd") + CGroupDriver() (string, error) // KubeletOptions returns kubelet options for a runtime. KubeletOptions() map[string]string // SocketPath returns the path to the socket file for a given runtime diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go index 971ed22d2c9c..cb92244ce37c 100644 --- a/pkg/minikube/cruntime/cruntime_test.go +++ b/pkg/minikube/cruntime/cruntime_test.go @@ -54,6 +54,33 @@ func TestName(t *testing.T) { } } +func TestCGroupDriver(t *testing.T) { + var tests = []struct { + runtime string + want string + }{ + {"docker", "cgroupfs"}, + {"crio", "cgroupfs"}, + {"containerd", "cgroupfs"}, + } + for _, tc := range tests { + t.Run(tc.runtime, func(t *testing.T) { + r, err := New(Config{Type: tc.runtime, Runner: NewFakeRunner(t)}) + if err != nil { + t.Fatalf("New(%s): %v", tc.runtime, err) + } + + got, err := r.CGroupDriver() + if err != nil { + t.Fatalf("CGroupDriver(): %v", err) + } + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("CGroupDriver(%s) returned diff (-want +got):\n%s", tc.runtime, diff) + } + }) + } +} + func TestKubeletOptions(t *testing.T) { var tests = []struct { runtime string @@ -201,6 +228,12 @@ func (f *FakeRunner) docker(args []string, _ bool) (string, error) { if args[1] == "--format" && args[2] == "'{{.Server.Version}}'" { return "18.06.2-ce", nil } + + case "info": + + if args[1] == "--format" && args[2] == "'{{.CgroupDriver}}'" { + return "cgroupfs", nil + } } return "", nil } @@ -210,6 +243,9 @@ func (f *FakeRunner) crio(args []string, _ bool) (string, error) { //nolint (res if args[0] == "--version" { return "crio version 1.13.0", nil } + if args[0] == "config" { + return "# Cgroup management implementation used for the runtime.\ncgroup_manager = \"cgroupfs\"\n", nil + } return "", nil } @@ -227,6 +263,15 @@ func (f *FakeRunner) containerd(args []string, _ bool) (string, error) { // crictl is a fake implementation of crictl func (f *FakeRunner) crictl(args []string, _ bool) (string, error) { switch cmd := args[0]; cmd { + case "info": + return `{ + "status": { + }, + "config": { + "systemdCgroup": false + }, + "golang": "go1.11.13" + }`, nil case "ps": // crictl ps -a --name=apiserver --state=Running --quiet if args[1] == "-a" && strings.HasPrefix(args[2], "--name") { diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 5ef472d73a5c..76ecf397b0ef 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -127,6 +127,17 @@ func (r *Docker) LoadImage(path string) error { } +// CGroupDriver returns cgroup driver ("cgroupfs" or "systemd") +func (r *Docker) CGroupDriver() (string, error) { + // Note: the server daemon has to be running, for this call to return successfully + c := exec.Command("docker", "info", "--format", "'{{.CgroupDriver}}'") + rr, err := r.Runner.RunCmd(c) + if err != nil { + return "", err + } + return strings.Split(rr.Stdout.String(), "\n")[0], nil +} + // KubeletOptions returns kubelet options for a runtime. func (r *Docker) KubeletOptions() map[string]string { return map[string]string{