From 27f8adf912de3b7401323aedb37d70d371161190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 12 Jan 2020 18:43:18 +0100 Subject: [PATCH 1/4] Add CGroupDriver function to cruntime interface --- pkg/minikube/cruntime/containerd.go | 23 +++++++++++++ pkg/minikube/cruntime/cri.go | 18 +++++++++++ pkg/minikube/cruntime/crio.go | 20 ++++++++++++ pkg/minikube/cruntime/cruntime.go | 2 ++ pkg/minikube/cruntime/cruntime_test.go | 45 ++++++++++++++++++++++++++ pkg/minikube/cruntime/docker.go | 11 +++++++ 6 files changed, 119 insertions(+) diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index d431ee2636ee..2496ee6fe78f 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -240,6 +240,29 @@ func (r *Containerd) LoadImage(path string) error { return nil } +// CGroupDriver returns cgroup driver ("cgroupfs" or "systemd") +func (r *Containerd) CGroupDriver() (string, error) { + info, err := getCRIInfo(r.Runner) + if err != nil { + return "", err + } + if info["config"] == nil { + return "", errors.Wrapf(err, "missing config") + } + config, ok := info["config"].(map[string]interface{}) + if !ok { + return "", errors.Wrapf(err, "config not map") + } + cgroupManager := "cgroupfs" // default + switch config["systemdCgroup"] { + case false: + cgroupManager = "cgroupfs" + case true: + cgroupManager = "systemd" + } + return cgroupManager, nil +} + // KubeletOptions returns kubelet options for a containerd func (r *Containerd) KubeletOptions() map[string]string { return map[string]string{ diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go index f053c7440454..72c29df4b6fd 100644 --- a/pkg/minikube/cruntime/cri.go +++ b/pkg/minikube/cruntime/cri.go @@ -19,6 +19,7 @@ package cruntime import ( "bytes" "encoding/base64" + "encoding/json" "fmt" "html/template" "os/exec" @@ -406,6 +407,23 @@ image-endpoint: unix://{{.Socket}} return nil } +// getCRIInfo returns current information +func getCRIInfo(cr CommandRunner) (map[string]interface{}, error) { + args := []string{"crictl", "info"} + c := exec.Command("sudo", args...) + rr, err := cr.RunCmd(c) + if err != nil { + return nil, errors.Wrap(err, "get cri info") + } + info := rr.Stdout.String() + jsonMap := make(map[string]interface{}) + err = json.Unmarshal([]byte(info), &jsonMap) + if err != nil { + return nil, err + } + return jsonMap, nil +} + // generateCRIOConfig sets up /etc/crio/crio.conf func generateCRIOConfig(cr CommandRunner, imageRepository string) error { cPath := crioConfigFile diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 1b297d2e2b45..09c9e78508fb 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -143,6 +143,26 @@ func (r *CRIO) LoadImage(path string) error { return nil } +// CGroupDriver returns cgroup driver ("cgroupfs" or "systemd") +func (r *CRIO) CGroupDriver() (string, error) { + c := exec.Command("crio", "config") + rr, err := r.Runner.RunCmd(c) + if err != nil { + return "", err + } + cgroupManager := "cgroupfs" // default + for _, line := range strings.Split(rr.Stdout.String(), "\n") { + if strings.HasPrefix(line, "cgroup_manager") { + // cgroup_manager = "cgroupfs" + f := strings.Split(strings.TrimSpace(line), " = ") + if len(f) == 2 { + cgroupManager = strings.Trim(f[1], "\"") + } + } + } + return cgroupManager, nil +} + // KubeletOptions returns kubelet options for a runtime. func (r *CRIO) KubeletOptions() map[string]string { return map[string]string{ diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index d59499d40e08..001594f5a72e 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -49,6 +49,8 @@ type Manager interface { // Style is an associated StyleEnum for Name() Style() out.StyleEnum + // CGroupDriver returns cgroup driver ("cgroupfs" or "systemd") + CGroupDriver() (string, error) // KubeletOptions returns kubelet options for a runtime. KubeletOptions() map[string]string // SocketPath returns the path to the socket file for a given runtime diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go index 13ed8bdde664..f3bd5567886d 100644 --- a/pkg/minikube/cruntime/cruntime_test.go +++ b/pkg/minikube/cruntime/cruntime_test.go @@ -54,6 +54,33 @@ func TestName(t *testing.T) { } } +func TestCGroupDriver(t *testing.T) { + var tests = []struct { + runtime string + want string + }{ + {"docker", "cgroupfs"}, + {"crio", "cgroupfs"}, + {"containerd", "cgroupfs"}, + } + for _, tc := range tests { + t.Run(tc.runtime, func(t *testing.T) { + r, err := New(Config{Type: tc.runtime, Runner: NewFakeRunner(t)}) + if err != nil { + t.Fatalf("New(%s): %v", tc.runtime, err) + } + + got, err := r.CGroupDriver() + if err != nil { + t.Fatalf("CGroupDriver(): %v", err) + } + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("CGroupDriver(%s) returned diff (-want +got):\n%s", tc.runtime, diff) + } + }) + } +} + func TestKubeletOptions(t *testing.T) { var tests = []struct { runtime string @@ -199,6 +226,12 @@ func (f *FakeRunner) docker(args []string, _ bool) (string, error) { if args[1] == "--format" && args[2] == "'{{.Server.Version}}'" { return "18.06.2-ce", nil } + + case "info": + + if args[1] == "--format" && args[2] == "'{{.CgroupDriver}}'" { + return "cgroupfs", nil + } } return "", nil } @@ -208,6 +241,9 @@ func (f *FakeRunner) crio(args []string, _ bool) (string, error) { //nolint (res if args[0] == "--version" { return "crio version 1.13.0", nil } + if args[0] == "config" { + return "# Cgroup management implementation used for the runtime.\ncgroup_manager = \"cgroupfs\"\n", nil + } return "", nil } @@ -225,6 +261,15 @@ func (f *FakeRunner) containerd(args []string, _ bool) (string, error) { // crictl is a fake implementation of crictl func (f *FakeRunner) crictl(args []string, _ bool) (string, error) { switch cmd := args[0]; cmd { + case "info": + return `{ + "status": { + }, + "config": { + "systemdCgroup": false + }, + "golang": "go1.11.13" + }`, nil case "ps": // crictl ps -a --name=apiserver --state=Running --quiet if args[1] == "-a" && strings.HasPrefix(args[2], "--name") { diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 5ef472d73a5c..76ecf397b0ef 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -127,6 +127,17 @@ func (r *Docker) LoadImage(path string) error { } +// CGroupDriver returns cgroup driver ("cgroupfs" or "systemd") +func (r *Docker) CGroupDriver() (string, error) { + // Note: the server daemon has to be running, for this call to return successfully + c := exec.Command("docker", "info", "--format", "'{{.CgroupDriver}}'") + rr, err := r.Runner.RunCmd(c) + if err != nil { + return "", err + } + return strings.Split(rr.Stdout.String(), "\n")[0], nil +} + // KubeletOptions returns kubelet options for a runtime. func (r *Docker) KubeletOptions() map[string]string { return map[string]string{ From 0e83dd4b4e048d5c1c1b255476ccadd34ba8e3d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 12 Jan 2020 19:29:07 +0100 Subject: [PATCH 2/4] Use the cgroup-driver from runtime for kubelet --- pkg/minikube/bootstrapper/bsutil/kubelet.go | 5 +++++ pkg/minikube/bootstrapper/bsutil/kubelet_test.go | 4 +++- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kubelet.go index 5c21698deba2..5046b67bb142 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubelet.go +++ b/pkg/minikube/bootstrapper/bsutil/kubelet.go @@ -41,6 +41,11 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, return nil, errors.Wrap(err, "generating extra configuration for kubelet") } + cgroupDriver, err := r.CGroupDriver() + if err == nil { + extraOpts["cgroup-driver"] = cgroupDriver + } + for k, v := range r.KubeletOptions() { extraOpts[k] = v } diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go index 3c93aa97c2d3..226f4869964d 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubelet_test.go +++ b/pkg/minikube/bootstrapper/bsutil/kubelet_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/pmezard/go-difflib/difflib" + "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" @@ -135,7 +136,8 @@ ExecStart=/var/lib/minikube/binaries/v1.17.0/kubelet --authorization-mode=Webhoo for _, tc := range tests { t.Run(tc.description, func(t *testing.T) { - runtime, err := cruntime.New(cruntime.Config{Type: tc.cfg.ContainerRuntime}) + runtime, err := cruntime.New(cruntime.Config{Type: tc.cfg.ContainerRuntime, + Runner: command.NewFakeCommandRunner()}) if err != nil { t.Fatalf("runtime: %v", err) } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index f5cfe92dc367..cc8a1f83c3f5 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -157,12 +157,13 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { return errors.Wrap(err, "parsing kubernetes version") } - extraFlags := bsutil.CreateFlagsFromExtraArgs(k8s.ExtraOptions) r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime}) if err != nil { return err } + extraFlags := bsutil.CreateFlagsFromExtraArgs(k8s.ExtraOptions) + ignore := []string{ fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestManifestsDir, "/", "-", -1)), fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestPersistentDir, "/", "-", -1)), From f959a21d874ec8c0b8e3e0cdbc8072a0ec773a84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sun, 12 Jan 2020 22:19:28 +0100 Subject: [PATCH 3/4] Make sure to pass runner to UpdateCluster runtime --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index cc8a1f83c3f5..e5ae47d0b9a1 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -369,7 +369,8 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { out.FailureT("Unable to load cached images: {{.error}}", out.V{"error": err}) } } - r, err := cruntime.New(cruntime.Config{Type: cfg.ContainerRuntime, Socket: cfg.KubernetesConfig.CRISocket}) + r, err := cruntime.New(cruntime.Config{Type: cfg.ContainerRuntime, + Runner: k.c, Socket: cfg.KubernetesConfig.CRISocket}) if err != nil { return errors.Wrap(err, "runtime") } From 708bf5c6d5023b759ef9d687dcc3113439e7ba7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Tue, 14 Jan 2020 17:47:56 +0100 Subject: [PATCH 4/4] Revert unnecessary change to the extra flags order --- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index e5ae47d0b9a1..323e5e7609ab 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -157,13 +157,12 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { return errors.Wrap(err, "parsing kubernetes version") } + extraFlags := bsutil.CreateFlagsFromExtraArgs(k8s.ExtraOptions) r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime}) if err != nil { return err } - extraFlags := bsutil.CreateFlagsFromExtraArgs(k8s.ExtraOptions) - ignore := []string{ fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestManifestsDir, "/", "-", -1)), fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestPersistentDir, "/", "-", -1)),