From a2d180ec075146c2a900fcbcc505434e2f4c01e9 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Mon, 20 Apr 2020 17:43:03 -0700
Subject: [PATCH 01/14] Docker driver: force docker to use systemd as cgroup
 manager

Since minikube is running systemd, kubeadm expects kubeadm to be the cgroup manager. If docker is using a different cgroup manager like cgroupfs, this can cause unstable resource allocation. We were seeing this in Cloud Shell, and forcing docker to use systemd resolved the issue.
---
 pkg/minikube/cruntime/containerd.go |  5 +++++
 pkg/minikube/cruntime/crio.go       |  5 +++++
 pkg/minikube/cruntime/cruntime.go   |  2 ++
 pkg/minikube/cruntime/docker.go     | 19 +++++++++++++++++++
 pkg/minikube/node/start.go          |  7 +++++++
 5 files changed, 38 insertions(+)

diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go
index 73261c98190c..1636a1425745 100644
--- a/pkg/minikube/cruntime/containerd.go
+++ b/pkg/minikube/cruntime/containerd.go
@@ -429,3 +429,8 @@ func addRepoTagToImageName(imgName string) string {
 	} // else it already has repo name dont add anything
 	return imgName
 }
+
+// TODO: Enable for containerd
+func (r *Containerd) ForceSystemdCgroupManager() error {
+	return nil
+}
diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go
index 250d765df4fc..a335f45134c8 100644
--- a/pkg/minikube/cruntime/crio.go
+++ b/pkg/minikube/cruntime/crio.go
@@ -229,3 +229,8 @@ func (r *CRIO) Preload(cfg config.KubernetesConfig) error {
 	}
 	return fmt.Errorf("not yet implemented for %s", r.Name())
 }
+
+// ForceSystemdCgroupManager does nothing since CRIO already uses systemd
+func (r *CRIO) ForceSystemdCgroupManager() error {
+	return nil
+}
diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go
index f1866ff1aacc..8b0e231376e2 100644
--- a/pkg/minikube/cruntime/cruntime.go
+++ b/pkg/minikube/cruntime/cruntime.go
@@ -104,6 +104,8 @@ type Manager interface {
 	SystemLogCmd(int) string
 	// Preload preloads the container runtime with k8s images
 	Preload(config.KubernetesConfig) error
+	// ForceSystemdCgroupManager forces the runtime to use systemd as cgroup manager
+	ForceSystemdCgroupManager() error
 }
 
 // Config is runtime configuration
diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
index eb11b58e839f..ce1785e50b31 100644
--- a/pkg/minikube/cruntime/docker.go
+++ b/pkg/minikube/cruntime/docker.go
@@ -365,6 +365,25 @@ func dockerImagesPreloaded(runner command.Runner, images []string) bool {
 	return true
 }
 
+// ForceSystemdCgroupManager forces docker to use systemd as cgroup manager, as recommended in k8s docs:
+// https://kubernetes.io/docs/setup/production-environment/container-runtimes/#docker
+func (r *Docker) ForceSystemdCgroupManager() error {
+	daemonConfig := `{
+"exec-opts": ["native.cgroupdriver=systemd"],
+"log-driver": "json-file",
+"log-opts": {
+	"max-size": "100m"
+},
+"storage-driver": "overlay2"
+}
+`
+	ma := assets.NewMemoryAsset([]byte(daemonConfig), "/etc/docker", "daemon.json", "0644")
+	if err := r.Runner.Copy(ma); err != nil {
+		return errors.Wrap(err, "copying daemon config")
+	}
+	return r.Restart()
+}
+
 func dockerBoundToContainerd(runner command.Runner) bool {
 	// NOTE: assumes systemd
 	rr, err := runner.RunCmd(exec.Command("sudo", "systemctl", "cat", "docker.service"))
diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index 5f2aecb0d96c..2bc18f473619 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -99,6 +99,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) {
 
 		// setup kubeadm (must come after setupKubeconfig)
 		bs = setupKubeAdm(starter.MachineAPI, *starter.Cfg, *starter.Node, starter.Runner)
+
 		err = bs.StartCluster(*starter.Cfg)
 
 		if err != nil {
@@ -261,6 +262,12 @@ func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, k
 		exit.WithError("Failed to enable container runtime", err)
 	}
 
+	if driver.IsKIC(cc.Driver) {
+		if err := cr.ForceSystemdCgroupManager(); err != nil {
+			glog.Warningf("Failed to force %s to use systemd as cgroup manager (this might be ok): %v", cr.Name(), err)
+		}
+	}
+
 	return cr
 }
 

From 80b331c6cdf05c845b3142d77d3206f18e3a7175 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 21 Apr 2020 13:46:17 -0700
Subject: [PATCH 02/14] Set systemd in kicbase Dockerfile

---
 pkg/minikube/cruntime/containerd.go |  5 -----
 pkg/minikube/cruntime/crio.go       |  5 -----
 pkg/minikube/cruntime/cruntime.go   |  2 --
 pkg/minikube/cruntime/docker.go     | 19 -------------------
 pkg/minikube/node/start.go          |  7 -------
 5 files changed, 38 deletions(-)

diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go
index 1636a1425745..73261c98190c 100644
--- a/pkg/minikube/cruntime/containerd.go
+++ b/pkg/minikube/cruntime/containerd.go
@@ -429,8 +429,3 @@ func addRepoTagToImageName(imgName string) string {
 	} // else it already has repo name dont add anything
 	return imgName
 }
-
-// TODO: Enable for containerd
-func (r *Containerd) ForceSystemdCgroupManager() error {
-	return nil
-}
diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go
index a335f45134c8..250d765df4fc 100644
--- a/pkg/minikube/cruntime/crio.go
+++ b/pkg/minikube/cruntime/crio.go
@@ -229,8 +229,3 @@ func (r *CRIO) Preload(cfg config.KubernetesConfig) error {
 	}
 	return fmt.Errorf("not yet implemented for %s", r.Name())
 }
-
-// ForceSystemdCgroupManager does nothing since CRIO already uses systemd
-func (r *CRIO) ForceSystemdCgroupManager() error {
-	return nil
-}
diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go
index 8b0e231376e2..f1866ff1aacc 100644
--- a/pkg/minikube/cruntime/cruntime.go
+++ b/pkg/minikube/cruntime/cruntime.go
@@ -104,8 +104,6 @@ type Manager interface {
 	SystemLogCmd(int) string
 	// Preload preloads the container runtime with k8s images
 	Preload(config.KubernetesConfig) error
-	// ForceSystemdCgroupManager forces the runtime to use systemd as cgroup manager
-	ForceSystemdCgroupManager() error
 }
 
 // Config is runtime configuration
diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
index ce1785e50b31..eb11b58e839f 100644
--- a/pkg/minikube/cruntime/docker.go
+++ b/pkg/minikube/cruntime/docker.go
@@ -365,25 +365,6 @@ func dockerImagesPreloaded(runner command.Runner, images []string) bool {
 	return true
 }
 
-// ForceSystemdCgroupManager forces docker to use systemd as cgroup manager, as recommended in k8s docs:
-// https://kubernetes.io/docs/setup/production-environment/container-runtimes/#docker
-func (r *Docker) ForceSystemdCgroupManager() error {
-	daemonConfig := `{
-"exec-opts": ["native.cgroupdriver=systemd"],
-"log-driver": "json-file",
-"log-opts": {
-	"max-size": "100m"
-},
-"storage-driver": "overlay2"
-}
-`
-	ma := assets.NewMemoryAsset([]byte(daemonConfig), "/etc/docker", "daemon.json", "0644")
-	if err := r.Runner.Copy(ma); err != nil {
-		return errors.Wrap(err, "copying daemon config")
-	}
-	return r.Restart()
-}
-
 func dockerBoundToContainerd(runner command.Runner) bool {
 	// NOTE: assumes systemd
 	rr, err := runner.RunCmd(exec.Command("sudo", "systemctl", "cat", "docker.service"))
diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index 2bc18f473619..5f2aecb0d96c 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -99,7 +99,6 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) {
 
 		// setup kubeadm (must come after setupKubeconfig)
 		bs = setupKubeAdm(starter.MachineAPI, *starter.Cfg, *starter.Node, starter.Runner)
-
 		err = bs.StartCluster(*starter.Cfg)
 
 		if err != nil {
@@ -262,12 +261,6 @@ func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, k
 		exit.WithError("Failed to enable container runtime", err)
 	}
 
-	if driver.IsKIC(cc.Driver) {
-		if err := cr.ForceSystemdCgroupManager(); err != nil {
-			glog.Warningf("Failed to force %s to use systemd as cgroup manager (this might be ok): %v", cr.Name(), err)
-		}
-	}
-
 	return cr
 }
 

From 3ae85a1f627ddfa72128f2930d5805dd9a70bc48 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Tue, 21 Apr 2020 14:17:48 -0700
Subject: [PATCH 03/14] Update kic to v0.0.10, upgrade crio and podman so that
 the kic base image builds

---
 hack/images/daemon.json        |  8 ++++++++
 hack/images/kicbase.Dockerfile | 13 +++++++++++--
 pkg/drivers/kic/types.go       |  4 ++--
 3 files changed, 21 insertions(+), 4 deletions(-)
 create mode 100644 hack/images/daemon.json

diff --git a/hack/images/daemon.json b/hack/images/daemon.json
new file mode 100644
index 000000000000..478546751510
--- /dev/null
+++ b/hack/images/daemon.json
@@ -0,0 +1,8 @@
+{
+    "exec-opts": ["native.cgroupdriver=systemd"],
+    "log-driver": "json-file",
+    "log-opts": {
+      "max-size": "100m"
+    },
+    "storage-driver": "overlay2"
+}
diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index 97a29b73710b..ccb8e38ff65f 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -15,17 +15,26 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     # libglib2.0-0 is required for conmon, which is required for podman
     libglib2.0-0=2.62.1-1 \
     && rm /etc/crictl.yaml
+
 # install cri-o based on https://github.com/cri-o/cri-o/commit/96b0c34b31a9fc181e46d7d8e34fb8ee6c4dc4e1#diff-04c6e90faac2675aa89e2176d2eec7d8R128
+ENV CRIO_VERSION="1.17=1.17.3~2"
 RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \    
     curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \
     apt-key add - < Release.key && apt-get update && \
-    apt-get install -y --no-install-recommends cri-o-1.17=1.17.2~1
+    apt-get install -y --no-install-recommends cri-o-${CRIO_VERSION}
+
 # install podman
-RUN apt-get install -y --no-install-recommends podman=1.8.2~144
+ENV PODMAN_VERSION=1.9.0~2
+RUN apt-get install -y --no-install-recommends podman=${PODMAN_VERSION}
+
 # disable non-docker runtimes by default
 RUN systemctl disable containerd && systemctl disable crio && rm /etc/crictl.yaml
+
+# copy over docker daemon config which sets cgroup manager to systemd
+COPY hack/images/daemon.json /etc/docker/daemon.json
 # enable docker which is default
 RUN systemctl enable docker
+
 # making SSH work for docker container 
 # based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile
 RUN mkdir /var/run/sshd
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index 2b01019b2ed8..b3840b45d328 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -30,9 +30,9 @@ const (
 	DefaultPodCIDR = "10.244.0.0/16"
 
 	// Version is the current version of kic
-	Version = "v0.0.9"
+	Version = "v0.0.10"
 	// SHA of the kic base image
-	baseImageSHA = "82a826cc03c3e59ead5969b8020ca138de98f366c1907293df91fc57205dbb53"
+	baseImageSHA = "d591f4ef60f4c48f2d6fddfefa88213e81d8be505d3b663cc764326a59cd0c31"
 	// OverlayImage is the cni plugin used for overlay image, created by kind.
 	// CNI plugin image used for kic drivers created by kind.
 	OverlayImage = "kindest/kindnetd:0.5.4"

From ae388d0edb43a49a1fb86fc92adbd4449f3b6cd1 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Wed, 22 Apr 2020 14:10:16 -0700
Subject: [PATCH 04/14] Add --force-systemd flag to force container runtime to
 use systemd

---
 cmd/minikube/cmd/start_flags.go     |  2 ++
 pkg/minikube/cruntime/containerd.go |  5 +++++
 pkg/minikube/cruntime/crio.go       |  5 +++++
 pkg/minikube/cruntime/cruntime.go   |  2 ++
 pkg/minikube/cruntime/docker.go     | 23 +++++++++++++++++++++++
 5 files changed, 37 insertions(+)

diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go
index c735e8025d67..af42ee7c5987 100644
--- a/cmd/minikube/cmd/start_flags.go
+++ b/cmd/minikube/cmd/start_flags.go
@@ -98,6 +98,7 @@ const (
 	nodes                   = "nodes"
 	preload                 = "preload"
 	deleteOnFailure         = "delete-on-failure"
+	forceSystemd            = "force-systemd"
 )
 
 // initMinikubeFlags includes commandline flags for minikube.
@@ -135,6 +136,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().IntP(nodes, "n", 1, "The number of nodes to spin up. Defaults to 1.")
 	startCmd.Flags().Bool(preload, true, "If set, download tarball of preloaded images if available to improve start time. Defaults to true.")
 	startCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.")
+	startCmd.Flags().Bool(forceSystemd, false, "If set, force the container runtime to use sytemd as cgroup manager. Currently avaiable for docker and crio. Defaults to false.")
 }
 
 // initKubernetesFlags inits the commandline flags for kubernetes related options
diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go
index 73261c98190c..570407b61a82 100644
--- a/pkg/minikube/cruntime/containerd.go
+++ b/pkg/minikube/cruntime/containerd.go
@@ -429,3 +429,8 @@ func addRepoTagToImageName(imgName string) string {
 	} // else it already has repo name dont add anything
 	return imgName
 }
+
+// TODO: Implement for containerd
+func (r *Containerd) ForceSystemd() error {
+	return nil
+}
diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go
index 250d765df4fc..4ff52cc37ea1 100644
--- a/pkg/minikube/cruntime/crio.go
+++ b/pkg/minikube/cruntime/crio.go
@@ -229,3 +229,8 @@ func (r *CRIO) Preload(cfg config.KubernetesConfig) error {
 	}
 	return fmt.Errorf("not yet implemented for %s", r.Name())
 }
+
+// ForceSystemd does nothing for crio since it already uses systemd as cgroup manager
+func (r *CRIO) ForceSystemd() error {
+	return nil
+}
diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go
index f1866ff1aacc..3a5ad43d8964 100644
--- a/pkg/minikube/cruntime/cruntime.go
+++ b/pkg/minikube/cruntime/cruntime.go
@@ -104,6 +104,8 @@ type Manager interface {
 	SystemLogCmd(int) string
 	// Preload preloads the container runtime with k8s images
 	Preload(config.KubernetesConfig) error
+	// ForceSystemd forces the container runtime to use systemd as cgroup manager
+	ForceSystemd() error
 }
 
 // Config is runtime configuration
diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
index eb11b58e839f..eb4c8030dfd7 100644
--- a/pkg/minikube/cruntime/docker.go
+++ b/pkg/minikube/cruntime/docker.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
+	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/command"
@@ -110,6 +111,10 @@ func (r *Docker) Enable(disOthers bool) error {
 		}
 	}
 
+	if err := r.ForceSystemd(); err != nil {
+		return errors.Wrap(err, "forcing systemd")
+	}
+
 	return r.Init.Start("docker")
 }
 
@@ -274,6 +279,24 @@ func (r *Docker) SystemLogCmd(len int) string {
 	return fmt.Sprintf("sudo journalctl -u docker -n %d", len)
 }
 
+// ForceSystemd forces the docker daemon to use systemd as cgroup manager
+func (r *Docker) ForceSystemd() error {
+	if !viper.GetBool("force-systemd") {
+		return nil
+	}
+	daemonConfig := `{
+"exec-opts": ["native.cgroupdriver=systemd"],
+"log-driver": "json-file",
+"log-opts": {
+	"max-size": "100m"
+},
+"storage-driver": "overlay2"
+}
+`
+	ma := assets.NewMemoryAsset([]byte(daemonConfig), "/etc/docker", "daemon.json", "0644")
+	return r.Runner.Copy(ma)
+}
+
 // Preload preloads docker with k8s images:
 // 1. Copy over the preloaded tarball into the VM
 // 2. Extract the preloaded tarball to the correct directory

From 1f2b32c4edbe9d18a567b01dd204276b3402827e Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Wed, 22 Apr 2020 14:11:23 -0700
Subject: [PATCH 05/14] Update docs

---
 site/content/en/docs/commands/start.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md
index 7f2a0b837e2c..a1b0c098fee1 100644
--- a/site/content/en/docs/commands/start.md
+++ b/site/content/en/docs/commands/start.md
@@ -49,6 +49,7 @@ minikube start [flags]
                                           		Valid kubeadm parameters: ignore-preflight-errors, dry-run, kubeconfig, kubeconfig-dir, node-name, cri-socket, experimental-upload-certs, certificate-key, rootfs, skip-phases, pod-network-cidr
       --feature-gates string              A set of key=value pairs that describe feature gates for alpha/experimental features.
       --force                             Force minikube to perform possibly dangerous operations
+      --force-systemd                     If set, force the container runtime to use sytemd as cgroup manager. Currently avaiable for docker and crio. Defaults to false.
   -h, --help                              help for start
       --host-dns-resolver                 Enable host resolver for NAT DNS requests (virtualbox driver only) (default true)
       --host-only-cidr string             The CIDR to be used for the minikube VM (virtualbox driver only) (default "192.168.99.1/24")

From 56fa90fd8763dc5f14cd51d1579e54488c9de679 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Wed, 22 Apr 2020 14:19:26 -0700
Subject: [PATCH 06/14] Add integration test

---
 hack/images/kicbase.Dockerfile  | 13 ++-----------
 test/integration/docker_test.go | 27 +++++++++++++++++++++++++++
 2 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/hack/images/kicbase.Dockerfile b/hack/images/kicbase.Dockerfile
index ccb8e38ff65f..97a29b73710b 100644
--- a/hack/images/kicbase.Dockerfile
+++ b/hack/images/kicbase.Dockerfile
@@ -15,26 +15,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     # libglib2.0-0 is required for conmon, which is required for podman
     libglib2.0-0=2.62.1-1 \
     && rm /etc/crictl.yaml
-
 # install cri-o based on https://github.com/cri-o/cri-o/commit/96b0c34b31a9fc181e46d7d8e34fb8ee6c4dc4e1#diff-04c6e90faac2675aa89e2176d2eec7d8R128
-ENV CRIO_VERSION="1.17=1.17.3~2"
 RUN sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_19.10/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" && \    
     curl -LO https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_19.10/Release.key && \
     apt-key add - < Release.key && apt-get update && \
-    apt-get install -y --no-install-recommends cri-o-${CRIO_VERSION}
-
+    apt-get install -y --no-install-recommends cri-o-1.17=1.17.2~1
 # install podman
-ENV PODMAN_VERSION=1.9.0~2
-RUN apt-get install -y --no-install-recommends podman=${PODMAN_VERSION}
-
+RUN apt-get install -y --no-install-recommends podman=1.8.2~144
 # disable non-docker runtimes by default
 RUN systemctl disable containerd && systemctl disable crio && rm /etc/crictl.yaml
-
-# copy over docker daemon config which sets cgroup manager to systemd
-COPY hack/images/daemon.json /etc/docker/daemon.json
 # enable docker which is default
 RUN systemctl enable docker
-
 # making SSH work for docker container 
 # based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile
 RUN mkdir /var/run/sshd
diff --git a/test/integration/docker_test.go b/test/integration/docker_test.go
index dd0a27de06af..25d0da8eb4cd 100644
--- a/test/integration/docker_test.go
+++ b/test/integration/docker_test.go
@@ -63,3 +63,30 @@ func TestDockerFlags(t *testing.T) {
 		}
 	}
 }
+
+func TestForceSystemd(t *testing.T) {
+	if NoneDriver() {
+		t.Skip("skipping: none driver does not support ssh or bundle docker")
+	}
+	MaybeParallel(t)
+
+	profile := UniqueProfileName("force-systemd")
+	ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
+	defer CleanupWithLogs(t, profile, cancel)
+
+	// Use the most verbose logging for the simplest test. If it fails, something is very wrong.
+	args := append([]string{"start", "-p", profile, "--force-systemd", "--alsologtostderr", "-v=5"}, StartArgs()...)
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
+	if err != nil {
+		t.Errorf("failed to start minikube with args: %q : %v", rr.Command(), err)
+	}
+
+	rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "docker info --format {{.CgroupDriver}}"))
+	if err != nil {
+		t.Errorf("failed to get docker cgroup driver. args %q: %v", rr.Command(), err)
+	}
+
+	if !strings.Contains(rr.Output(), "systemd") {
+		t.Fatalf("expected systemd cgroup driver, got: %v", rr.Output())
+	}
+}

From 9bba297cf153261633dab20a0dee4f0de2ff5d1b Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Wed, 22 Apr 2020 14:19:33 -0700
Subject: [PATCH 07/14] revert unneeded changes

---
 hack/images/daemon.json  | 8 --------
 pkg/drivers/kic/types.go | 4 ++--
 2 files changed, 2 insertions(+), 10 deletions(-)
 delete mode 100644 hack/images/daemon.json

diff --git a/hack/images/daemon.json b/hack/images/daemon.json
deleted file mode 100644
index 478546751510..000000000000
--- a/hack/images/daemon.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-    "exec-opts": ["native.cgroupdriver=systemd"],
-    "log-driver": "json-file",
-    "log-opts": {
-      "max-size": "100m"
-    },
-    "storage-driver": "overlay2"
-}
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
index b3840b45d328..2b01019b2ed8 100644
--- a/pkg/drivers/kic/types.go
+++ b/pkg/drivers/kic/types.go
@@ -30,9 +30,9 @@ const (
 	DefaultPodCIDR = "10.244.0.0/16"
 
 	// Version is the current version of kic
-	Version = "v0.0.10"
+	Version = "v0.0.9"
 	// SHA of the kic base image
-	baseImageSHA = "d591f4ef60f4c48f2d6fddfefa88213e81d8be505d3b663cc764326a59cd0c31"
+	baseImageSHA = "82a826cc03c3e59ead5969b8020ca138de98f366c1907293df91fc57205dbb53"
 	// OverlayImage is the cni plugin used for overlay image, created by kind.
 	// CNI plugin image used for kic drivers created by kind.
 	OverlayImage = "kindest/kindnetd:0.5.4"

From e5d633df9f9c94527fba94e730f240ac8c7a8f11 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Wed, 22 Apr 2020 14:51:36 -0700
Subject: [PATCH 08/14] fix lint

---
 cmd/minikube/cmd/start_flags.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go
index af42ee7c5987..144f8caf95d8 100644
--- a/cmd/minikube/cmd/start_flags.go
+++ b/cmd/minikube/cmd/start_flags.go
@@ -136,7 +136,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().IntP(nodes, "n", 1, "The number of nodes to spin up. Defaults to 1.")
 	startCmd.Flags().Bool(preload, true, "If set, download tarball of preloaded images if available to improve start time. Defaults to true.")
 	startCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.")
-	startCmd.Flags().Bool(forceSystemd, false, "If set, force the container runtime to use sytemd as cgroup manager. Currently avaiable for docker and crio. Defaults to false.")
+	startCmd.Flags().Bool(forceSystemd, false, "If set, force the container runtime to use sytemd as cgroup manager. Currently available for docker and crio. Defaults to false.")
 }
 
 // initKubernetesFlags inits the commandline flags for kubernetes related options

From 045b1e9a3ac9e9d213037d1d2e46d0867185abf1 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Thu, 23 Apr 2020 15:02:14 -0700
Subject: [PATCH 09/14] update docs

---
 site/content/en/docs/commands/start.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md
index 5f1e19b23ec1..ede8a83f0ba3 100644
--- a/site/content/en/docs/commands/start.md
+++ b/site/content/en/docs/commands/start.md
@@ -50,7 +50,7 @@ minikube start [flags]
                                           		Valid kubeadm parameters: ignore-preflight-errors, dry-run, kubeconfig, kubeconfig-dir, node-name, cri-socket, experimental-upload-certs, certificate-key, rootfs, skip-phases, pod-network-cidr
       --feature-gates string              A set of key=value pairs that describe feature gates for alpha/experimental features.
       --force                             Force minikube to perform possibly dangerous operations
-      --force-systemd                     If set, force the container runtime to use sytemd as cgroup manager. Currently avaiable for docker and crio. Defaults to false.
+      --force-systemd                     If set, force the container runtime to use sytemd as cgroup manager. Currently available for docker and crio. Defaults to false.
   -h, --help                              help for start
       --host-dns-resolver                 Enable host resolver for NAT DNS requests (virtualbox driver only) (default true)
       --host-only-cidr string             The CIDR to be used for the minikube VM (virtualbox driver only) (default "192.168.99.1/24")

From 1087f56b7a674e17d1a284938d7e3eb357d1eed9 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Thu, 23 Apr 2020 15:11:40 -0700
Subject: [PATCH 10/14] Pass forceSystemd into cRuntime lib to avoid errors
 with cobra

---
 pkg/minikube/cruntime/containerd.go |  7 ++++++-
 pkg/minikube/cruntime/crio.go       |  7 ++++++-
 pkg/minikube/cruntime/cruntime.go   |  2 +-
 pkg/minikube/cruntime/docker.go     | 12 +++++-------
 pkg/minikube/node/start.go          |  2 +-
 5 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go
index 570407b61a82..864f70b067c2 100644
--- a/pkg/minikube/cruntime/containerd.go
+++ b/pkg/minikube/cruntime/containerd.go
@@ -197,12 +197,17 @@ func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semve
 }
 
 // Enable idempotently enables containerd on a host
-func (r *Containerd) Enable(disOthers bool) error {
+func (r *Containerd) Enable(disOthers, forceSystemd bool) error {
 	if disOthers {
 		if err := disableOthers(r, r.Runner); err != nil {
 			glog.Warningf("disableOthers: %v", err)
 		}
 	}
+	if forceSystemd {
+		if err := r.ForceSystemd(); err != nil {
+			return err
+		}
+	}
 	if err := populateCRIConfig(r.Runner, r.SocketPath()); err != nil {
 		return err
 	}
diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go
index 4ff52cc37ea1..b9ab11745997 100644
--- a/pkg/minikube/cruntime/crio.go
+++ b/pkg/minikube/cruntime/crio.go
@@ -110,12 +110,17 @@ func (r *CRIO) Active() bool {
 }
 
 // Enable idempotently enables CRIO on a host
-func (r *CRIO) Enable(disOthers bool) error {
+func (r *CRIO) Enable(disOthers, forceSystemd bool) error {
 	if disOthers {
 		if err := disableOthers(r, r.Runner); err != nil {
 			glog.Warningf("disableOthers: %v", err)
 		}
 	}
+	if forceSystemd {
+		if err := r.ForceSystemd(); err != nil {
+			return err
+		}
+	}
 	if err := populateCRIConfig(r.Runner, r.SocketPath()); err != nil {
 		return err
 	}
diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go
index 3a5ad43d8964..212709ee5b52 100644
--- a/pkg/minikube/cruntime/cruntime.go
+++ b/pkg/minikube/cruntime/cruntime.go
@@ -63,7 +63,7 @@ type Manager interface {
 	// Version retrieves the current version of this runtime
 	Version() (string, error)
 	// Enable idempotently enables this runtime on a host
-	Enable(bool) error
+	Enable(bool, bool) error
 	// Disable idempotently disables this runtime on a host
 	Disable() error
 	// Active returns whether or not a runtime is active on a host
diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
index eb4c8030dfd7..7f51fb3a3df9 100644
--- a/pkg/minikube/cruntime/docker.go
+++ b/pkg/minikube/cruntime/docker.go
@@ -25,7 +25,6 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
-	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/command"
@@ -104,15 +103,17 @@ func (r *Docker) Active() bool {
 }
 
 // Enable idempotently enables Docker on a host
-func (r *Docker) Enable(disOthers bool) error {
+func (r *Docker) Enable(disOthers, forceSystemd bool) error {
 	if disOthers {
 		if err := disableOthers(r, r.Runner); err != nil {
 			glog.Warningf("disableOthers: %v", err)
 		}
 	}
 
-	if err := r.ForceSystemd(); err != nil {
-		return errors.Wrap(err, "forcing systemd")
+	if forceSystemd {
+		if err := r.ForceSystemd(); err != nil {
+			return err
+		}
 	}
 
 	return r.Init.Start("docker")
@@ -281,9 +282,6 @@ func (r *Docker) SystemLogCmd(len int) string {
 
 // ForceSystemd forces the docker daemon to use systemd as cgroup manager
 func (r *Docker) ForceSystemd() error {
-	if !viper.GetBool("force-systemd") {
-		return nil
-	}
 	daemonConfig := `{
 "exec-opts": ["native.cgroupdriver=systemd"],
 "log-driver": "json-file",
diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go
index 5f2aecb0d96c..aa6a908e0b0b 100644
--- a/pkg/minikube/node/start.go
+++ b/pkg/minikube/node/start.go
@@ -255,7 +255,7 @@ func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, k
 		}
 	}
 
-	err = cr.Enable(disableOthers)
+	err = cr.Enable(disableOthers, viper.GetBool("force-systemd"))
 	if err != nil {
 		debug.PrintStack()
 		exit.WithError("Failed to enable container runtime", err)

From 746d225bfbd3d6f51165b47430e8c23d7dc135ba Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Thu, 23 Apr 2020 15:15:57 -0700
Subject: [PATCH 11/14] remove unused code

---
 pkg/minikube/cruntime/containerd.go | 7 +------
 pkg/minikube/cruntime/crio.go       | 7 +------
 2 files changed, 2 insertions(+), 12 deletions(-)

diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go
index 864f70b067c2..4694a58edb50 100644
--- a/pkg/minikube/cruntime/containerd.go
+++ b/pkg/minikube/cruntime/containerd.go
@@ -197,17 +197,12 @@ func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semve
 }
 
 // Enable idempotently enables containerd on a host
-func (r *Containerd) Enable(disOthers, forceSystemd bool) error {
+func (r *Containerd) Enable(disOthers, _ bool) error {
 	if disOthers {
 		if err := disableOthers(r, r.Runner); err != nil {
 			glog.Warningf("disableOthers: %v", err)
 		}
 	}
-	if forceSystemd {
-		if err := r.ForceSystemd(); err != nil {
-			return err
-		}
-	}
 	if err := populateCRIConfig(r.Runner, r.SocketPath()); err != nil {
 		return err
 	}
diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go
index b9ab11745997..015b723ae032 100644
--- a/pkg/minikube/cruntime/crio.go
+++ b/pkg/minikube/cruntime/crio.go
@@ -110,17 +110,12 @@ func (r *CRIO) Active() bool {
 }
 
 // Enable idempotently enables CRIO on a host
-func (r *CRIO) Enable(disOthers, forceSystemd bool) error {
+func (r *CRIO) Enable(disOthers, _ bool) error {
 	if disOthers {
 		if err := disableOthers(r, r.Runner); err != nil {
 			glog.Warningf("disableOthers: %v", err)
 		}
 	}
-	if forceSystemd {
-		if err := r.ForceSystemd(); err != nil {
-			return err
-		}
-	}
 	if err := populateCRIConfig(r.Runner, r.SocketPath()); err != nil {
 		return err
 	}

From cfb1d2198202b2685aa38a70b7d7dd0f4180de24 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Thu, 23 Apr 2020 16:10:20 -0700
Subject: [PATCH 12/14] fix lint

---
 hack/preload-images/generate.go        | 2 +-
 pkg/minikube/cruntime/cruntime_test.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go
index daa9e77f092a..54c680bc480c 100644
--- a/hack/preload-images/generate.go
+++ b/hack/preload-images/generate.go
@@ -87,7 +87,7 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string
 	if err != nil {
 		return errors.Wrap(err, "failed create new runtime")
 	}
-	if err := cr.Enable(true); err != nil {
+	if err := cr.Enable(true, false); err != nil {
 		return errors.Wrap(err, "enable container runtime")
 	}
 
diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go
index aef420cf8496..c2a5b4324d56 100644
--- a/pkg/minikube/cruntime/cruntime_test.go
+++ b/pkg/minikube/cruntime/cruntime_test.go
@@ -581,7 +581,7 @@ func TestEnable(t *testing.T) {
 			if err != nil {
 				t.Fatalf("New(%s): %v", tc.runtime, err)
 			}
-			err = cr.Enable(true)
+			err = cr.Enable(true, false)
 			if err != nil {
 				t.Errorf("%s disable unexpected error: %v", tc.runtime, err)
 			}

From 52b57374fcc02d9b17ae62bc4f499a63d617fd44 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Thu, 23 Apr 2020 16:56:30 -0700
Subject: [PATCH 13/14] make forceSystemd private

---
 pkg/minikube/cruntime/containerd.go | 5 -----
 pkg/minikube/cruntime/crio.go       | 5 -----
 pkg/minikube/cruntime/cruntime.go   | 2 --
 pkg/minikube/cruntime/docker.go     | 4 ++--
 4 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go
index 4694a58edb50..9dabf8a4668c 100644
--- a/pkg/minikube/cruntime/containerd.go
+++ b/pkg/minikube/cruntime/containerd.go
@@ -429,8 +429,3 @@ func addRepoTagToImageName(imgName string) string {
 	} // else it already has repo name dont add anything
 	return imgName
 }
-
-// TODO: Implement for containerd
-func (r *Containerd) ForceSystemd() error {
-	return nil
-}
diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go
index 015b723ae032..8caa5cda972e 100644
--- a/pkg/minikube/cruntime/crio.go
+++ b/pkg/minikube/cruntime/crio.go
@@ -229,8 +229,3 @@ func (r *CRIO) Preload(cfg config.KubernetesConfig) error {
 	}
 	return fmt.Errorf("not yet implemented for %s", r.Name())
 }
-
-// ForceSystemd does nothing for crio since it already uses systemd as cgroup manager
-func (r *CRIO) ForceSystemd() error {
-	return nil
-}
diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go
index 212709ee5b52..3ba208a47a81 100644
--- a/pkg/minikube/cruntime/cruntime.go
+++ b/pkg/minikube/cruntime/cruntime.go
@@ -104,8 +104,6 @@ type Manager interface {
 	SystemLogCmd(int) string
 	// Preload preloads the container runtime with k8s images
 	Preload(config.KubernetesConfig) error
-	// ForceSystemd forces the container runtime to use systemd as cgroup manager
-	ForceSystemd() error
 }
 
 // Config is runtime configuration
diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
index 7f51fb3a3df9..882a4a18d89b 100644
--- a/pkg/minikube/cruntime/docker.go
+++ b/pkg/minikube/cruntime/docker.go
@@ -111,7 +111,7 @@ func (r *Docker) Enable(disOthers, forceSystemd bool) error {
 	}
 
 	if forceSystemd {
-		if err := r.ForceSystemd(); err != nil {
+		if err := r.forceSystemd(); err != nil {
 			return err
 		}
 	}
@@ -281,7 +281,7 @@ func (r *Docker) SystemLogCmd(len int) string {
 }
 
 // ForceSystemd forces the docker daemon to use systemd as cgroup manager
-func (r *Docker) ForceSystemd() error {
+func (r *Docker) forceSystemd() error {
 	daemonConfig := `{
 "exec-opts": ["native.cgroupdriver=systemd"],
 "log-driver": "json-file",

From 18255467574c7da109aec80d20253b77775b78b5 Mon Sep 17 00:00:00 2001
From: Priya Wadhwa <priyawadhwa@google.com>
Date: Fri, 24 Apr 2020 14:05:35 -0700
Subject: [PATCH 14/14] Restart docker if we've changed the daemon config

---
 pkg/minikube/cruntime/docker.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go
index 882a4a18d89b..bbc5b7ffcf1a 100644
--- a/pkg/minikube/cruntime/docker.go
+++ b/pkg/minikube/cruntime/docker.go
@@ -114,6 +114,7 @@ func (r *Docker) Enable(disOthers, forceSystemd bool) error {
 		if err := r.forceSystemd(); err != nil {
 			return err
 		}
+		return r.Init.Restart("docker")
 	}
 
 	return r.Init.Start("docker")
@@ -282,6 +283,7 @@ func (r *Docker) SystemLogCmd(len int) string {
 
 // ForceSystemd forces the docker daemon to use systemd as cgroup manager
 func (r *Docker) forceSystemd() error {
+	glog.Infof("Forcing docker to use systemd as cgroup manager...")
 	daemonConfig := `{
 "exec-opts": ["native.cgroupdriver=systemd"],
 "log-driver": "json-file",