Update gvisor runsc version (refs #4482)

- Updates the gvisor addon to use containerd shim v2
- Updates the version of runsc
- Auto-installs a gvisor RuntimeClass

deploy/addons/gvisor/README.md

@@ -1,5 +1,5 @@
 ## gVisor Addon
-[gVisor](https://github.com/google/gvisor/blob/master/README.md), a sandboxed container runtime, allows users to securely run pods with untrusted workloads within Minikube.
+[gVisor](https://gvisor.dev/), a sandboxed container runtime, allows users to securely run pods with untrusted workloads within Minikube.
 
 ### Starting Minikube
 gVisor depends on the containerd runtime to run in Minikube.
@@ -17,21 +17,27 @@ To enable this addon, simply run:
 $ minikube addons enable gvisor
 ```
 
-Within one minute, the addon manager should pick up the change and you should see the `gvisor` pod:
+Within one minute, the addon manager should pick up the change and you should
+see the `gvisor` pod and `gvisor` [Runtime Class](https://kubernetes.io/docs/concepts/containers/runtime-class/):
 
 ```
-$ kubectl get pod gvisor -n kube-system
-NAME      READY     STATUS    RESTARTS   AGE
-gvisor    1/1       Running   0          3m
+$ kubectl get pod,runtimeclass gvisor -n kube-system
+NAME         READY   STATUS    RESTARTS   AGE
+pod/gvisor   1/1     Running   0          2m52s
+
+NAME                              CREATED AT
+runtimeclass.node.k8s.io/gvisor   2019-06-15T04:35:09Z
 ```
 
-Once the pod has status `Running`, gVisor is enabled in Minikube. 
+Once the pod has status `Running`, gVisor is enabled in Minikube.
 
 ### Running pods in gVisor
-To run a pod in gVisor, add this annotation to the Kubernetes yaml:
+
+To run a pod in gVisor, add the `gvisor` runtime class to the Pod spec in your
+Kubernetes yaml:
 
 ```
-io.kubernetes.cri.untrusted-workload: "true"
+runtimeClassName: gvisor
 ```
 
 An example Pod is shown below:
@@ -41,17 +47,15 @@ apiVersion: v1
 kind: Pod
 metadata:
   name: nginx-untrusted
-  annotations:
-    io.kubernetes.cri.untrusted-workload: "true"
 spec:
+  runtimeClassName: gvisor
   containers:
   - name: nginx
     image: nginx
 ```
 
-_Note: this annotation will not be necessary once the RuntimeClass Kubernetes feature is available broadly._
-
 ### Disabling gVisor
+
 To disable gVisor, run:
 
 ```
@@ -67,4 +71,4 @@ NAME      READY     STATUS        RESTARTS   AGE
 gvisor    1/1       Terminating   0          5m
 ```
 
-_Note: Once gVisor is disabled, any pod with the `io.kubernetes.cri.untrusted-workload` annotation will fail with a FailedCreatePodSandBox error._
+_Note: Once gVisor is disabled, any pod with the `gvisor` Runtime Class or `io.kubernetes.cri.untrusted-workload` annotation will fail with a FailedCreatePodSandBox error._

deploy/addons/gvisor/gvisor-config.toml

@@ -41,10 +41,10 @@ oom_score = 0
         runtime_type = "io.containerd.runtime.v1.linux"
         runtime_engine = ""
         runtime_root = ""
-      [plugins.cri.containerd.untrusted_workload_runtime]
-        runtime_type = "io.containerd.runtime.v1.linux"
-        runtime_engine = "/usr/local/bin/runsc"
-        runtime_root = "/run/containerd/runsc"
+      [plugins.cri.containerd.runtimes.untrusted]
+        runtime_type = "io.containerd.runsc.v1"
+      [plugins.cri.containerd.runtimes.runsc]
+        runtime_type = "io.containerd.runsc.v1"
     [plugins.cri.cni]
       bin_dir = "/opt/cni/bin"
       conf_dir = "/etc/cni/net.d"
@@ -56,7 +56,6 @@ oom_score = 0
   [plugins.diff-service]
     default = ["walking"]
   [plugins.linux]
-    shim = "gvisor-containerd-shim"
     runtime = "runc"
     runtime_root = ""
     no_shim = false
@@ -66,4 +65,4 @@ oom_score = 0
     deletion_threshold = 0
     mutation_threshold = 100
     schedule_delay = "0s"
-    startup_delay = "100ms"
+    startup_delay = "100ms"

deploy/addons/gvisor/gvisor-pod.yaml.tmpl

@@ -14,15 +14,15 @@
 
 apiVersion: v1
 kind: Pod
-metadata: 
+metadata:
   name: gvisor
   namespace: kube-system
   labels:
     addonmanager.kubernetes.io/mode: Reconcile
     kubernetes.io/minikube-addons: gvisor
-spec: 
+spec:
   hostPID: true
-  containers: 
+  containers:
     - name: gvisor
       image: {{default "gcr.io/k8s-minikube" .ImageRepository}}/gvisor-addon:latest
       securityContext:

deploy/addons/gvisor/gvisor-runtimeclass.yaml

@@ -0,0 +1,22 @@
+# Copyright 2018 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: node.k8s.io/v1beta1
+kind: RuntimeClass
+metadata:
+  name: gvisor
+  labels:
+    kubernetes.io/minikube-addons: gvisor
+    addonmanager.kubernetes.io/mode: Reconcile
+handler: runsc

pkg/gvisor/enable.go

@@ -80,12 +80,6 @@ func makeGvisorDirs() error {
 		return errors.Wrap(err, "creating runsc dir")
 	}
 
-	// Make /usr/local/bin to store the runsc binary
-	fp = filepath.Join(nodeDir, "usr/local/bin")
-	if err := os.MkdirAll(fp, 0755); err != nil {
-		return errors.Wrap(err, "creating usr/local/bin dir")
-	}
-
 	// Make /tmp/runsc to also hold logs
 	fp = filepath.Join(nodeDir, "tmp/runsc")
 	if err := os.MkdirAll(fp, 0755); err != nil {
@@ -107,13 +101,13 @@ func downloadBinaries() error {
 
 // downloads the gvisor-containerd-shim
 func gvisorContainerdShim() error {
-	dest := filepath.Join(nodeDir, "usr/bin/gvisor-containerd-shim")
+	dest := filepath.Join(nodeDir, "usr/bin/containerd-shim-runsc-v1")
 	return downloadFileToDest(constants.GvisorContainerdShimURL, dest)
 }
 
 // downloads the runsc binary and returns a path to the binary
 func runsc() error {
-	dest := filepath.Join(nodeDir, "usr/local/bin/runsc")
+	dest := filepath.Join(nodeDir, "usr/bin/runsc")
 	return downloadFileToDest(constants.GvisorURL, dest)
 }
 
@@ -159,10 +153,6 @@ func copyConfigFiles() error {
 	if err := mcnutils.CopyFile(filepath.Join(nodeDir, constants.ContainerdConfigTomlPath), filepath.Join(nodeDir, constants.StoredContainerdConfigTomlPath)); err != nil {
 		return errors.Wrap(err, "copying default config.toml")
 	}
-	log.Print("Copying gvisor-containerd-shim.toml...")
-	if err := copyAssetToDest(constants.GvisorContainerdShimTargetName, filepath.Join(nodeDir, constants.GvisorContainerdShimTomlPath)); err != nil {
-		return errors.Wrap(err, "copying gvisor-containerd-shim.toml")
-	}
 	log.Print("Copying containerd config.toml with gvisor...")
 	if err := copyAssetToDest(constants.GvisorConfigTomlTargetName, filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil {
 		return errors.Wrap(err, "copying gvisor version of config.toml")

pkg/minikube/assets/addons.go

@@ -320,18 +320,18 @@ var Addons = map[string]*Addon{
 			"gvisor-pod.yaml",
 			"0640",
 			true),
+		MustBinAsset(
+			"deploy/addons/gvisor/gvisor-runtimeclass.yaml",
+			constants.AddonsPath,
+			"gvisor-runtimeclass.yaml",
+			"0640",
+			false),
 		MustBinAsset(
 			"deploy/addons/gvisor/gvisor-config.toml",
 			constants.GvisorFilesPath,
 			constants.GvisorConfigTomlTargetName,
 			"0640",
 			true),
-		MustBinAsset(
-			"deploy/addons/gvisor/gvisor-containerd-shim.toml",
-			constants.GvisorFilesPath,
-			constants.GvisorContainerdShimTargetName,
-			"0640",
-			false),
 	}, false, "gvisor"),
 }
 

pkg/minikube/constants/constants.go

@@ -414,18 +414,14 @@ const (
 	GvisorFilesPath = "/tmp/gvisor"
 	// ContainerdConfigTomlPath is the path to the containerd config.toml
 	ContainerdConfigTomlPath = "/etc/containerd/config.toml"
-	// GvisorContainerdShimTomlPath is the path to gvisor-containerd-shim.toml
-	GvisorContainerdShimTomlPath = "/etc/containerd/gvisor-containerd-shim.toml"
 	// StoredContainerdConfigTomlPath is the path where the default config.toml will be stored
 	StoredContainerdConfigTomlPath = "/tmp/config.toml"
 
 	//GvisorConfigTomlTargetName is the go-bindata target name for the gvisor config.toml
 	GvisorConfigTomlTargetName = "gvisor-config.toml"
-	// GvisorContainerdShimTargetName is the go-bindata target name for gvisor-containerd-shim
-	GvisorContainerdShimTargetName = "gvisor-containerd-shim.toml"
 
 	// GvisorContainerdShimURL is the url to download gvisor-containerd-shim
-	GvisorContainerdShimURL = "https://github.com/google/gvisor-containerd-shim/releases/download/v0.0.1-rc.0/gvisor-containerd-shim-v0.0.1-rc.0.linux-amd64"
+	GvisorContainerdShimURL = "https://github.com/google/gvisor-containerd-shim/releases/download/v0.0.3/containerd-shim-runsc-v1.linux-amd64"
 	// GvisorURL is the url to download gvisor
-	GvisorURL = "https://storage.googleapis.com/gvisor/releases/nightly/2018-12-07/runsc"
+	GvisorURL = "https://storage.googleapis.com/gvisor/releases/nightly/2019-01-14/runsc"
 )

pkg/minikube/extract/testdata/test.json

@@ -1,4 +1,7 @@
 {
+	"Hint: This is not a URL, come on.": "",
 	"Holy cow I'm in a loop!": "Something else",
-	"This was a choice: %s": "Something"
+	"This is a variable with a string assigned": "",
+	"This was a choice: %s": "Something",
+	"Wow another string: %s": ""
 }

test/integration/addons_test.go

@@ -191,7 +191,7 @@ func testServicesList(t *testing.T) {
 	}
 }
 
-func testGvisor(t *testing.T) {
+func testGvisorUntrustedWorkload(t *testing.T) {
 	minikubeRunner := NewMinikubeRunner(t)
 	minikubeRunner.RunCommand("addons enable gvisor", true)
 
@@ -201,7 +201,6 @@ func testGvisor(t *testing.T) {
 	}
 
 	createUntrustedWorkload(t)
-
 	t.Log("making sure untrusted workload is Running")
 	if err := util.WaitForUntrustedNginxRunning(); err != nil {
 		t.Fatalf("waiting for nginx to be up: %v", err)
@@ -215,14 +214,43 @@ func testGvisor(t *testing.T) {
 	}
 
 	createUntrustedWorkload(t)
-
 	t.Log("waiting for FailedCreatePodSandBox event")
 	if err := util.WaitForFailedCreatePodSandBoxEvent(); err != nil {
 		t.Fatalf("waiting for FailedCreatePodSandBox event: %v", err)
 	}
 	deleteUntrustedWorkload(t)
 }
 
+func testGvisorRuntimeClass(t *testing.T) {
+	minikubeRunner := NewMinikubeRunner(t)
+	minikubeRunner.RunCommand("addons enable gvisor", true)
+
+	t.Log("waiting for gvisor controller to come up")
+	if err := util.WaitForGvisorControllerRunning(t); err != nil {
+		t.Fatalf("waiting for gvisor controller to be up: %v", err)
+	}
+
+	createGvisorWorkload(t)
+	t.Log("making sure gvisor workload is Running")
+	if err := util.WaitForGvisorNginxRunning(); err != nil {
+		t.Fatalf("waiting for nginx to be up: %v", err)
+	}
+
+	t.Log("disabling gvisor addon")
+	minikubeRunner.RunCommand("addons disable gvisor", true)
+	t.Log("waiting for gvisor controller pod to be deleted")
+	if err := util.WaitForGvisorControllerDeleted(); err != nil {
+		t.Fatalf("waiting for gvisor controller to be deleted: %v", err)
+	}
+
+	createGvisorWorkload(t)
+	t.Log("waiting for FailedCreatePodSandBox event")
+	if err := util.WaitForFailedCreatePodSandBoxEvent(); err != nil {
+		t.Fatalf("waiting for FailedCreatePodSandBox event: %v", err)
+	}
+	deleteGvisorWorkload(t)
+}
+
 func testGvisorRestart(t *testing.T) {
 	minikubeRunner := NewMinikubeRunner(t)
 	minikubeRunner.EnsureRunning()
@@ -236,7 +264,8 @@ func testGvisorRestart(t *testing.T) {
 	// TODO: @priyawadhwa to add test for stop as well
 	minikubeRunner.RunCommand("delete", false)
 	minikubeRunner.CheckStatus(state.None.String())
-	minikubeRunner.Start()
+	// Minikube must be started with containerd as the runtime for runsc to work.
+	minikubeRunner.Start("--container-runtime=containerd", "--docker-opt containerd=/var/run/containerd/containerd.sock")
 	minikubeRunner.CheckStatus(state.Running.String())
 
 	t.Log("waiting for gvisor controller to come up")
@@ -249,9 +278,18 @@ func testGvisorRestart(t *testing.T) {
 	if err := util.WaitForUntrustedNginxRunning(); err != nil {
 		t.Fatalf("waiting for nginx to be up: %v", err)
 	}
+
+	createGvisorWorkload(t)
+	t.Log("making sure gvisor workload is Running")
+	if err := util.WaitForGvisorNginxRunning(); err != nil {
+		t.Fatalf("waiting for nginx to be up: %v", err)
+	}
+
 	deleteUntrustedWorkload(t)
+	deleteGvisorWorkload(t)
 }
 
+// createUntrustedWorkload creates a nginx pod with the untrusted workload annotation.
 func createUntrustedWorkload(t *testing.T) {
 	kubectlRunner := util.NewKubectlRunner(t)
 	curdir, err := filepath.Abs("")
@@ -265,6 +303,20 @@ func createUntrustedWorkload(t *testing.T) {
 	}
 }
 
+// createGvisorWorkload creates a nginx pod with gvisor as the RuntimeClass.
+func createGvisorWorkload(t *testing.T) {
+	kubectlRunner := util.NewKubectlRunner(t)
+	curdir, err := filepath.Abs("")
+	if err != nil {
+		t.Errorf("Error getting the file path for current directory: %s", curdir)
+	}
+	untrustedPath := path.Join(curdir, "testdata", "nginx-gvisor.yaml")
+	t.Log("creating pod with gvisor runtime class annotation")
+	if _, err := kubectlRunner.RunCommand([]string{"replace", "-f", untrustedPath, "--force"}); err != nil {
+		t.Fatalf("creating nginx(gVisor) resource: %v", err)
+	}
+}
+
 func deleteUntrustedWorkload(t *testing.T) {
 	kubectlRunner := util.NewKubectlRunner(t)
 	curdir, err := filepath.Abs("")
@@ -276,3 +328,15 @@ func deleteUntrustedWorkload(t *testing.T) {
 		t.Logf("error deleting untrusted nginx resource: %v", err)
 	}
 }
+
+func deleteGvisorWorkload(t *testing.T) {
+	kubectlRunner := util.NewKubectlRunner(t)
+	curdir, err := filepath.Abs("")
+	if err != nil {
+		t.Errorf("Error getting the file path for current directory: %s", curdir)
+	}
+	gvisorPath := path.Join(curdir, "testdata", "nginx-gvisor.yaml")
+	if _, err := kubectlRunner.RunCommand([]string{"delete", "-f", gvisorPath}); err != nil {
+		t.Logf("error deleting gvisor nginx resource: %v", err)
+	}
+}

test/integration/functional_test.go

@@ -60,7 +60,8 @@ func TestFunctionalContainerd(t *testing.T) {
 		r.RunCommand("delete", true)
 	}
 	r.Start("--container-runtime=containerd", "--docker-opt containerd=/var/run/containerd/containerd.sock")
-	t.Run("Gvisor", testGvisor)
+	t.Run("GvisorUntrustedWorkload", testGvisorUntrustedWorkload)
+	t.Run("GvisorRuntimeClass", testGvisorRuntimeClass)
 	t.Run("GvisorRestart", testGvisorRestart)
 	r.RunCommand("delete", true)
 }

test/integration/testdata/nginx-gvisor.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-gvisor
+  labels:
+    run: nginx
+    runtime: gvisor
+spec:
+  runtimeClassName: gvisor
+  containers:
+  - name: nginx
+    image: nginx

test/integration/testdata/nginx-untrusted.yaml

@@ -4,6 +4,7 @@ metadata:
   name: nginx-untrusted
   labels:
     run: nginx
+    untrusted: "true"
   annotations:
     io.kubernetes.cri.untrusted-workload: "true"
 spec: