Enable gvisor addon in minikube

Makefile

@@ -303,6 +303,18 @@ storage-provisioner-image: out/storage-provisioner
 push-storage-provisioner-image: storage-provisioner-image
 	gcloud docker -- push $(REGISTRY)/storage-provisioner:$(STORAGE_PROVISIONER_TAG)
 
+.PHONY: out/gvisor-addon
+out/gvisor-addon:
+	GOOS=linux CGO_ENABLED=0 go build -o $@ cmd/gvisor/gvisor.go
+
+.PHONY: gvisor-addon-image
+gvisor-addon-image: out/gvisor-addon
+	docker build -t $(REGISTRY)/gvisor-addon:latest -f deploy/gvisor/Dockerfile .
+
+.PHONY: push-gvisor-addon-image
+push-gvisor-addon-image: gvisor-addon-image
+	gcloud docker -- push $(REGISTRY)/gvisor-addon:latest
+
 .PHONY: release-iso
 release-iso: minikube_iso checksum
 	gsutil cp out/minikube.iso gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION).iso

cmd/gvisor/gvisor.go

@@ -0,0 +1,31 @@
+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"log"
+	"os"
+
+	"k8s.io/minikube/pkg/gvisor"
+)
+
+func main() {
+	if err := gvisor.Enable(); err != nil {
+		log.Print(err)
+		os.Exit(1)
+	}
+}

cmd/minikube/cmd/config/config.go

@@ -224,6 +224,12 @@ var settings = []Setting{
 		validations: []setFn{IsValidAddon},
 		callbacks:   []setFn{EnableOrDisableAddon},
 	},
+	{
+		name:        "gvisor",
+		set:         SetBool,
+		validations: []setFn{IsValidAddon, IsContainerdRuntime},
+		callbacks:   []setFn{EnableOrDisableAddon},
+	},
 	{
 		name: "hyperv-virtual-switch",
 		set:  SetString,

cmd/minikube/cmd/config/validations.go

@@ -18,15 +18,17 @@ package config
 
 import (
 	"fmt"
-	"github.com/docker/go-units"
-	"github.com/pkg/errors"
-	"k8s.io/minikube/pkg/minikube/assets"
-	"k8s.io/minikube/pkg/minikube/constants"
 	"net"
 	"net/url"
 	"os"
 	"strconv"
 	"strings"
+
+	"github.com/docker/go-units"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/assets"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
 )
 
 func IsValidDriver(string, driver string) error {
@@ -125,3 +127,19 @@ func IsValidAddon(name string, val string) error {
 	}
 	return errors.Errorf("Cannot enable/disable invalid addon %s", name)
 }
+
+func IsContainerdRuntime(_, _ string) error {
+	config, err := config.Load()
+	if err != nil {
+		return fmt.Errorf("error getting cluster config: %v", err)
+	}
+	if config.KubernetesConfig.ContainerRuntime != constants.ContainerdRuntime {
+		return fmt.Errorf(`This addon can only be enabled with the containerd runtime backend.
+
+To enable this backend, please start minikube again with the following flags:
+
+minikube start --container-runtime=containerd  --docker-opt containerd=/var/run/containerd/containerd.sock --network-plugin=cni`)
+	}
+
+	return nil
+}

deploy/addons/gvisor/README.md

@@ -0,0 +1,69 @@
+## gVisor Addon
+[gVisor](https://github.com/google/gvisor/blob/master/README.md), a sandboxed container runtime, allows users to securely run pods with untrusted workloads within Minikube.
+
+### Starting Minikube
+gVisor depends on the containerd runtime to run in Minikube.
+When starting minikube, specify the following flags, along with any additional desired flags:
+
+```shell
+$ minikube start --container-runtime=containerd  \
+    --docker-opt containerd=/var/run/containerd/containerd.sock \
+    --network-plugin=cni
+```
+
+### Enabling gVisor
+To enable this addon, simply run:
+
+```
+$ minikube addons enable gvisor
+```
+
+Within one minute, the addon manager should pick up the change and you should see the `gvisor` pod:
+
+```
+$ kubectl get pod gvisor -n kube-system
+NAME      READY     STATUS    RESTARTS   AGE
+gvisor    1/1       Running   0          3m
+```
+
+Once the pod has status `Running`, gVisor is enabled in Minikube. 
+
+### Running pods in gVisor
+To run a pod in gVisor, add this annotation to the Kubernetes yaml:
+
+```
+io.kubernetes.cri.untrusted-workload: "true"
+```
+
+An example Pod is shown below:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-untrusted
+  annotations:
+    io.kubernetes.cri.untrusted-workload: "true"
+spec:
+  containers:
+  - name: nginx
+    image: nginx
+```
+
+### Disabling gVisor
+To disable gVisor, run:
+
+```
+$ minikube addons disable gvisor
+```
+
+Within one minute, the addon manager should pick up the change.
+Once the `gvisor` pod has status `Terminating`, or has been deleted, the gvisor addon should be disabled.
+
+```
+$ kubectl get pod gvisor -n kube-system
+NAME      READY     STATUS        RESTARTS   AGE
+gvisor    1/1       Terminating   0          5m
+```
+
+_Note: Once gVisor is disabled, any pod with the `io.kubernetes.cri.untrusted-workload` annotation will fail with a FailedCreatePodSandBox error._

deploy/addons/gvisor/gvisor-config.toml

@@ -0,0 +1,69 @@
+root = "/var/lib/containerd"
+state = "/run/containerd"
+oom_score = 0
+
+[grpc]
+  address = "/run/containerd/containerd.sock"
+  uid = 0
+  gid = 0
+  max_recv_message_size = 16777216
+  max_send_message_size = 16777216
+
+[debug]
+  address = ""
+  uid = 0
+  gid = 0
+  level = ""
+
+[metrics]
+  address = ""
+  grpc_histogram = false
+
+[cgroup]
+  path = ""
+
+[plugins]
+  [plugins.cgroups]
+    no_prometheus = false
+  [plugins.cri]
+    stream_server_address = ""
+    stream_server_port = "10010"
+    enable_selinux = false
+    sandbox_image = "k8s.gcr.io/pause:3.1"
+    stats_collect_period = 10
+    systemd_cgroup = false
+    enable_tls_streaming = false
+    max_container_log_line_size = 16384
+    [plugins.cri.containerd]
+      snapshotter = "overlayfs"
+      no_pivot = true
+      [plugins.cri.containerd.default_runtime]
+        runtime_type = "io.containerd.runtime.v1.linux"
+        runtime_engine = ""
+        runtime_root = ""
+      [plugins.cri.containerd.untrusted_workload_runtime]
+        runtime_type = "io.containerd.runtime.v1.linux"
+        runtime_engine = "/usr/local/bin/runsc"
+        runtime_root = "/run/containerd/runsc"
+    [plugins.cri.cni]
+      bin_dir = "/opt/cni/bin"
+      conf_dir = "/etc/cni/net.d"
+      conf_template = ""
+    [plugins.cri.registry]
+      [plugins.cri.registry.mirrors]
+        [plugins.cri.registry.mirrors."docker.io"]
+          endpoint = ["https://registry-1.docker.io"]
+  [plugins.diff-service]
+    default = ["walking"]
+  [plugins.linux]
+    shim = "gvisor-containerd-shim"
+    runtime = "runc"
+    runtime_root = ""
+    no_shim = false
+    shim_debug = true
+  [plugins.scheduler]
+    pause_threshold = 0.02
+    deletion_threshold = 0
+    mutation_threshold = 100
+    schedule_delay = "0s"
+    startup_delay = "100ms"

deploy/addons/gvisor/gvisor-containerd-shim.toml

@@ -0,0 +1,5 @@
+runc_shim = "/bin/containerd-shim"
+[runsc_config]
+ debug-log="/tmp/runsc/"
+ debug="true"
+ user-log="/tmp/runsc/user-log-%ID%.log"

deploy/addons/gvisor/gvisor-pod.yaml

@@ -0,0 +1,71 @@
+# Copyright 2018 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata: 
+  name: gvisor
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+spec: 
+  hostPID: true
+  containers: 
+    - name: gvisor
+      image: gcr.io/k8s-minikube/gvisor-addon:latest
+      securityContext:
+        privileged: true
+      volumeMounts:
+      - mountPath: /node/
+        name: node
+      - mountPath: /usr/libexec/sudo
+        name: sudo
+      - mountPath: /var/run
+        name: varrun
+      - mountPath: /usr/bin
+        name: usrbin
+      - mountPath: /usr/lib
+        name: usrlib
+      - mountPath: /bin
+        name: bin
+      - mountPath: /tmp/gvisor
+        name: gvisor
+      env:
+        - name: PATH
+          value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/node/bin
+        - name: SYSTEMD_IGNORE_CHROOT
+          value: "yes"
+  volumes:
+  - name: node
+    hostPath:
+      path: /
+  - name: sudo
+    hostPath:
+      path: /usr/libexec/sudo
+  - name: varrun
+    hostPath:
+      path: /var/run
+  - name: usrlib
+    hostPath:
+      path: /usr/lib
+  - name: usrbin
+    hostPath:
+      path: /usr/bin
+  - name: bin
+    hostPath:
+      path: /bin
+  - name: gvisor
+    hostPath:
+      path: /tmp/gvisor
+  restartPolicy: Never

deploy/gvisor/Dockerfile

@@ -0,0 +1,20 @@
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:18.04
+RUN apt-get update && \
+    apt-get install -y kmod gcc wget xz-utils libc6-dev bc libelf-dev bison flex openssl libssl-dev libidn2-0 sudo libcap2 && \
+    rm -rf /var/lib/apt/lists/*
+COPY out/gvisor-addon /gvisor-addon
+CMD ["/gvisor-addon"]

docs/addons.md

@@ -40,6 +40,7 @@ The currently supported addons include:
 * [Freshpod](https://github.com/GoogleCloudPlatform/freshpod)
 * [nvidia-driver-installer](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/nvidia-driver-installer/minikube)
 * [nvidia-gpu-device-plugin](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/cmd/nvidia_gpu)
+* [gvisor](../deploy/addons/gvisor/README.md)
 
 If you would like to have minikube properly start/restart custom addons, place the addon(s) you wish to be launched with minikube in the `.minikube/addons` directory. Addons in this folder will be moved to the minikube VM and launched each time minikube is started/restarted.