From 1bce4519be854017d5576c3e549845ae7377a74c Mon Sep 17 00:00:00 2001 From: Ian Lewis Date: Fri, 29 Mar 2019 11:16:02 +0000 Subject: [PATCH] [WIP] Update gvisor runsc version (refs #4482) - Updates the gvisor addon to use containerd shim v2 - Updates the version of runsc - Auto-installs a gvisor RuntimeClass --- deploy/addons/gvisor/README.md | 30 +++++++++++-------- deploy/addons/gvisor/gvisor-config.toml | 7 +++-- .../addons/gvisor/gvisor-containerd-shim.toml | 3 -- deploy/addons/gvisor/gvisor-pod.yaml.tmpl | 6 ++-- deploy/addons/gvisor/gvisor-runtimeclass.yaml | 22 ++++++++++++++ pkg/gvisor/enable.go | 14 ++------- pkg/minikube/assets/addons.go | 12 ++++---- pkg/minikube/constants/constants.go | 8 ++--- 8 files changed, 56 insertions(+), 46 deletions(-) delete mode 100644 deploy/addons/gvisor/gvisor-containerd-shim.toml create mode 100644 deploy/addons/gvisor/gvisor-runtimeclass.yaml diff --git a/deploy/addons/gvisor/README.md b/deploy/addons/gvisor/README.md index 6433189f615a..d6414715b6f4 100644 --- a/deploy/addons/gvisor/README.md +++ b/deploy/addons/gvisor/README.md @@ -1,5 +1,5 @@ ## gVisor Addon -[gVisor](https://github.com/google/gvisor/blob/master/README.md), a sandboxed container runtime, allows users to securely run pods with untrusted workloads within Minikube. +[gVisor](https://gvisor.dev/), a sandboxed container runtime, allows users to securely run pods with untrusted workloads within Minikube. ### Starting Minikube gVisor depends on the containerd runtime to run in Minikube. @@ -17,21 +17,27 @@ To enable this addon, simply run: $ minikube addons enable gvisor ``` -Within one minute, the addon manager should pick up the change and you should see the `gvisor` pod: +Within one minute, the addon manager should pick up the change and you should +see the `gvisor` pod and `gvisor` [Runtime Class](https://kubernetes.io/docs/concepts/containers/runtime-class/): ``` -$ kubectl get pod gvisor -n kube-system -NAME READY STATUS RESTARTS AGE -gvisor 1/1 Running 0 3m +$ kubectl get pod,runtimeclass gvisor -n kube-system +NAME READY STATUS RESTARTS AGE +pod/gvisor 1/1 Running 0 2m52s + +NAME CREATED AT +runtimeclass.node.k8s.io/gvisor 2019-06-15T04:35:09Z ``` -Once the pod has status `Running`, gVisor is enabled in Minikube. +Once the pod has status `Running`, gVisor is enabled in Minikube. ### Running pods in gVisor -To run a pod in gVisor, add this annotation to the Kubernetes yaml: + +To run a pod in gVisor, add the `gvisor` runtime class to the Pod spec in your +Kubernetes yaml: ``` -io.kubernetes.cri.untrusted-workload: "true" +runtimeClassName: gvisor ``` An example Pod is shown below: @@ -41,17 +47,15 @@ apiVersion: v1 kind: Pod metadata: name: nginx-untrusted - annotations: - io.kubernetes.cri.untrusted-workload: "true" spec: + runtimeClassName: gvisor containers: - name: nginx image: nginx ``` -_Note: this annotation will not be necessary once the RuntimeClass Kubernetes feature is available broadly._ - ### Disabling gVisor + To disable gVisor, run: ``` @@ -67,4 +71,4 @@ NAME READY STATUS RESTARTS AGE gvisor 1/1 Terminating 0 5m ``` -_Note: Once gVisor is disabled, any pod with the `io.kubernetes.cri.untrusted-workload` annotation will fail with a FailedCreatePodSandBox error._ +_Note: Once gVisor is disabled, any pod with the `gvisor` Runtime Class or `io.kubernetes.cri.untrusted-workload` annotation will fail with a FailedCreatePodSandBox error._ diff --git a/deploy/addons/gvisor/gvisor-config.toml b/deploy/addons/gvisor/gvisor-config.toml index 034be8a4e57c..c80a71e9626e 100644 --- a/deploy/addons/gvisor/gvisor-config.toml +++ b/deploy/addons/gvisor/gvisor-config.toml @@ -43,8 +43,10 @@ oom_score = 0 runtime_root = "" [plugins.cri.containerd.untrusted_workload_runtime] runtime_type = "io.containerd.runtime.v1.linux" - runtime_engine = "/usr/local/bin/runsc" + runtime_engine = "/usr/bin/runsc" runtime_root = "/run/containerd/runsc" + [plugins.cri.containerd.runtimes.runsc] + runtime_type = "io.containerd.runsc.v1" [plugins.cri.cni] bin_dir = "/opt/cni/bin" conf_dir = "/etc/cni/net.d" @@ -56,7 +58,6 @@ oom_score = 0 [plugins.diff-service] default = ["walking"] [plugins.linux] - shim = "gvisor-containerd-shim" runtime = "runc" runtime_root = "" no_shim = false @@ -66,4 +67,4 @@ oom_score = 0 deletion_threshold = 0 mutation_threshold = 100 schedule_delay = "0s" - startup_delay = "100ms" \ No newline at end of file + startup_delay = "100ms" diff --git a/deploy/addons/gvisor/gvisor-containerd-shim.toml b/deploy/addons/gvisor/gvisor-containerd-shim.toml deleted file mode 100644 index b4afb9c5150c..000000000000 --- a/deploy/addons/gvisor/gvisor-containerd-shim.toml +++ /dev/null @@ -1,3 +0,0 @@ -runc_shim = "/bin/containerd-shim" -[runsc_config] - user-log="/tmp/runsc/user-log-%ID%.log" \ No newline at end of file diff --git a/deploy/addons/gvisor/gvisor-pod.yaml.tmpl b/deploy/addons/gvisor/gvisor-pod.yaml.tmpl index 08ecd7225f53..821cef10ec28 100644 --- a/deploy/addons/gvisor/gvisor-pod.yaml.tmpl +++ b/deploy/addons/gvisor/gvisor-pod.yaml.tmpl @@ -14,15 +14,15 @@ apiVersion: v1 kind: Pod -metadata: +metadata: name: gvisor namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/minikube-addons: gvisor -spec: +spec: hostPID: true - containers: + containers: - name: gvisor image: {{default "gcr.io/k8s-minikube" .ImageRepository}}/gvisor-addon:latest securityContext: diff --git a/deploy/addons/gvisor/gvisor-runtimeclass.yaml b/deploy/addons/gvisor/gvisor-runtimeclass.yaml new file mode 100644 index 000000000000..f37fa4a9ba64 --- /dev/null +++ b/deploy/addons/gvisor/gvisor-runtimeclass.yaml @@ -0,0 +1,22 @@ +# Copyright 2018 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: node.k8s.io/v1beta1 +kind: RuntimeClass +metadata: + name: gvisor + labels: + kubernetes.io/minikube-addons: gvisor + addonmanager.kubernetes.io/mode: Reconcile +handler: runsc diff --git a/pkg/gvisor/enable.go b/pkg/gvisor/enable.go index 7eacf10f9ae1..403ab0d4d8e8 100644 --- a/pkg/gvisor/enable.go +++ b/pkg/gvisor/enable.go @@ -80,12 +80,6 @@ func makeGvisorDirs() error { return errors.Wrap(err, "creating runsc dir") } - // Make /usr/local/bin to store the runsc binary - fp = filepath.Join(nodeDir, "usr/local/bin") - if err := os.MkdirAll(fp, 0755); err != nil { - return errors.Wrap(err, "creating usr/local/bin dir") - } - // Make /tmp/runsc to also hold logs fp = filepath.Join(nodeDir, "tmp/runsc") if err := os.MkdirAll(fp, 0755); err != nil { @@ -107,13 +101,13 @@ func downloadBinaries() error { // downloads the gvisor-containerd-shim func gvisorContainerdShim() error { - dest := filepath.Join(nodeDir, "usr/bin/gvisor-containerd-shim") + dest := filepath.Join(nodeDir, "usr/bin/containerd-shim-runsc-v1") return downloadFileToDest(constants.GvisorContainerdShimURL, dest) } // downloads the runsc binary and returns a path to the binary func runsc() error { - dest := filepath.Join(nodeDir, "usr/local/bin/runsc") + dest := filepath.Join(nodeDir, "usr/bin/runsc") return downloadFileToDest(constants.GvisorURL, dest) } @@ -159,10 +153,6 @@ func copyConfigFiles() error { if err := mcnutils.CopyFile(filepath.Join(nodeDir, constants.ContainerdConfigTomlPath), filepath.Join(nodeDir, constants.StoredContainerdConfigTomlPath)); err != nil { return errors.Wrap(err, "copying default config.toml") } - log.Print("Copying gvisor-containerd-shim.toml...") - if err := copyAssetToDest(constants.GvisorContainerdShimTargetName, filepath.Join(nodeDir, constants.GvisorContainerdShimTomlPath)); err != nil { - return errors.Wrap(err, "copying gvisor-containerd-shim.toml") - } log.Print("Copying containerd config.toml with gvisor...") if err := copyAssetToDest(constants.GvisorConfigTomlTargetName, filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil { return errors.Wrap(err, "copying gvisor version of config.toml") diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index f3d7a56307ae..b28067ca76c0 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -309,18 +309,18 @@ var Addons = map[string]*Addon{ "gvisor-pod.yaml", "0640", true), + MustBinAsset( + "deploy/addons/gvisor/gvisor-runtimeclass.yaml", + constants.AddonsPath, + "gvisor-runtimeclass.yaml", + "0640", + false), MustBinAsset( "deploy/addons/gvisor/gvisor-config.toml", constants.GvisorFilesPath, constants.GvisorConfigTomlTargetName, "0640", true), - MustBinAsset( - "deploy/addons/gvisor/gvisor-containerd-shim.toml", - constants.GvisorFilesPath, - constants.GvisorContainerdShimTargetName, - "0640", - false), }, false, "gvisor"), } diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 1a3453b896c2..117eb0b09834 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -387,18 +387,14 @@ const ( GvisorFilesPath = "/tmp/gvisor" // ContainerdConfigTomlPath is the path to the containerd config.toml ContainerdConfigTomlPath = "/etc/containerd/config.toml" - // GvisorContainerdShimTomlPath is the path to gvisor-containerd-shim.toml - GvisorContainerdShimTomlPath = "/etc/containerd/gvisor-containerd-shim.toml" // StoredContainerdConfigTomlPath is the path where the default config.toml will be stored StoredContainerdConfigTomlPath = "/tmp/config.toml" //GvisorConfigTomlTargetName is the go-bindata target name for the gvisor config.toml GvisorConfigTomlTargetName = "gvisor-config.toml" - // GvisorContainerdShimTargetName is the go-bindata target name for gvisor-containerd-shim - GvisorContainerdShimTargetName = "gvisor-containerd-shim.toml" // GvisorContainerdShimURL is the url to download gvisor-containerd-shim - GvisorContainerdShimURL = "https://github.com/google/gvisor-containerd-shim/releases/download/v0.0.1-rc.0/gvisor-containerd-shim-v0.0.1-rc.0.linux-amd64" + GvisorContainerdShimURL = "https://github.com/google/gvisor-containerd-shim/releases/download/v0.0.3/containerd-shim-runsc-v1.linux-amd64" // GvisorURL is the url to download gvisor - GvisorURL = "https://storage.googleapis.com/gvisor/releases/nightly/2018-12-07/runsc" + GvisorURL = "https://storage.googleapis.com/gvisor/releases/nightly/2019-01-14/runsc" )