diff --git a/Makefile b/Makefile index 290d625ab2d9..10797d1a3e78 100755 --- a/Makefile +++ b/Makefile @@ -303,6 +303,18 @@ storage-provisioner-image: out/storage-provisioner push-storage-provisioner-image: storage-provisioner-image gcloud docker -- push $(REGISTRY)/storage-provisioner:$(STORAGE_PROVISIONER_TAG) +.PHONY: out/gvisor-addon +out/gvisor-addon: + GOOS=linux CGO_ENABLED=0 go build -o $@ cmd/gvisor/gvisor.go + +.PHONY: gvisor-addon-image +gvisor-addon-image: out/gvisor-addon + docker build -t $(REGISTRY)/gvisor-addon:latest -f deploy/gvisor/Dockerfile . + +.PHONY: push-gvisor-addon-image +push-gvisor-addon-image: gvisor-addon-image + gcloud docker -- push $(REGISTRY)/gvisor-addon:latest + .PHONY: release-iso release-iso: minikube_iso checksum gsutil cp out/minikube.iso gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION).iso diff --git a/cmd/gvisor/gvisor.go b/cmd/gvisor/gvisor.go new file mode 100644 index 000000000000..d8c52b644b8c --- /dev/null +++ b/cmd/gvisor/gvisor.go @@ -0,0 +1,31 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "log" + "os" + + "k8s.io/minikube/pkg/gvisor" +) + +func main() { + if err := gvisor.Enable(); err != nil { + log.Print(err) + os.Exit(1) + } +} diff --git a/cmd/minikube/cmd/config/config.go b/cmd/minikube/cmd/config/config.go index 5fc59494793b..effe390dec36 100644 --- a/cmd/minikube/cmd/config/config.go +++ b/cmd/minikube/cmd/config/config.go @@ -212,6 +212,12 @@ var settings = []Setting{ validations: []setFn{IsValidAddon}, callbacks: []setFn{EnableOrDisableAddon}, }, + { + name: "gvisor", + set: SetBool, + validations: []setFn{IsValidAddon, IsContainerdRuntime}, + callbacks: []setFn{EnableOrDisableAddon}, + }, { name: "hyperv-virtual-switch", set: SetString, diff --git a/cmd/minikube/cmd/config/validations.go b/cmd/minikube/cmd/config/validations.go index e9ee37d8ece0..3c1b28a64136 100644 --- a/cmd/minikube/cmd/config/validations.go +++ b/cmd/minikube/cmd/config/validations.go @@ -18,15 +18,17 @@ package config import ( "fmt" - "github.com/docker/go-units" - "github.com/pkg/errors" - "k8s.io/minikube/pkg/minikube/assets" - "k8s.io/minikube/pkg/minikube/constants" "net" "net/url" "os" "strconv" "strings" + + "github.com/docker/go-units" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/constants" ) func IsValidDriver(string, driver string) error { @@ -125,3 +127,23 @@ func IsValidAddon(name string, val string) error { } return errors.Errorf("Cannot enable/disable invalid addon %s", name) } + +func IsContainerdRuntime(_, _ string) error { + config, err := config.Load() + if err != nil { + return fmt.Errorf("error getting cluster config: %v", err) + } + if config.KubernetesConfig.ContainerRuntime != constants.ContainerdRuntime { + return fmt.Errorf(`This addon can only be enabled with the containerd runtime backend. + +To enable this backend, please first stop minikube with: + +minikube stop + +and then start minikube again with the following flags: + +minikube start --container-runtime=containerd --docker-opt containerd=/var/run/containerd/containerd.sock --network-plugin=cni`) + } + + return nil +} diff --git a/deploy/addons/gvisor/README.md b/deploy/addons/gvisor/README.md new file mode 100644 index 000000000000..4bae2b2a7388 --- /dev/null +++ b/deploy/addons/gvisor/README.md @@ -0,0 +1,71 @@ +## gVisor Addon +[gVisor](https://github.com/google/gvisor/blob/master/README.md), a sandboxed container runtime, allows users to securely run pods with untrusted workloads within Minikube. + +### Starting Minikube +gVisor depends on the containerd runtime to run in Minikube. +When starting minikube, specify the following flags, along with any additional desired flags: + +```shell +$ minikube start --container-runtime=containerd \ + --docker-opt containerd=/var/run/containerd/containerd.sock \ + --network-plugin=cni +``` + +### Enabling gVisor +To enable this addon, simply run: + +``` +$ minikube addons enable gvisor +``` + +Within one minute, the addon manager should pick up the change and you should see the `gvisor` pod: + +``` +$ kubectl get pod gvisor -n kube-system +NAME READY STATUS RESTARTS AGE +gvisor 1/1 Running 0 3m +``` + +Once the pod has status `Running`, gVisor is enabled in Minikube. + +### Running pods in gVisor +To run a pod in gVisor, add this annotation to the Kubernetes yaml: + +``` +io.kubernetes.cri.untrusted-workload: "true" +``` + +An example Pod is shown below: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: nginx-untrusted + annotations: + io.kubernetes.cri.untrusted-workload: "true" +spec: + containers: + - name: nginx + image: nginx +``` + +_Note: this annotation will not be necessary once the RuntimeClass Kubernetes feature is available broadly._ + +### Disabling gVisor +To disable gVisor, run: + +``` +$ minikube addons disable gvisor +``` + +Within one minute, the addon manager should pick up the change. +Once the `gvisor` pod has status `Terminating`, or has been deleted, the gvisor addon should be disabled. + +``` +$ kubectl get pod gvisor -n kube-system +NAME READY STATUS RESTARTS AGE +gvisor 1/1 Terminating 0 5m +``` + +_Note: Once gVisor is disabled, any pod with the `io.kubernetes.cri.untrusted-workload` annotation will fail with a FailedCreatePodSandBox error._ diff --git a/deploy/addons/gvisor/gvisor-config.toml b/deploy/addons/gvisor/gvisor-config.toml new file mode 100644 index 000000000000..c891cddbee6d --- /dev/null +++ b/deploy/addons/gvisor/gvisor-config.toml @@ -0,0 +1,69 @@ +root = "/var/lib/containerd" +state = "/run/containerd" +oom_score = 0 + +[grpc] + address = "/run/containerd/containerd.sock" + uid = 0 + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + +[debug] + address = "" + uid = 0 + gid = 0 + level = "" + +[metrics] + address = "" + grpc_histogram = false + +[cgroup] + path = "" + +[plugins] + [plugins.cgroups] + no_prometheus = false + [plugins.cri] + stream_server_address = "" + stream_server_port = "10010" + enable_selinux = false + sandbox_image = "k8s.gcr.io/pause:3.1" + stats_collect_period = 10 + systemd_cgroup = false + enable_tls_streaming = false + max_container_log_line_size = 16384 + [plugins.cri.containerd] + snapshotter = "overlayfs" + no_pivot = true + [plugins.cri.containerd.default_runtime] + runtime_type = "io.containerd.runtime.v1.linux" + runtime_engine = "" + runtime_root = "" + [plugins.cri.containerd.untrusted_workload_runtime] + runtime_type = "io.containerd.runtime.v1.linux" + runtime_engine = "/usr/local/bin/runsc" + runtime_root = "/run/containerd/runsc" + [plugins.cri.cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + conf_template = "" + [plugins.cri.registry] + [plugins.cri.registry.mirrors] + [plugins.cri.registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + [plugins.diff-service] + default = ["walking"] + [plugins.linux] + shim = "gvisor-containerd-shim" + runtime = "runc" + runtime_root = "" + no_shim = false + shim_debug = true + [plugins.scheduler] + pause_threshold = 0.02 + deletion_threshold = 0 + mutation_threshold = 100 + schedule_delay = "0s" + startup_delay = "100ms" \ No newline at end of file diff --git a/deploy/addons/gvisor/gvisor-containerd-shim.toml b/deploy/addons/gvisor/gvisor-containerd-shim.toml new file mode 100644 index 000000000000..b4afb9c5150c --- /dev/null +++ b/deploy/addons/gvisor/gvisor-containerd-shim.toml @@ -0,0 +1,3 @@ +runc_shim = "/bin/containerd-shim" +[runsc_config] + user-log="/tmp/runsc/user-log-%ID%.log" \ No newline at end of file diff --git a/deploy/addons/gvisor/gvisor-pod.yaml b/deploy/addons/gvisor/gvisor-pod.yaml new file mode 100644 index 000000000000..50a54bb69336 --- /dev/null +++ b/deploy/addons/gvisor/gvisor-pod.yaml @@ -0,0 +1,72 @@ +# Copyright 2018 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: gvisor + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/minikube-addons: gvisor +spec: + hostPID: true + containers: + - name: gvisor + image: gcr.io/k8s-minikube/gvisor-addon:latest + securityContext: + privileged: true + volumeMounts: + - mountPath: /node/ + name: node + - mountPath: /usr/libexec/sudo + name: sudo + - mountPath: /var/run + name: varrun + - mountPath: /usr/bin + name: usrbin + - mountPath: /usr/lib + name: usrlib + - mountPath: /bin + name: bin + - mountPath: /tmp/gvisor + name: gvisor + env: + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/node/bin + - name: SYSTEMD_IGNORE_CHROOT + value: "yes" + volumes: + - name: node + hostPath: + path: / + - name: sudo + hostPath: + path: /usr/libexec/sudo + - name: varrun + hostPath: + path: /var/run + - name: usrlib + hostPath: + path: /usr/lib + - name: usrbin + hostPath: + path: /usr/bin + - name: bin + hostPath: + path: /bin + - name: gvisor + hostPath: + path: /tmp/gvisor + restartPolicy: Never diff --git a/deploy/gvisor/Dockerfile b/deploy/gvisor/Dockerfile new file mode 100644 index 000000000000..9dacfa546644 --- /dev/null +++ b/deploy/gvisor/Dockerfile @@ -0,0 +1,20 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:18.04 +RUN apt-get update && \ + apt-get install -y kmod gcc wget xz-utils libc6-dev bc libelf-dev bison flex openssl libssl-dev libidn2-0 sudo libcap2 && \ + rm -rf /var/lib/apt/lists/* +COPY out/gvisor-addon /gvisor-addon +CMD ["/gvisor-addon"] diff --git a/docs/addons.md b/docs/addons.md index b737409e1570..fdd22a88d61a 100644 --- a/docs/addons.md +++ b/docs/addons.md @@ -36,6 +36,7 @@ The currently supported addons include: * [Freshpod](https://github.com/GoogleCloudPlatform/freshpod) * [nvidia-driver-installer](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/nvidia-driver-installer/minikube) * [nvidia-gpu-device-plugin](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/cmd/nvidia_gpu) +* [gvisor](../deploy/addons/gvisor/README.md) If you would like to have minikube properly start/restart custom addons, place the addon(s) you wish to be launched with minikube in the `.minikube/addons` directory. Addons in this folder will be moved to the minikube VM and launched each time minikube is started/restarted. diff --git a/pkg/gvisor/disable.go b/pkg/gvisor/disable.go new file mode 100644 index 000000000000..4c00bbaea0c9 --- /dev/null +++ b/pkg/gvisor/disable.go @@ -0,0 +1,45 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gvisor + +import ( + "log" + "os" + "path/filepath" + + "github.com/docker/machine/libmachine/mcnutils" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/constants" +) + +// Disable reverts containerd config files and restarts containerd +func Disable() error { + log.Print("Disabling gvisor...") + if err := os.Remove(filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil { + return errors.Wrapf(err, "removing %s", constants.ContainerdConfigTomlPath) + } + log.Printf("Restoring default config.toml at %s", constants.ContainerdConfigTomlPath) + if err := mcnutils.CopyFile(filepath.Join(nodeDir, constants.StoredContainerdConfigTomlPath), filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil { + return errors.Wrap(err, "reverting back to default config.toml") + } + // restart containerd + if err := restartContainerd(); err != nil { + return errors.Wrap(err, "restarting containerd") + } + log.Print("Successfully disabled gvisor") + return nil +} diff --git a/pkg/gvisor/enable.go b/pkg/gvisor/enable.go new file mode 100644 index 000000000000..2268c7381d2e --- /dev/null +++ b/pkg/gvisor/enable.go @@ -0,0 +1,230 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gvisor + +import ( + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "os/signal" + "path/filepath" + "syscall" + + "github.com/docker/machine/libmachine/mcnutils" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/constants" +) + +const ( + nodeDir = "/node" +) + +// Enable follows these steps for enabling gvisor in minikube: +// 1. creates necessary directories for storing binaries and runsc logs +// 2. downloads runsc and gvisor-containerd-shim +// 3. copies necessary containerd config files +// 4. restarts containerd +func Enable() error { + if err := makeGvisorDirs(); err != nil { + return errors.Wrap(err, "creating directories on node") + } + if err := downloadBinaries(); err != nil { + return errors.Wrap(err, "downloading binaries") + } + if err := copyConfigFiles(); err != nil { + return errors.Wrap(err, "copying config files") + } + if err := restartContainerd(); err != nil { + return errors.Wrap(err, "restarting containerd") + } + // When pod is terminated, disable gvisor and exit + c := make(chan os.Signal) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + if err := Disable(); err != nil { + log.Printf("Error disabling gvisor: %v", err) + os.Exit(1) + } + os.Exit(0) + }() + log.Print("gvisor successfully enabled in cluster") + // sleep for one year so the pod continuously runs + select {} +} + +// makeGvisorDirs creates necessary directories on the node +func makeGvisorDirs() error { + // Make /run/containerd/runsc to hold logs + fp := filepath.Join(nodeDir, "run/containerd/runsc") + if err := os.MkdirAll(fp, 0755); err != nil { + return errors.Wrap(err, "creating runsc dir") + } + + // Make /usr/local/bin to store the runsc binary + fp = filepath.Join(nodeDir, "usr/local/bin") + if err := os.MkdirAll(fp, 0755); err != nil { + return errors.Wrap(err, "creating usr/local/bin dir") + } + + // Make /tmp/runsc to also hold logs + fp = filepath.Join(nodeDir, "tmp/runsc") + if err := os.MkdirAll(fp, 0755); err != nil { + return errors.Wrap(err, "creating runsc logs dir") + } + + return nil +} + +func downloadBinaries() error { + if err := runsc(); err != nil { + return errors.Wrap(err, "downloading runsc") + } + if err := gvisorContainerdShim(); err != nil { + return errors.Wrap(err, "downloading gvisor-containerd-shim") + } + return nil +} + +// downloads the gvisor-containerd-shim +func gvisorContainerdShim() error { + dest := filepath.Join(nodeDir, "usr/bin/gvisor-containerd-shim") + return downloadFileToDest(constants.GvisorContainerdShimURL, dest) +} + +// downloads the runsc binary and returns a path to the binary +func runsc() error { + dest := filepath.Join(nodeDir, "usr/local/bin/runsc") + return downloadFileToDest(constants.GvisorURL, dest) +} + +// downloadFileToDest downlaods the given file to the dest +// if something already exists at dest, first remove it +func downloadFileToDest(url, dest string) error { + client := &http.Client{} + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return errors.Wrapf(err, "creating request for %s", url) + } + req.Header.Set("User-Agent", "minikube") + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if _, err := os.Stat(dest); err == nil { + if err := os.Remove(dest); err != nil { + return errors.Wrapf(err, "removing %s for overwrite", dest) + } + } + fi, err := os.Create(dest) + if err != nil { + return errors.Wrapf(err, "creating %s", dest) + } + defer fi.Close() + if _, err := io.Copy(fi, resp.Body); err != nil { + return errors.Wrap(err, "copying binary") + } + if err := fi.Chmod(0777); err != nil { + return errors.Wrap(err, "fixing perms") + } + return nil +} + +// Must write the following files: +// 1. gvisor-containerd-shim.toml +// 2. gvisor containerd config.toml +// and save the default version of config.toml +func copyConfigFiles() error { + log.Printf("Storing default config.toml at %s", constants.StoredContainerdConfigTomlPath) + if err := mcnutils.CopyFile(filepath.Join(nodeDir, constants.ContainerdConfigTomlPath), filepath.Join(nodeDir, constants.StoredContainerdConfigTomlPath)); err != nil { + return errors.Wrap(err, "copying default config.toml") + } + log.Print("Copying gvisor-containerd-shim.toml...") + if err := copyAssetToDest(constants.GvisorContainerdShimTargetName, filepath.Join(nodeDir, constants.GvisorContainerdShimTomlPath)); err != nil { + return errors.Wrap(err, "copying gvisor-containerd-shim.toml") + } + log.Print("Copying containerd config.toml with gvisor...") + if err := copyAssetToDest(constants.GvisorConfigTomlTargetName, filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil { + return errors.Wrap(err, "copying gvisor version of config.toml") + } + return nil +} + +func copyAssetToDest(targetName, dest string) error { + var asset *assets.BinDataAsset + for _, a := range assets.Addons["gvisor"].Assets { + if a.GetTargetName() == targetName { + asset = a + } + } + // Now, copy the data from this asset to dest + src := filepath.Join(constants.GvisorFilesPath, asset.GetTargetName()) + contents, err := ioutil.ReadFile(src) + if err != nil { + return errors.Wrapf(err, "getting contents of %s", asset.GetAssetName()) + } + if _, err := os.Stat(dest); err == nil { + if err := os.Remove(dest); err != nil { + return errors.Wrapf(err, "removing %s", dest) + } + } + f, err := os.Create(dest) + if err != nil { + return errors.Wrapf(err, "creating %s", dest) + } + if _, err := f.Write(contents); err != nil { + return errors.Wrapf(err, "writing contents to %s", f.Name()) + } + return nil +} + +func restartContainerd() error { + dir := filepath.Join(nodeDir, "usr/libexec/sudo") + if err := os.Setenv("LD_LIBRARY_PATH", dir); err != nil { + return errors.Wrap(err, dir) + } + + log.Print("Stopping rpc-statd.service...") + // first, stop rpc-statd.service + cmd := exec.Command("sudo", "-E", "systemctl", "stop", "rpc-statd.service") + if out, err := cmd.CombinedOutput(); err != nil { + fmt.Println(string(out)) + return errors.Wrap(err, "stopping rpc-statd.service") + } + // restart containerd + log.Print("Restarting containerd...") + cmd = exec.Command("sudo", "-E", "systemctl", "restart", "containerd") + if out, err := cmd.CombinedOutput(); err != nil { + log.Print(string(out)) + return errors.Wrap(err, "restarting containerd") + } + // start rpc-statd.service + log.Print("Starting rpc-statd...") + cmd = exec.Command("sudo", "-E", "systemctl", "start", "rpc-statd.service") + if out, err := cmd.CombinedOutput(); err != nil { + log.Print(string(out)) + return errors.Wrap(err, "restarting rpc-statd.service") + } + log.Print("containerd restart complete") + return nil +} diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index 329cef3b5662..825de3b8e5bd 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -227,6 +227,23 @@ var Addons = map[string]*Addon{ "nvidia-gpu-device-plugin.yaml", "0640"), }, false, "nvidia-gpu-device-plugin"), + "gvisor": NewAddon([]*BinDataAsset{ + NewBinDataAsset( + "deploy/addons/gvisor/gvisor-pod.yaml", + constants.AddonsPath, + "gvisor-pod.yaml", + "0640"), + NewBinDataAsset( + "deploy/addons/gvisor/gvisor-config.toml", + constants.GvisorFilesPath, + constants.GvisorConfigTomlTargetName, + "0640"), + NewBinDataAsset( + "deploy/addons/gvisor/gvisor-containerd-shim.toml", + constants.GvisorFilesPath, + constants.GvisorContainerdShimTargetName, + "0640"), + }, false, "gvisor"), } func AddMinikubeDirAssets(assets *[]CopyableFile) error { diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 0b9966dc580e..c278dcbefc2a 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -22,13 +22,13 @@ import ( "path/filepath" "runtime" "strings" + "time" "github.com/blang/semver" "github.com/golang/glog" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" minikubeVersion "k8s.io/minikube/pkg/version" - "time" ) // APIServerPort is the port that the API server should listen on. @@ -261,3 +261,24 @@ func GetKubeadmCachedImages(kubernetesVersionStr string) []string { } var ImageCacheDir = MakeMiniPath("cache", "images") + +const ( + // GvisorFilesPath is the path to the gvisor files saved by go-bindata + GvisorFilesPath = "/tmp/gvisor" + // ContainerdConfigTomlPath is the path to the containerd config.toml + ContainerdConfigTomlPath = "/etc/containerd/config.toml" + // GvisorContainerdShimTomlPath is the path to givosr-containerd-shim.toml + GvisorContainerdShimTomlPath = "/etc/containerd/gvisor-containerd-shim.toml" + // StoredContainerdConfigTomlPath is the path where the default config.toml will be stored + StoredContainerdConfigTomlPath = "/tmp/config.toml" + + //GvisorConfigTomlTargetName is the go-bindata target name for the gvisor config.toml + GvisorConfigTomlTargetName = "gvisor-config.toml" + // GvisorContainerdShimTargetName is the go-bindata target name for gvisor-containerd-shim + GvisorContainerdShimTargetName = "gvisor-containerd-shim.toml" + + // GvisorContainerdShimURL is the url to download gvisor-containerd-shim + GvisorContainerdShimURL = "https://github.com/google/gvisor-containerd-shim/releases/download/v0.0.1-rc.0/gvisor-containerd-shim-v0.0.1-rc.0.linux-amd64" + // GvisorURL is the url to download gvisor + GvisorURL = "https://storage.googleapis.com/gvisor/releases/nightly/2018-12-07/runsc" +) diff --git a/pkg/util/kubernetes.go b/pkg/util/kubernetes.go index 45b3ee1e5f02..927cf1dd7835 100644 --- a/pkg/util/kubernetes.go +++ b/pkg/util/kubernetes.go @@ -139,6 +139,36 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels }) } +// Wait up to 10 minutes for a pod to be deleted +func WaitForPodDelete(c kubernetes.Interface, ns string, label labels.Selector) error { + return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) { + listOpts := metav1.ListOptions{LabelSelector: label.String()} + pods, err := c.CoreV1().Pods(ns).List(listOpts) + if err != nil { + glog.Infof("error getting Pods with label selector %q [%v]\n", label.String(), err) + return false, nil + } + return len(pods.Items) == 0, nil + }) +} + +// Wait up to 10 minutes for the given event to appear +func WaitForEvent(c kubernetes.Interface, ns string, reason string) error { + return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) { + events, err := c.Events().Events("default").List(metav1.ListOptions{}) + if err != nil { + glog.Infof("error getting events: %v", err) + return false, nil + } + for _, e := range events.Items { + if e.Reason == reason { + return true, nil + } + } + return false, nil + }) +} + // WaitForRCToStabilize waits till the RC has a matching generation/replica count between spec and status. func WaitForRCToStabilize(c kubernetes.Interface, ns, name string, timeout time.Duration) error { options := metav1.ListOptions{FieldSelector: fields.Set{ diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 9c2dbdfa8428..7d65b970222a 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -183,3 +183,46 @@ func testServicesList(t *testing.T) { t.Fatalf(err.Error()) } } + +func testGvisor(t *testing.T) { + minikubeRunner := NewMinikubeRunner(t) + kubectlRunner := util.NewKubectlRunner(t) + minikubeRunner.RunCommand("addons enable gvisor", true) + + t.Log("waiting for gvisor controller to come up") + if err := util.WaitForGvisorControllerRunning(t); err != nil { + t.Fatalf("waiting for gvisor controller to be up: %v", err) + } + + untrustedPath, _ := filepath.Abs("testdata/nginx-untrusted.yaml") + t.Log("creating pod with untrusted workload annotation") + if _, err := kubectlRunner.RunCommand([]string{"create", "-f", untrustedPath}); err != nil { + t.Fatalf("creating untrusted nginx resource: %v", err) + } + + t.Log("making sure untrusted workload is Running") + if err := util.WaitForUntrustedNginxRunning(); err != nil { + t.Fatalf("waiting for nginx to be up: %v", err) + } + + t.Log("disabling gvisor addon") + minikubeRunner.RunCommand("addons disable gvisor", true) + t.Log("waiting for gvisor controller pod to be deleted") + if err := util.WaitForGvisorControllerDeleted(); err != nil { + t.Fatalf("waiting for gvisor controller to be deleted: %v", err) + } + + t.Log("recreating untrusted workload pod") + if _, err := kubectlRunner.RunCommand([]string{"replace", "-f", untrustedPath, "--force"}); err != nil { + t.Fatalf("replacing untrusted nginx resource: %v", err) + } + + t.Log("waiting for FailedCreatePodSandBox event") + if err := util.WaitForFailedCreatePodSandBoxEvent(); err != nil { + t.Fatalf("waiting for FailedCreatePodSandBox event: %v", err) + } + + if _, err := kubectlRunner.RunCommand([]string{"delete", "-f", untrustedPath}); err != nil { + t.Logf("error deleting untrusted nginx resource: %v", err) + } +} diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index b3bf68df542a..08ca9e171b8c 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -22,6 +22,8 @@ import ( "strings" "testing" + "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/test/integration/util" ) @@ -48,6 +50,24 @@ func TestFunctional(t *testing.T) { } } +func TestFunctionalContainerd(t *testing.T) { + minikubeRunner := NewMinikubeRunner(t) + + if usingNoneDriver(minikubeRunner) { + t.Skip("Can't run containerd backend with none driver") + } + + if minikubeRunner.GetStatus() != state.None.String() { + minikubeRunner.RunCommand("delete", true) + } + + minikubeRunner.SetRuntime(constants.ContainerdRuntime) + minikubeRunner.EnsureRunning() + + t.Run("Gvisor", testGvisor) + minikubeRunner.RunCommand("delete", true) +} + // usingNoneDriver returns true if using the none driver func usingNoneDriver(runner util.MinikubeRunner) bool { return strings.Contains(runner.StartArgs, "--vm-driver=none") diff --git a/test/integration/testdata/nginx-untrusted.yaml b/test/integration/testdata/nginx-untrusted.yaml new file mode 100644 index 000000000000..7ff9ce9fb704 --- /dev/null +++ b/test/integration/testdata/nginx-untrusted.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx-untrusted + labels: + run: nginx + annotations: + io.kubernetes.cri.untrusted-workload: "true" +spec: + containers: + - name: nginx + image: nginx diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 8bb069ba9200..081a60fb0f7d 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -126,7 +126,8 @@ func (m *MinikubeRunner) SSH(command string) (string, error) { func (m *MinikubeRunner) Start() { switch r := m.Runtime; r { case constants.ContainerdRuntime: - containerdFlags := "--container-runtime=containerd --network-plugin=cni --docker-opt containerd=/var/run/containerd/containerd.sock" + // TODO: priyawadhwa@ remove iso url once updated iso is being used in integration tests + containerdFlags := "--container-runtime=containerd --network-plugin=cni --docker-opt containerd=/var/run/containerd/containerd.sock --iso-url=https://storage.googleapis.com/k8s-minikube/gvisor-preview.iso" m.RunCommand(fmt.Sprintf("start %s %s %s", m.StartArgs, m.Args, containerdFlags), true) default: m.RunCommand(fmt.Sprintf("start %s %s", m.StartArgs, m.Args), true) @@ -298,6 +299,60 @@ func WaitForIngressDefaultBackendRunning(t *testing.T) error { return nil } +// WaitForGvisorControllerRunning waits for the gvisor controller pod to be running +func WaitForGvisorControllerRunning(t *testing.T) error { + client, err := commonutil.GetClient() + if err != nil { + return errors.Wrap(err, "getting kubernetes client") + } + + selector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "gvisor"})) + if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil { + return errors.Wrap(err, "waiting for gvisor controller pod to stabilize") + } + return nil +} + +// WaitForGvisorControllerDeleted waits for the gvisor controller pod to be deleted +func WaitForGvisorControllerDeleted() error { + client, err := commonutil.GetClient() + if err != nil { + return errors.Wrap(err, "getting kubernetes client") + } + + selector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "gvisor"})) + if err := commonutil.WaitForPodDelete(client, "kube-system", selector); err != nil { + return errors.Wrap(err, "waiting for gvisor controller pod deletion") + } + return nil +} + +// WaitForUntrustedNginxRunning waits for the untrusted nginx pod to start running +func WaitForUntrustedNginxRunning() error { + client, err := commonutil.GetClient() + if err != nil { + return errors.Wrap(err, "getting kubernetes client") + } + + selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx"})) + if err := commonutil.WaitForPodsWithLabelRunning(client, "default", selector); err != nil { + return errors.Wrap(err, "waiting for nginx pods") + } + return nil +} + +// WaitForFailedCreatePodSandBoxEvent waits for a FailedCreatePodSandBox event to appear +func WaitForFailedCreatePodSandBoxEvent() error { + client, err := commonutil.GetClient() + if err != nil { + return errors.Wrap(err, "getting kubernetes client") + } + if err := commonutil.WaitForEvent(client, "default", "FailedCreatePodSandBox"); err != nil { + return errors.Wrap(err, "waiting for FailedCreatePodSandBox event") + } + return nil +} + func WaitForNginxRunning(t *testing.T) error { client, err := commonutil.GetClient()