From 04c12154326b7f3cf25fb213460382d280ae410c Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 3 Dec 2018 12:09:55 -0800 Subject: [PATCH 01/23] Add integration test to make sure minikube starts and stops with containerd --- test/integration/start_stop_delete_test.go | 63 ++++++++++++++-------- test/integration/util/util.go | 15 +++++- 2 files changed, 55 insertions(+), 23 deletions(-) diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index 9efd5cf35cef..3a89db1811f1 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -25,37 +25,56 @@ import ( "time" "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/test/integration/util" ) func TestStartStop(t *testing.T) { + tests := []struct { + name string + runtime string + }{ + { + name: "default", + runtime: "", + }, + { + name: "start stop with containerd runtime", + runtime: constants.ContainerdRuntime, + }, + } - runner := NewMinikubeRunner(t) - runner.RunCommand("config set WantReportErrorPrompt false", true) - runner.RunCommand("delete", false) - runner.CheckStatus(state.None.String()) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + runner := NewMinikubeRunner(t) + runner.RunCommand("config set WantReportErrorPrompt false", true) + runner.RunCommand("delete", false) + runner.CheckStatus(state.None.String()) - runner.Start() - runner.CheckStatus(state.Running.String()) + runner.SetRuntime(test.runtime) + runner.Start() + runner.CheckStatus(state.Running.String()) - ip := runner.RunCommand("ip", true) - ip = strings.TrimRight(ip, "\n") - if net.ParseIP(ip) == nil { - t.Fatalf("IP command returned an invalid address: %s", ip) - } + ip := runner.RunCommand("ip", true) + ip = strings.TrimRight(ip, "\n") + if net.ParseIP(ip) == nil { + t.Fatalf("IP command returned an invalid address: %s", ip) + } - checkStop := func() error { - runner.RunCommand("stop", true) - return runner.CheckStatusNoFail(state.Stopped.String()) - } + checkStop := func() error { + runner.RunCommand("stop", true) + return runner.CheckStatusNoFail(state.Stopped.String()) + } - if err := util.Retry(t, checkStop, 5*time.Second, 6); err != nil { - t.Fatalf("timed out while checking stopped status: %v", err) - } + if err := util.Retry(t, checkStop, 5*time.Second, 6); err != nil { + t.Fatalf("timed out while checking stopped status: %v", err) + } - runner.Start() - runner.CheckStatus(state.Running.String()) + runner.Start() + runner.CheckStatus(state.Running.String()) - runner.RunCommand("delete", true) - runner.CheckStatus(state.None.String()) + runner.RunCommand("delete", true) + runner.CheckStatus(state.None.String()) + }) + } } diff --git a/test/integration/util/util.go b/test/integration/util/util.go index f9760141ff92..8bb069ba9200 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -33,6 +33,7 @@ import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/constants" commonutil "k8s.io/minikube/pkg/util" ) @@ -43,6 +44,7 @@ type MinikubeRunner struct { BinaryPath string Args string StartArgs string + Runtime string } func (m *MinikubeRunner) Run(cmd string) error { @@ -105,6 +107,11 @@ func (m *MinikubeRunner) RunDaemon(command string) (*exec.Cmd, *bufio.Reader) { } +// SetRuntime saves the runtime backend +func (m *MinikubeRunner) SetRuntime(runtime string) { + m.Runtime = runtime +} + func (m *MinikubeRunner) SSH(command string) (string, error) { path, _ := filepath.Abs(m.BinaryPath) cmd := exec.Command(path, "ssh", command) @@ -117,7 +124,13 @@ func (m *MinikubeRunner) SSH(command string) (string, error) { } func (m *MinikubeRunner) Start() { - m.RunCommand(fmt.Sprintf("start %s %s", m.StartArgs, m.Args), true) + switch r := m.Runtime; r { + case constants.ContainerdRuntime: + containerdFlags := "--container-runtime=containerd --network-plugin=cni --docker-opt containerd=/var/run/containerd/containerd.sock" + m.RunCommand(fmt.Sprintf("start %s %s %s", m.StartArgs, m.Args, containerdFlags), true) + default: + m.RunCommand(fmt.Sprintf("start %s %s", m.StartArgs, m.Args), true) + } } func (m *MinikubeRunner) EnsureRunning() { From 01ad85eec892c8dcda51a448771c57a00b9275f2 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 3 Dec 2018 13:05:03 -0800 Subject: [PATCH 02/23] Skip testing containerd if using none driver --- test/integration/functional_test.go | 7 ++++++- test/integration/start_stop_delete_test.go | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 2fa3a439c1cf..ceeefabb0ec4 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -38,10 +38,15 @@ func TestFunctional(t *testing.T) { t.Run("Provisioning", testProvisioning) t.Run("Tunnel", testTunnel) - if !strings.Contains(minikubeRunner.StartArgs, "--vm-driver=none") { + if !usingNoneDriver(minikubeRunner) { t.Run("EnvVars", testClusterEnv) t.Run("SSH", testClusterSSH) t.Run("IngressController", testIngressController) t.Run("Mounting", testMounting) } } + +// usingNoneDriver returns true if using the none driver +func usingNoneDriver(runner *util.MinikuberRunner ) bool { + return strings.Contains(minikubeRunner.StartArgs, "--vm-driver=none") +} \ No newline at end of file diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index 3a89db1811f1..da9f17c1f7be 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -47,6 +47,10 @@ func TestStartStop(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { runner := NewMinikubeRunner(t) + if test.runtime != "" && usingNoneDriver(runner) { + t.SkipNow() + } + runner.RunCommand("config set WantReportErrorPrompt false", true) runner.RunCommand("delete", false) runner.CheckStatus(state.None.String()) From 8c7290f7d106899dbf6c6289de4fa1177c25bd87 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 3 Dec 2018 13:11:25 -0800 Subject: [PATCH 03/23] add newline --- test/integration/functional_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index ceeefabb0ec4..59975d9e7bdc 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -49,4 +49,4 @@ func TestFunctional(t *testing.T) { // usingNoneDriver returns true if using the none driver func usingNoneDriver(runner *util.MinikuberRunner ) bool { return strings.Contains(minikubeRunner.StartArgs, "--vm-driver=none") -} \ No newline at end of file +} From e3854493221cb66e019c47ef23410cc112faf2b7 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 3 Dec 2018 13:26:41 -0800 Subject: [PATCH 04/23] Fixed imports --- test/integration/functional_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 59975d9e7bdc..b3bf68df542a 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -21,6 +21,8 @@ package integration import ( "strings" "testing" + + "k8s.io/minikube/test/integration/util" ) func TestFunctional(t *testing.T) { @@ -47,6 +49,6 @@ func TestFunctional(t *testing.T) { } // usingNoneDriver returns true if using the none driver -func usingNoneDriver(runner *util.MinikuberRunner ) bool { - return strings.Contains(minikubeRunner.StartArgs, "--vm-driver=none") +func usingNoneDriver(runner util.MinikubeRunner) bool { + return strings.Contains(runner.StartArgs, "--vm-driver=none") } From 40ad9e7837acc1a48e7773ff071d1f21d22362f1 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 3 Dec 2018 13:58:33 -0800 Subject: [PATCH 05/23] Add skip message when using none driver and containerd --- test/integration/start_stop_delete_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index da9f17c1f7be..063d3a92feeb 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -48,7 +48,7 @@ func TestStartStop(t *testing.T) { t.Run(test.name, func(t *testing.T) { runner := NewMinikubeRunner(t) if test.runtime != "" && usingNoneDriver(runner) { - t.SkipNow() + t.Skipf("skipping, can't use %s with none driver", test.runtime) } runner.RunCommand("config set WantReportErrorPrompt false", true) From 85a4b8b7d13ee345e79943ade40d452e0959e11b Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 3 Dec 2018 16:53:12 -0800 Subject: [PATCH 06/23] Enable gvisor addon in minikube This PR adds the code for enabling gvisor in minikube. It adds the pod that will run when the addon is enabled, and the code for the image which will run when this happens. When gvisor is enabled, the pod will download runsc and the gvisor-containerd-shim. It will replace the containerd config.toml and restart containerd. When gvisor is disabled, the pod will be deleted by the addon manager. This will trigger a pre-stop hook which will revert the config.toml to it's original state and restart containerd. --- Makefile | 12 + cmd/gvisor/gvisor.go | 48 ++++ cmd/minikube/cmd/config/config.go | 6 + deploy/addons/gvisor/config.toml | 69 ++++++ deploy/addons/gvisor/gvisor-config.toml | 69 ++++++ .../addons/gvisor/gvisor-containerd-shim.toml | 5 + deploy/addons/gvisor/gvisor-pod.yaml | 75 +++++++ deploy/gvisor/Dockerfile | 20 ++ pkg/gvisor/disable.go | 38 ++++ pkg/gvisor/enable.go | 208 ++++++++++++++++++ pkg/minikube/assets/addons.go | 22 ++ pkg/minikube/constants/constants.go | 19 +- 12 files changed, 590 insertions(+), 1 deletion(-) create mode 100644 cmd/gvisor/gvisor.go create mode 100644 deploy/addons/gvisor/config.toml create mode 100644 deploy/addons/gvisor/gvisor-config.toml create mode 100644 deploy/addons/gvisor/gvisor-containerd-shim.toml create mode 100644 deploy/addons/gvisor/gvisor-pod.yaml create mode 100644 deploy/gvisor/Dockerfile create mode 100644 pkg/gvisor/disable.go create mode 100644 pkg/gvisor/enable.go diff --git a/Makefile b/Makefile index 290d625ab2d9..10797d1a3e78 100755 --- a/Makefile +++ b/Makefile @@ -303,6 +303,18 @@ storage-provisioner-image: out/storage-provisioner push-storage-provisioner-image: storage-provisioner-image gcloud docker -- push $(REGISTRY)/storage-provisioner:$(STORAGE_PROVISIONER_TAG) +.PHONY: out/gvisor-addon +out/gvisor-addon: + GOOS=linux CGO_ENABLED=0 go build -o $@ cmd/gvisor/gvisor.go + +.PHONY: gvisor-addon-image +gvisor-addon-image: out/gvisor-addon + docker build -t $(REGISTRY)/gvisor-addon:latest -f deploy/gvisor/Dockerfile . + +.PHONY: push-gvisor-addon-image +push-gvisor-addon-image: gvisor-addon-image + gcloud docker -- push $(REGISTRY)/gvisor-addon:latest + .PHONY: release-iso release-iso: minikube_iso checksum gsutil cp out/minikube.iso gs://$(ISO_BUCKET)/minikube-$(ISO_VERSION).iso diff --git a/cmd/gvisor/gvisor.go b/cmd/gvisor/gvisor.go new file mode 100644 index 000000000000..73821cff20a0 --- /dev/null +++ b/cmd/gvisor/gvisor.go @@ -0,0 +1,48 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "log" + "os" + + "k8s.io/minikube/pkg/gvisor" +) + +var ( + disable bool +) + +func init() { + flag.BoolVar(&disable, "disable", false, "disable gvisor addon") + flag.Parse() +} + +func main() { + if err := execute(); err != nil { + log.Print(err) + os.Exit(1) + } +} + +func execute() error { + if disable { + return gvisor.Disable() + } + return gvisor.Enable() +} diff --git a/cmd/minikube/cmd/config/config.go b/cmd/minikube/cmd/config/config.go index 473e264c9267..c9181839cb68 100644 --- a/cmd/minikube/cmd/config/config.go +++ b/cmd/minikube/cmd/config/config.go @@ -224,6 +224,12 @@ var settings = []Setting{ validations: []setFn{IsValidAddon}, callbacks: []setFn{EnableOrDisableAddon}, }, + { + name: "gvisor", + set: SetBool, + validations: []setFn{IsValidAddon}, + callbacks: []setFn{EnableOrDisableAddon}, + }, { name: "hyperv-virtual-switch", set: SetString, diff --git a/deploy/addons/gvisor/config.toml b/deploy/addons/gvisor/config.toml new file mode 100644 index 000000000000..ae7fe958038a --- /dev/null +++ b/deploy/addons/gvisor/config.toml @@ -0,0 +1,69 @@ +root = "/var/lib/containerd" +state = "/run/containerd" +oom_score = 0 + +[grpc] + address = "/run/containerd/containerd.sock" + uid = 0 + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + +[debug] + address = "" + uid = 0 + gid = 0 + level = "" + +[metrics] + address = "" + grpc_histogram = false + +[cgroup] + path = "" + +[plugins] + [plugins.cgroups] + no_prometheus = false + [plugins.cri] + stream_server_address = "" + stream_server_port = "10010" + enable_selinux = false + sandbox_image = "k8s.gcr.io/pause:3.1" + stats_collect_period = 10 + systemd_cgroup = false + enable_tls_streaming = false + max_container_log_line_size = 16384 + [plugins.cri.containerd] + snapshotter = "overlayfs" + no_pivot = true + [plugins.cri.containerd.default_runtime] + runtime_type = "io.containerd.runtime.v1.linux" + runtime_engine = "" + runtime_root = "" + [plugins.cri.containerd.untrusted_workload_runtime] + runtime_type = "" + runtime_engine = "" + runtime_root = "" + [plugins.cri.cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + conf_template = "" + [plugins.cri.registry] + [plugins.cri.registry.mirrors] + [plugins.cri.registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + [plugins.diff-service] + default = ["walking"] + [plugins.linux] + shim = "containerd-shim" + runtime = "runc" + runtime_root = "" + no_shim = false + shim_debug = false + [plugins.scheduler] + pause_threshold = 0.02 + deletion_threshold = 0 + mutation_threshold = 100 + schedule_delay = "0s" + startup_delay = "100ms" \ No newline at end of file diff --git a/deploy/addons/gvisor/gvisor-config.toml b/deploy/addons/gvisor/gvisor-config.toml new file mode 100644 index 000000000000..a62f216817a1 --- /dev/null +++ b/deploy/addons/gvisor/gvisor-config.toml @@ -0,0 +1,69 @@ +root = "/var/lib/containerd" +state = "/run/containerd" +oom_score = 0 + +[grpc] + address = "/run/containerd/containerd.sock" + uid = 0 + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + +[debug] + address = "" + uid = 0 + gid = 0 + level = "" + +[metrics] + address = "" + grpc_histogram = false + +[cgroup] + path = "" + +[plugins] + [plugins.cgroups] + no_prometheus = false + [plugins.cri] + stream_server_address = "" + stream_server_port = "10010" + enable_selinux = false + sandbox_image = "k8s.gcr.io/pause:3.1" + stats_collect_period = 10 + systemd_cgroup = false + enable_tls_streaming = false + max_container_log_line_size = 16384 + [plugins.cri.containerd] + snapshotter = "overlayfs" + no_pivot = true + [plugins.cri.containerd.default_runtime] + runtime_type = "io.containerd.runtime.v1.linux" + runtime_engine = "" + runtime_root = "" + [plugins.cri.containerd.untrusted_workload_runtime] + runtime_type = "io.containerd.runtime.v1.linux" + runtime_engine = "/usr/local/bin/runsc" + runtime_root = "/run/containerd/runsc" + [plugins.cri.cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + conf_template = "" + [plugins.cri.registry] + [plugins.cri.registry.mirrors] + [plugins.cri.registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + [plugins.diff-service] + default = ["walking"] + [plugins.linux] + shim = "gvisor-containerd-shim" + runtime = "runc" + runtime_root = "" + no_shim = false + shim_debug = true + [plugins.scheduler] + pause_threshold = 0.02 + deletion_threshold = 0 + mutation_threshold = 100 + schedule_delay = "0s" + startup_delay = "100ms" \ No newline at end of file diff --git a/deploy/addons/gvisor/gvisor-containerd-shim.toml b/deploy/addons/gvisor/gvisor-containerd-shim.toml new file mode 100644 index 000000000000..a6eba601bc3b --- /dev/null +++ b/deploy/addons/gvisor/gvisor-containerd-shim.toml @@ -0,0 +1,5 @@ +runc_shim = "/bin/containerd-shim" +[runsc_config] + debug-log="/tmp/runsc/" + debug="true" + log="/tmp/runsc/out.log" \ No newline at end of file diff --git a/deploy/addons/gvisor/gvisor-pod.yaml b/deploy/addons/gvisor/gvisor-pod.yaml new file mode 100644 index 000000000000..241a420491fc --- /dev/null +++ b/deploy/addons/gvisor/gvisor-pod.yaml @@ -0,0 +1,75 @@ +# Copyright 2018 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: gvisor + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +spec: + hostPID: true + containers: + - name: gvisor + image: gcr.io/k8s-minikube/gvisor-addon:latest + securityContext: + privileged: true + volumeMounts: + - mountPath: /node/ + name: node + - mountPath: /usr/libexec/sudo + name: sudo + - mountPath: /var/run + name: varrun + - mountPath: /usr/bin + name: usrbin + - mountPath: /usr/lib + name: usrlib + - mountPath: /bin + name: bin + - mountPath: /tmp/gvisor + name: gvisor + env: + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/node/bin + - name: SYSTEMD_IGNORE_CHROOT + value: "yes" + lifecycle: + preStop: + exec: + command: ["/gvisor-addon","-disable"] + volumes: + - name: node + hostPath: + path: / + - name: sudo + hostPath: + path: /usr/libexec/sudo + - name: varrun + hostPath: + path: /var/run + - name: usrlib + hostPath: + path: /usr/lib + - name: usrbin + hostPath: + path: /usr/bin + - name: bin + hostPath: + path: /bin + - name: gvisor + hostPath: + path: /tmp/gvisor + restartPolicy: Never diff --git a/deploy/gvisor/Dockerfile b/deploy/gvisor/Dockerfile new file mode 100644 index 000000000000..010e3c948d1e --- /dev/null +++ b/deploy/gvisor/Dockerfile @@ -0,0 +1,20 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:18.04 +RUN apt-get update && \ + apt-get install -y kmod gcc wget xz-utils libc6-dev bc libelf-dev bison flex openssl libssl-dev libidn2-0 sudo libcap2 && \ + rm -rf /var/lib/apt/lists/* +COPY out/gvisor-addon /gvisor-addon +CMD /gvisor-addon diff --git a/pkg/gvisor/disable.go b/pkg/gvisor/disable.go new file mode 100644 index 000000000000..0899a3dd8f41 --- /dev/null +++ b/pkg/gvisor/disable.go @@ -0,0 +1,38 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gvisor + +import ( + "log" + "path/filepath" + + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/constants" +) + +// Disable reverts containerd config files and restarts containerd +func Disable() error { + log.Print("Disabling gvisor...") + if err := copyAssetToDest(constants.DefaultConfigTomlTargetName, filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil { + return errors.Wrap(err, "reverting config.toml to default") + } + // restart containerd + if err := restartContainerd(); err != nil { + return errors.Wrap(err, "restarting containerd") + } + return nil +} diff --git a/pkg/gvisor/enable.go b/pkg/gvisor/enable.go new file mode 100644 index 000000000000..e1d89f833028 --- /dev/null +++ b/pkg/gvisor/enable.go @@ -0,0 +1,208 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gvisor + +import ( + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "os/exec" + "path/filepath" + "time" + + "k8s.io/minikube/pkg/minikube/constants" + + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/assets" +) + +const ( + nodeDir = "/node" +) + +// Enable follows these steps for enabling gvisor in minikube: +// 1. creates necessary directories for storing binaries and runsc logs +// 2. downloads runsc and gvisor-containerd-shim +// 3. copies necessary containerd config files +// 4. restarts containerd +func Enable() error { + if err := makeDirs(); err != nil { + return errors.Wrap(err, "creating directories on node") + } + if err := downloadBinaries(); err != nil { + return errors.Wrap(err, "downloading binaries") + } + if err := copyFiles(); err != nil { + return errors.Wrap(err, "copying files") + } + if err := restartContainerd(); err != nil { + return errors.Wrap(err, "restarting containerd") + } + // sleep for one year so the pod continuously runs + time.Sleep(24 * 7 * 52 * time.Hour) + return nil +} + +// makeDirs creates necessary directories on the node +func makeDirs() error { + // Make /run/containerd/runsc to hold logs + fp := filepath.Join(nodeDir, "run/containerd/runsc") + if err := os.MkdirAll(fp, 0755); err != nil { + return errors.Wrap(err, "creating runsc dir") + } + + // Make /usr/local/bin to store the runsc binary + fp = filepath.Join(nodeDir, "usr/local/bin") + if err := os.MkdirAll(fp, 0755); err != nil { + return errors.Wrap(err, "creating usr/local/bin dir") + } + + // Make /tmp/runsc to also hold logs + fp = filepath.Join(nodeDir, "tmp/runsc") + if err := os.MkdirAll(fp, 0755); err != nil { + return errors.Wrap(err, "creating runsc logs dir") + } + + return nil +} + +func downloadBinaries() error { + if err := runsc(); err != nil { + return errors.Wrap(err, "downloading runsc") + } + if err := gvisorContainerdShim(); err != nil { + return errors.Wrap(err, "downloading gvisor-containerd-shim") + } + return nil +} + +// downloads the gvisor-containerd-shim +func gvisorContainerdShim() error { + dest := filepath.Join(nodeDir, "usr/bin/gvisor-containerd-shim") + // TODO: priyawadhwa@, replace with official release + return downloadFileToDest("http://storage.googleapis.com/balintp-minikube/gvisor-containerd-shim", dest) +} + +// downloads the runsc binary and returns a path to the binary +func runsc() error { + dest := filepath.Join(nodeDir, "usr/local/bin/runsc") + return downloadFileToDest("http://storage.googleapis.com/gvisor/releases/nightly/latest/runsc", dest) +} + +// downloadFileToDest downlaods the given file to the dest +// if something already exists at dest, first remove it +func downloadFileToDest(url, dest string) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + if _, err := os.Stat(dest); err == nil { + if err := os.Remove(dest); err != nil { + return errors.Wrapf(err, "removing %s for overwrite", dest) + } + } + fi, err := os.Create(dest) + if err != nil { + return errors.Wrapf(err, "creating %s", dest) + } + defer fi.Close() + if _, err := io.Copy(fi, resp.Body); err != nil { + return errors.Wrap(err, "copying binary") + } + if err := fi.Chmod(0777); err != nil { + return errors.Wrap(err, "fixing perms") + } + return nil +} + +// Must write the following files: +// 1. gvisor-containerd-shim.toml +// 2. gvisor containerd config.toml +func copyFiles() error { + log.Print("Copying gvisor-containerd-shim.toml...") + if err := copyAssetToDest(constants.GvisorContainerdShimTargetName, filepath.Join(nodeDir, constants.GvisorContainerdShimTomlPath)); err != nil { + return errors.Wrap(err, "copying gvisor-containerd-shim.toml") + } + log.Print("Copying containerd config.toml with gvisor...") + if err := copyAssetToDest(constants.GvisorConfigTomlTargetName, filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil { + return errors.Wrap(err, "copying gvisor version of config.toml") + } + return nil +} + +func copyAssetToDest(targetName, dest string) error { + var asset *assets.BinDataAsset + for _, a := range assets.Addons["gvisor"].Assets { + if a.GetTargetName() == targetName { + asset = a + } + } + // Now, copy the data from this asset to dest + src := filepath.Join(constants.GvisorFilesPath, asset.GetTargetName()) + contents, err := ioutil.ReadFile(src) + if err != nil { + return errors.Wrapf(err, "getting contents of %s", asset.GetAssetName()) + } + if _, err := os.Stat(dest); err == nil { + if err := os.Remove(dest); err != nil { + return errors.Wrapf(err, "removing %s", dest) + } + } + f, err := os.Create(dest) + if err != nil { + return errors.Wrapf(err, "creating %s", dest) + } + if _, err := f.Write(contents); err != nil { + return errors.Wrapf(err, "writing contents to %s", f.Name()) + } + return nil +} + +func restartContainerd() error { + dir := filepath.Join(nodeDir, "usr/libexec/sudo") + if err := os.Setenv("LD_LIBRARY_PATH", dir); err != nil { + return errors.Wrap(err, dir) + } + + log.Print("Stopping rpc-statd.service...") + // first, stop rpc-statd.service + cmd := exec.Command("sudo", "-E", "systemctl", "stop", "rpc-statd.service") + if out, err := cmd.CombinedOutput(); err != nil { + fmt.Println(string(out)) + return errors.Wrap(err, "stopping rpc-statd.service") + } + // restart containerd + log.Print("Restarting containerd...") + cmd = exec.Command("sudo", "-E", "systemctl", "restart", "containerd") + if out, err := cmd.CombinedOutput(); err != nil { + log.Print(string(out)) + return errors.Wrap(err, "restarting containerd") + } + // start rpc-statd.service + log.Print("Starting rpc-statd...") + cmd = exec.Command("sudo", "-E", "systemctl", "start", "rpc-statd.service") + if out, err := cmd.CombinedOutput(); err != nil { + log.Print(string(out)) + return errors.Wrap(err, "restarting rpc-statd.service") + } + log.Print("containerd restart complete") + return nil +} diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index e4c7357bfb2b..5b137e2b3005 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -276,6 +276,28 @@ var Addons = map[string]*Addon{ "nvidia-gpu-device-plugin.yaml", "0640"), }, false, "nvidia-gpu-device-plugin"), + "gvisor": NewAddon([]*BinDataAsset{ + NewBinDataAsset( + "deploy/addons/gvisor/gvisor-pod.yaml", + constants.AddonsPath, + "gvisor-pod.yaml", + "0640"), + NewBinDataAsset( + "deploy/addons/gvisor/gvisor-config.toml", + constants.GvisorFilesPath, + constants.GvisorConfigTomlTargetName, + "0640"), + NewBinDataAsset( + "deploy/addons/gvisor/gvisor-containerd-shim.toml", + constants.GvisorFilesPath, + constants.GvisorContainerdShimTargetName, + "0640"), + NewBinDataAsset( + "deploy/addons/gvisor/config.toml", + constants.GvisorFilesPath, + constants.DefaultConfigTomlTargetName, + "0640"), + }, false, "gvisor"), } func AddMinikubeDirAssets(assets *[]CopyableFile) error { diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 0b9966dc580e..b1e0caa466ad 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -23,12 +23,13 @@ import ( "runtime" "strings" + "time" + "github.com/blang/semver" "github.com/golang/glog" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/homedir" minikubeVersion "k8s.io/minikube/pkg/version" - "time" ) // APIServerPort is the port that the API server should listen on. @@ -261,3 +262,19 @@ func GetKubeadmCachedImages(kubernetesVersionStr string) []string { } var ImageCacheDir = MakeMiniPath("cache", "images") + +const ( + // GvisorFilesPath is the path to the gvisor files saved by go-bindata + GvisorFilesPath = "/tmp/gvisor" + // ContainerdConfigTomlPath is the path to the containerd config.toml + ContainerdConfigTomlPath = "/etc/containerd/config.toml" + // GvisorContainerdShimTomlPath is the path to givosr-containerd-shim.toml + GvisorContainerdShimTomlPath = "/etc/containerd/gvisor-containerd-shim.toml" + + //GvisorConfigTomlTargetName is the go-bindata target name for the gvisor config.toml + GvisorConfigTomlTargetName = "gvisor-config.toml" + // GvisorContainerdShimTargetName is the go-bindata target name for gvisor-containerd-shim + GvisorContainerdShimTargetName = "gvisor-containerd-shim.toml" + // DefaultConfigTomlTargetName is the go-bindata target name for the default config.toml + DefaultConfigTomlTargetName = "config.toml" +) From f885f4b91bf09fb13cfc565a8692c96a3e4b45bb Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Mon, 3 Dec 2018 17:53:02 -0800 Subject: [PATCH 07/23] Added doc for enabling gvisor in minikube --- docs/addons.md | 1 + docs/gvisor.md | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 docs/gvisor.md diff --git a/docs/addons.md b/docs/addons.md index ea38dc9995f1..58cfc176c1f2 100644 --- a/docs/addons.md +++ b/docs/addons.md @@ -40,6 +40,7 @@ The currently supported addons include: * [Freshpod](https://github.com/GoogleCloudPlatform/freshpod) * [nvidia-driver-installer](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/nvidia-driver-installer/minikube) * [nvidia-gpu-device-plugin](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/cmd/nvidia_gpu) +* [gvisor](gvisor.md) If you would like to have minikube properly start/restart custom addons, place the addon(s) you wish to be launched with minikube in the `.minikube/addons` directory. Addons in this folder will be moved to the minikube VM and launched each time minikube is started/restarted. diff --git a/docs/gvisor.md b/docs/gvisor.md new file mode 100644 index 000000000000..bac4e891cc29 --- /dev/null +++ b/docs/gvisor.md @@ -0,0 +1,69 @@ +## gVisor Addon +[gVisor](https://github.com/google/gvisor/blob/master/README.md), a sandboxed container runtime, allows users to securely run pods with untrusted workloads within Minikube. + +### Starting Minikube +gVisor depends on the containerd runtime to run in Minikube. +When starting minikube, specify the following flags, along with any additional desired flags: + +```shell +$ minikube start --container-runtime=containerd \ + --docker-opt containerd=/var/run/containerd/containerd.sock \ + --network-plugin=cni +``` + +### Enabling gVisor +To enable this addon, simply run: + +``` +$ minikube addons enable gvisor +``` + +Within one minute, the addon manager should pick up the change and you should see the `gvisor` pod: + +``` +$ kubectl get pod gvisor -n kube-system +NAME READY STATUS RESTARTS AGE +gvisor 1/1 Running 0 3m +``` + +Once the pod has status `Running`, gVisor is enabled in Minikube. + +### Running pods in gVisor +To run a pod in gVisor, add this annotation to the Kubernetes yaml: + +``` +io.kubernetes.cri.untrusted-workload: "true" +``` + +An example Pod is shown below: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: nginx-untrusted + annotations: + io.kubernetes.cri.untrusted-workload: "true" +spec: + containers: + - name: nginx + image: nginx +``` + +### Disabling gVisor +To disable gVisor, run: + +``` +$ minikube addons disable gvisor +``` + +Within one minute, the addon manager should pick up the change. +Once the `gvisor` pod has status `Terminating`, or has been deleted, the gvisor addon should be disabled. + +``` +$ kubectl get pod gvisor -n kube-system +NAME READY STATUS RESTARTS AGE +gvisor 1/1 Terminating 0 5m +``` + +_Note: Once gVisor is disabled, any pod with the `io.kubernetes.cri.untrusted-workload` annotation will fail with a FailedCreatePodSandBox error._ From ac963e2bd3c95efaedd675188da7dbfa2377766c Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 4 Dec 2018 13:09:49 -0800 Subject: [PATCH 08/23] Code review comments Instead of checking in default config.toml, save it at /tmp/config.toml on the node upon enable and copy it back upon disable. Also, instead of using the prestop hook, intercept the SIGTERM kill signal upon pod termination, disable gvisor, and then exit with code 0. This should work better because now we will be able to see the logs from disabling, and because the prestop hook wouldn't consistenly run the disable command and clean up the pod correctly. --- .../addons/gvisor/README.md | 0 deploy/addons/gvisor/config.toml | 69 ------------------- deploy/addons/gvisor/gvisor-config.toml | 2 +- .../addons/gvisor/gvisor-containerd-shim.toml | 2 +- deploy/addons/gvisor/gvisor-pod.yaml | 4 -- deploy/gvisor/Dockerfile | 2 +- docs/addons.md | 2 +- pkg/gvisor/disable.go | 11 ++- pkg/gvisor/enable.go | 27 ++++++-- pkg/minikube/assets/addons.go | 5 -- pkg/minikube/constants/constants.go | 9 ++- 11 files changed, 42 insertions(+), 91 deletions(-) rename docs/gvisor.md => deploy/addons/gvisor/README.md (100%) delete mode 100644 deploy/addons/gvisor/config.toml diff --git a/docs/gvisor.md b/deploy/addons/gvisor/README.md similarity index 100% rename from docs/gvisor.md rename to deploy/addons/gvisor/README.md diff --git a/deploy/addons/gvisor/config.toml b/deploy/addons/gvisor/config.toml deleted file mode 100644 index ae7fe958038a..000000000000 --- a/deploy/addons/gvisor/config.toml +++ /dev/null @@ -1,69 +0,0 @@ -root = "/var/lib/containerd" -state = "/run/containerd" -oom_score = 0 - -[grpc] - address = "/run/containerd/containerd.sock" - uid = 0 - gid = 0 - max_recv_message_size = 16777216 - max_send_message_size = 16777216 - -[debug] - address = "" - uid = 0 - gid = 0 - level = "" - -[metrics] - address = "" - grpc_histogram = false - -[cgroup] - path = "" - -[plugins] - [plugins.cgroups] - no_prometheus = false - [plugins.cri] - stream_server_address = "" - stream_server_port = "10010" - enable_selinux = false - sandbox_image = "k8s.gcr.io/pause:3.1" - stats_collect_period = 10 - systemd_cgroup = false - enable_tls_streaming = false - max_container_log_line_size = 16384 - [plugins.cri.containerd] - snapshotter = "overlayfs" - no_pivot = true - [plugins.cri.containerd.default_runtime] - runtime_type = "io.containerd.runtime.v1.linux" - runtime_engine = "" - runtime_root = "" - [plugins.cri.containerd.untrusted_workload_runtime] - runtime_type = "" - runtime_engine = "" - runtime_root = "" - [plugins.cri.cni] - bin_dir = "/opt/cni/bin" - conf_dir = "/etc/cni/net.d" - conf_template = "" - [plugins.cri.registry] - [plugins.cri.registry.mirrors] - [plugins.cri.registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - [plugins.diff-service] - default = ["walking"] - [plugins.linux] - shim = "containerd-shim" - runtime = "runc" - runtime_root = "" - no_shim = false - shim_debug = false - [plugins.scheduler] - pause_threshold = 0.02 - deletion_threshold = 0 - mutation_threshold = 100 - schedule_delay = "0s" - startup_delay = "100ms" \ No newline at end of file diff --git a/deploy/addons/gvisor/gvisor-config.toml b/deploy/addons/gvisor/gvisor-config.toml index a62f216817a1..c891cddbee6d 100644 --- a/deploy/addons/gvisor/gvisor-config.toml +++ b/deploy/addons/gvisor/gvisor-config.toml @@ -44,7 +44,7 @@ oom_score = 0 [plugins.cri.containerd.untrusted_workload_runtime] runtime_type = "io.containerd.runtime.v1.linux" runtime_engine = "/usr/local/bin/runsc" - runtime_root = "/run/containerd/runsc" + runtime_root = "/run/containerd/runsc" [plugins.cri.cni] bin_dir = "/opt/cni/bin" conf_dir = "/etc/cni/net.d" diff --git a/deploy/addons/gvisor/gvisor-containerd-shim.toml b/deploy/addons/gvisor/gvisor-containerd-shim.toml index a6eba601bc3b..13da23b0fa8d 100644 --- a/deploy/addons/gvisor/gvisor-containerd-shim.toml +++ b/deploy/addons/gvisor/gvisor-containerd-shim.toml @@ -2,4 +2,4 @@ runc_shim = "/bin/containerd-shim" [runsc_config] debug-log="/tmp/runsc/" debug="true" - log="/tmp/runsc/out.log" \ No newline at end of file + user-log="/tmp/runsc/user-log-%ID%.log" \ No newline at end of file diff --git a/deploy/addons/gvisor/gvisor-pod.yaml b/deploy/addons/gvisor/gvisor-pod.yaml index 241a420491fc..516893a0a97f 100644 --- a/deploy/addons/gvisor/gvisor-pod.yaml +++ b/deploy/addons/gvisor/gvisor-pod.yaml @@ -46,10 +46,6 @@ spec: value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/node/bin - name: SYSTEMD_IGNORE_CHROOT value: "yes" - lifecycle: - preStop: - exec: - command: ["/gvisor-addon","-disable"] volumes: - name: node hostPath: diff --git a/deploy/gvisor/Dockerfile b/deploy/gvisor/Dockerfile index 010e3c948d1e..9dacfa546644 100644 --- a/deploy/gvisor/Dockerfile +++ b/deploy/gvisor/Dockerfile @@ -17,4 +17,4 @@ RUN apt-get update && \ apt-get install -y kmod gcc wget xz-utils libc6-dev bc libelf-dev bison flex openssl libssl-dev libidn2-0 sudo libcap2 && \ rm -rf /var/lib/apt/lists/* COPY out/gvisor-addon /gvisor-addon -CMD /gvisor-addon +CMD ["/gvisor-addon"] diff --git a/docs/addons.md b/docs/addons.md index 58cfc176c1f2..20ebedc818a4 100644 --- a/docs/addons.md +++ b/docs/addons.md @@ -40,7 +40,7 @@ The currently supported addons include: * [Freshpod](https://github.com/GoogleCloudPlatform/freshpod) * [nvidia-driver-installer](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/nvidia-driver-installer/minikube) * [nvidia-gpu-device-plugin](https://github.com/GoogleCloudPlatform/container-engine-accelerators/tree/master/cmd/nvidia_gpu) -* [gvisor](gvisor.md) +* [gvisor](../deploy/addons/gvisor/README.md) If you would like to have minikube properly start/restart custom addons, place the addon(s) you wish to be launched with minikube in the `.minikube/addons` directory. Addons in this folder will be moved to the minikube VM and launched each time minikube is started/restarted. diff --git a/pkg/gvisor/disable.go b/pkg/gvisor/disable.go index 0899a3dd8f41..881a4eca47ec 100644 --- a/pkg/gvisor/disable.go +++ b/pkg/gvisor/disable.go @@ -18,8 +18,10 @@ package gvisor import ( "log" + "os" "path/filepath" + "github.com/docker/machine/libmachine/mcnutils" "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/constants" ) @@ -27,12 +29,17 @@ import ( // Disable reverts containerd config files and restarts containerd func Disable() error { log.Print("Disabling gvisor...") - if err := copyAssetToDest(constants.DefaultConfigTomlTargetName, filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil { - return errors.Wrap(err, "reverting config.toml to default") + if err := os.Remove(filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil { + return errors.Wrapf(err, "removing %s", constants.ContainerdConfigTomlPath) + } + log.Printf("Storing default config.toml at %s", constants.ContainerdConfigTomlPath) + if err := mcnutils.CopyFile(filepath.Join(nodeDir, constants.StoredContainerdConfigTomlPath), filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil { + return errors.Wrap(err, "reverting back to default config.toml") } // restart containerd if err := restartContainerd(); err != nil { return errors.Wrap(err, "restarting containerd") } + log.Print("Successfully disabled gvisor") return nil } diff --git a/pkg/gvisor/enable.go b/pkg/gvisor/enable.go index e1d89f833028..ec553ec76c64 100644 --- a/pkg/gvisor/enable.go +++ b/pkg/gvisor/enable.go @@ -24,13 +24,15 @@ import ( "net/http" "os" "os/exec" + "os/signal" "path/filepath" + "syscall" "time" - "k8s.io/minikube/pkg/minikube/constants" - + "github.com/docker/machine/libmachine/mcnutils" "github.com/pkg/errors" "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/constants" ) const ( @@ -55,6 +57,17 @@ func Enable() error { if err := restartContainerd(); err != nil { return errors.Wrap(err, "restarting containerd") } + // When pod is terminated, disable gvisor and exit + c := make(chan os.Signal) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + if err := Disable(); err != nil { + log.Printf("Error disabling gvisor: %v", err) + } + os.Exit(0) + }() + log.Print("gvisor successfully enabled in cluster") // sleep for one year so the pod continuously runs time.Sleep(24 * 7 * 52 * time.Hour) return nil @@ -96,14 +109,13 @@ func downloadBinaries() error { // downloads the gvisor-containerd-shim func gvisorContainerdShim() error { dest := filepath.Join(nodeDir, "usr/bin/gvisor-containerd-shim") - // TODO: priyawadhwa@, replace with official release - return downloadFileToDest("http://storage.googleapis.com/balintp-minikube/gvisor-containerd-shim", dest) + return downloadFileToDest(constants.GvisorContainerdShimURL, dest) } // downloads the runsc binary and returns a path to the binary func runsc() error { dest := filepath.Join(nodeDir, "usr/local/bin/runsc") - return downloadFileToDest("http://storage.googleapis.com/gvisor/releases/nightly/latest/runsc", dest) + return downloadFileToDest(constants.GvisorURL, dest) } // downloadFileToDest downlaods the given file to the dest @@ -136,7 +148,12 @@ func downloadFileToDest(url, dest string) error { // Must write the following files: // 1. gvisor-containerd-shim.toml // 2. gvisor containerd config.toml +// and save the default version of config.toml func copyFiles() error { + log.Printf("Storing default config.toml at %s", constants.StoredContainerdConfigTomlPath) + if err := mcnutils.CopyFile(filepath.Join(nodeDir, constants.ContainerdConfigTomlPath), filepath.Join(nodeDir, constants.StoredContainerdConfigTomlPath)); err != nil { + return errors.Wrap(err, "copying default config.toml") + } log.Print("Copying gvisor-containerd-shim.toml...") if err := copyAssetToDest(constants.GvisorContainerdShimTargetName, filepath.Join(nodeDir, constants.GvisorContainerdShimTomlPath)); err != nil { return errors.Wrap(err, "copying gvisor-containerd-shim.toml") diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index 5b137e2b3005..90921449b440 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -292,11 +292,6 @@ var Addons = map[string]*Addon{ constants.GvisorFilesPath, constants.GvisorContainerdShimTargetName, "0640"), - NewBinDataAsset( - "deploy/addons/gvisor/config.toml", - constants.GvisorFilesPath, - constants.DefaultConfigTomlTargetName, - "0640"), }, false, "gvisor"), } diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index b1e0caa466ad..26287279737a 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -270,11 +270,16 @@ const ( ContainerdConfigTomlPath = "/etc/containerd/config.toml" // GvisorContainerdShimTomlPath is the path to givosr-containerd-shim.toml GvisorContainerdShimTomlPath = "/etc/containerd/gvisor-containerd-shim.toml" + // StoredContainerdConfigTomlPath is the path where the default config.toml will be stored + StoredContainerdConfigTomlPath = "/tmp/config.toml" //GvisorConfigTomlTargetName is the go-bindata target name for the gvisor config.toml GvisorConfigTomlTargetName = "gvisor-config.toml" // GvisorContainerdShimTargetName is the go-bindata target name for gvisor-containerd-shim GvisorContainerdShimTargetName = "gvisor-containerd-shim.toml" - // DefaultConfigTomlTargetName is the go-bindata target name for the default config.toml - DefaultConfigTomlTargetName = "config.toml" + + // GvisorContainerdShimURL is the url to download gvisor-containerd-shim + GvisorContainerdShimURL = "https://github.com/google/gvisor-containerd-shim/releases/download/v0.0.1-rc.0/gvisor-containerd-shim-v0.0.1-rc.0.linux-amd64" + // GvisorURL is the url to download gvisor + GvisorURL = "http://storage.googleapis.com/gvisor/releases/nightly/latest/runsc" ) From c894943bc0016225a377a8b442e9ca99084ca98b Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 4 Dec 2018 14:09:26 -0800 Subject: [PATCH 09/23] Simplify gvisor and check for containerd runtime When enabling gvisor, first validate that the container runtime is containerd. --- cmd/gvisor/gvisor.go | 19 +----------------- cmd/minikube/cmd/config/config.go | 2 +- cmd/minikube/cmd/config/validations.go | 27 ++++++++++++++++++++++---- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/cmd/gvisor/gvisor.go b/cmd/gvisor/gvisor.go index 73821cff20a0..d8c52b644b8c 100644 --- a/cmd/gvisor/gvisor.go +++ b/cmd/gvisor/gvisor.go @@ -17,32 +17,15 @@ limitations under the License. package main import ( - "flag" "log" "os" "k8s.io/minikube/pkg/gvisor" ) -var ( - disable bool -) - -func init() { - flag.BoolVar(&disable, "disable", false, "disable gvisor addon") - flag.Parse() -} - func main() { - if err := execute(); err != nil { + if err := gvisor.Enable(); err != nil { log.Print(err) os.Exit(1) } } - -func execute() error { - if disable { - return gvisor.Disable() - } - return gvisor.Enable() -} diff --git a/cmd/minikube/cmd/config/config.go b/cmd/minikube/cmd/config/config.go index c9181839cb68..47029bd8d170 100644 --- a/cmd/minikube/cmd/config/config.go +++ b/cmd/minikube/cmd/config/config.go @@ -227,7 +227,7 @@ var settings = []Setting{ { name: "gvisor", set: SetBool, - validations: []setFn{IsValidAddon}, + validations: []setFn{IsValidAddon, IsContainerdRuntime}, callbacks: []setFn{EnableOrDisableAddon}, }, { diff --git a/cmd/minikube/cmd/config/validations.go b/cmd/minikube/cmd/config/validations.go index e9ee37d8ece0..c7fd3d349123 100644 --- a/cmd/minikube/cmd/config/validations.go +++ b/cmd/minikube/cmd/config/validations.go @@ -18,15 +18,17 @@ package config import ( "fmt" - "github.com/docker/go-units" - "github.com/pkg/errors" - "k8s.io/minikube/pkg/minikube/assets" - "k8s.io/minikube/pkg/minikube/constants" "net" "net/url" "os" "strconv" "strings" + + "github.com/docker/go-units" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/constants" ) func IsValidDriver(string, driver string) error { @@ -125,3 +127,20 @@ func IsValidAddon(name string, val string) error { } return errors.Errorf("Cannot enable/disable invalid addon %s", name) } + +func IsContainerdRuntime(_, _ string) error { + config, err := config.Load() + if err != nil { + return fmt.Errorf("error getting cluster config: %v", err) + } + if config.KubernetesConfig.ContainerRuntime != constants.ContainerdRuntime { + return fmt.Errorf(`This addon can only be enabled with the containerd runtime backend. + +To enable this backend, please start minikube again with the following flags: + +minikube start --container-runtime=containerd --docker-opt containerd=/var/run/containerd/containerd.sock --network-plugin=cni + `) + } + + return nil +} From f9915ac044c601742d894f5803281956d7943532 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Tue, 4 Dec 2018 14:19:50 -0800 Subject: [PATCH 10/23] small improvements --- cmd/minikube/cmd/config/validations.go | 3 +-- pkg/gvisor/enable.go | 13 +++++++------ pkg/minikube/constants/constants.go | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/minikube/cmd/config/validations.go b/cmd/minikube/cmd/config/validations.go index c7fd3d349123..fe088cd7243c 100644 --- a/cmd/minikube/cmd/config/validations.go +++ b/cmd/minikube/cmd/config/validations.go @@ -138,8 +138,7 @@ func IsContainerdRuntime(_, _ string) error { To enable this backend, please start minikube again with the following flags: -minikube start --container-runtime=containerd --docker-opt containerd=/var/run/containerd/containerd.sock --network-plugin=cni - `) +minikube start --container-runtime=containerd --docker-opt containerd=/var/run/containerd/containerd.sock --network-plugin=cni`) } return nil diff --git a/pkg/gvisor/enable.go b/pkg/gvisor/enable.go index ec553ec76c64..89f651196c20 100644 --- a/pkg/gvisor/enable.go +++ b/pkg/gvisor/enable.go @@ -45,14 +45,14 @@ const ( // 3. copies necessary containerd config files // 4. restarts containerd func Enable() error { - if err := makeDirs(); err != nil { + if err := makeGvisorDirs(); err != nil { return errors.Wrap(err, "creating directories on node") } if err := downloadBinaries(); err != nil { return errors.Wrap(err, "downloading binaries") } - if err := copyFiles(); err != nil { - return errors.Wrap(err, "copying files") + if err := copyConfigFiles(); err != nil { + return errors.Wrap(err, "copying config files") } if err := restartContainerd(); err != nil { return errors.Wrap(err, "restarting containerd") @@ -64,6 +64,7 @@ func Enable() error { <-c if err := Disable(); err != nil { log.Printf("Error disabling gvisor: %v", err) + os.Exit(1) } os.Exit(0) }() @@ -73,8 +74,8 @@ func Enable() error { return nil } -// makeDirs creates necessary directories on the node -func makeDirs() error { +// makeGvisorDirs creates necessary directories on the node +func makeGvisorDirs() error { // Make /run/containerd/runsc to hold logs fp := filepath.Join(nodeDir, "run/containerd/runsc") if err := os.MkdirAll(fp, 0755); err != nil { @@ -149,7 +150,7 @@ func downloadFileToDest(url, dest string) error { // 1. gvisor-containerd-shim.toml // 2. gvisor containerd config.toml // and save the default version of config.toml -func copyFiles() error { +func copyConfigFiles() error { log.Printf("Storing default config.toml at %s", constants.StoredContainerdConfigTomlPath) if err := mcnutils.CopyFile(filepath.Join(nodeDir, constants.ContainerdConfigTomlPath), filepath.Join(nodeDir, constants.StoredContainerdConfigTomlPath)); err != nil { return errors.Wrap(err, "copying default config.toml") diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 26287279737a..3d05b99f437c 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -281,5 +281,5 @@ const ( // GvisorContainerdShimURL is the url to download gvisor-containerd-shim GvisorContainerdShimURL = "https://github.com/google/gvisor-containerd-shim/releases/download/v0.0.1-rc.0/gvisor-containerd-shim-v0.0.1-rc.0.linux-amd64" // GvisorURL is the url to download gvisor - GvisorURL = "http://storage.googleapis.com/gvisor/releases/nightly/latest/runsc" + GvisorURL = "https://storage.googleapis.com/gvisor/releases/nightly/2018-11-08/runsc" ) From 4a8de41c397b92f1157c9b484c763762fac20c21 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 5 Dec 2018 11:19:39 -0800 Subject: [PATCH 11/23] Add gvisor integration test Added integration test which follows these steps: 1. enable gvisor 2. make sure untrusted workload runs correctly 3. disable gvisor 4. make sure untrusted workload results in FailedCreateSandboxEvent event I also added a link to the iso url for starting containerd until the integration tests start using the new version of the iso. --- deploy/addons/gvisor/gvisor-pod.yaml | 1 + pkg/util/kubernetes.go | 30 ++++++++++ test/integration/addons_test.go | 36 ++++++++++++ test/integration/functional_test.go | 14 +++++ .../integration/testdata/nginx-untrusted.yaml | 12 ++++ test/integration/util/util.go | 57 ++++++++++++++++++- 6 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 test/integration/testdata/nginx-untrusted.yaml diff --git a/deploy/addons/gvisor/gvisor-pod.yaml b/deploy/addons/gvisor/gvisor-pod.yaml index 516893a0a97f..8b281d7390a4 100644 --- a/deploy/addons/gvisor/gvisor-pod.yaml +++ b/deploy/addons/gvisor/gvisor-pod.yaml @@ -18,6 +18,7 @@ metadata: name: gvisor namespace: kube-system labels: + minikube-addon: gvisor addonmanager.kubernetes.io/mode: Reconcile spec: hostPID: true diff --git a/pkg/util/kubernetes.go b/pkg/util/kubernetes.go index 45b3ee1e5f02..927cf1dd7835 100644 --- a/pkg/util/kubernetes.go +++ b/pkg/util/kubernetes.go @@ -139,6 +139,36 @@ func WaitForPodsWithLabelRunning(c kubernetes.Interface, ns string, label labels }) } +// Wait up to 10 minutes for a pod to be deleted +func WaitForPodDelete(c kubernetes.Interface, ns string, label labels.Selector) error { + return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) { + listOpts := metav1.ListOptions{LabelSelector: label.String()} + pods, err := c.CoreV1().Pods(ns).List(listOpts) + if err != nil { + glog.Infof("error getting Pods with label selector %q [%v]\n", label.String(), err) + return false, nil + } + return len(pods.Items) == 0, nil + }) +} + +// Wait up to 10 minutes for the given event to appear +func WaitForEvent(c kubernetes.Interface, ns string, reason string) error { + return wait.PollImmediate(constants.APICallRetryInterval, time.Minute*10, func() (bool, error) { + events, err := c.Events().Events("default").List(metav1.ListOptions{}) + if err != nil { + glog.Infof("error getting events: %v", err) + return false, nil + } + for _, e := range events.Items { + if e.Reason == reason { + return true, nil + } + } + return false, nil + }) +} + // WaitForRCToStabilize waits till the RC has a matching generation/replica count between spec and status. func WaitForRCToStabilize(c kubernetes.Interface, ns, name string, timeout time.Duration) error { options := metav1.ListOptions{FieldSelector: fields.Set{ diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 9c2dbdfa8428..11bc96172a40 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -183,3 +183,39 @@ func testServicesList(t *testing.T) { t.Fatalf(err.Error()) } } + +func testGvisor(t *testing.T) { + minikubeRunner := NewMinikubeRunner(t) + kubectlRunner := util.NewKubectlRunner(t) + + minikubeRunner.RunCommand("addons enable gvisor", true) + if err := util.WaitForGvisorControllerRunning(); err != nil { + t.Fatalf("waiting for gvisor controller to be up: %v", err) + } + + untrustedPath, _ := filepath.Abs("testdata/nginx-untrusted.yaml") + if _, err := kubectlRunner.RunCommand([]string{"create", "-f", untrustedPath}); err != nil { + t.Fatalf("creating untrusted nginx resource: %v", err) + } + + if err := util.WaitForUntrustedNginxRunning(); err != nil { + t.Fatalf("waiting for nginx to be up: %v", err) + } + + minikubeRunner.RunCommand("addons disable gvisor", true) + if err := util.WaitForGvisorControllerDeleted(); err != nil { + t.Fatalf("waiting for gvisor controller to be deleted: %v", err) + } + + if _, err := kubectlRunner.RunCommand([]string{"replace", "-f", untrustedPath, "--force"}); err != nil { + t.Fatalf("replacing untrusted nginx resource: %v", err) + } + + if err := util.WaitForFailedCreatePodSandBoxEvent(); err != nil { + t.Fatalf("waiting for FailedCreatePodSandBox event: %v", err) + } + + if _, err := kubectlRunner.RunCommand([]string{"delete", "-f", untrustedPath}); err != nil { + t.Logf("deleting untrusted nginx resource: %v", err) + } +} diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index b3bf68df542a..4dcc0aa01585 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -22,6 +22,7 @@ import ( "strings" "testing" + "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/test/integration/util" ) @@ -48,6 +49,19 @@ func TestFunctional(t *testing.T) { } } +func TestFunctionalContainerd(t *testing.T) { + minikubeRunner := NewMinikubeRunner(t) + + if usingNoneDriver(minikubeRunner) { + t.Skip("Can't run containerd backend with none driver") + } + + minikubeRunner.SetRuntime(constants.ContainerdRuntime) + minikubeRunner.EnsureRunning() + + t.Run("Gvisor", testGvisor) +} + // usingNoneDriver returns true if using the none driver func usingNoneDriver(runner util.MinikubeRunner) bool { return strings.Contains(runner.StartArgs, "--vm-driver=none") diff --git a/test/integration/testdata/nginx-untrusted.yaml b/test/integration/testdata/nginx-untrusted.yaml new file mode 100644 index 000000000000..7ff9ce9fb704 --- /dev/null +++ b/test/integration/testdata/nginx-untrusted.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx-untrusted + labels: + run: nginx + annotations: + io.kubernetes.cri.untrusted-workload: "true" +spec: + containers: + - name: nginx + image: nginx diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 8bb069ba9200..e02226837124 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -126,7 +126,8 @@ func (m *MinikubeRunner) SSH(command string) (string, error) { func (m *MinikubeRunner) Start() { switch r := m.Runtime; r { case constants.ContainerdRuntime: - containerdFlags := "--container-runtime=containerd --network-plugin=cni --docker-opt containerd=/var/run/containerd/containerd.sock" + // TODO: priyawadhwa@ remove iso url once updated iso is being used in integration tests + containerdFlags := "--container-runtime=containerd --network-plugin=cni --docker-opt containerd=/var/run/containerd/containerd.sock --iso-url=https://storage.googleapis.com/minikube-iso/minikube.iso" m.RunCommand(fmt.Sprintf("start %s %s %s", m.StartArgs, m.Args, containerdFlags), true) default: m.RunCommand(fmt.Sprintf("start %s %s", m.StartArgs, m.Args), true) @@ -298,6 +299,60 @@ func WaitForIngressDefaultBackendRunning(t *testing.T) error { return nil } +// WaitForGvisorControllerRunning waits for the gvisor controller pod to be running +func WaitForGvisorControllerRunning() error { + client, err := commonutil.GetClient() + if err != nil { + return errors.Wrap(err, "getting kubernetes client") + } + + selector := labels.SelectorFromSet(labels.Set(map[string]string{"minikube-addon": "gvisor"})) + if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil { + return errors.Wrap(err, "waiting for gvisor controller pod to stabilize") + } + return nil +} + +// WaitForGvisorControllerDeleted waits for the gvisor controller pod to be deleted +func WaitForGvisorControllerDeleted() error { + client, err := commonutil.GetClient() + if err != nil { + return errors.Wrap(err, "getting kubernetes client") + } + + selector := labels.SelectorFromSet(labels.Set(map[string]string{"minikube-addon": "gvisor"})) + if err := commonutil.WaitForPodDelete(client, "kube-system", selector); err != nil { + return errors.Wrap(err, "waiting for gvisor controller pod deletion") + } + return nil +} + +// WaitForUntrustedNginxRunning waits for the untrusted nginx pod to start running +func WaitForUntrustedNginxRunning() error { + client, err := commonutil.GetClient() + if err != nil { + return errors.Wrap(err, "getting kubernetes client") + } + + selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx"})) + if err := commonutil.WaitForPodsWithLabelRunning(client, "default", selector); err != nil { + return errors.Wrap(err, "waiting for nginx pods") + } + return nil +} + +// WaitForFailedCreatePodSandBoxEvent waits for a FailedCreatePodSandBox event to show appear +func WaitForFailedCreatePodSandBoxEvent() error { + client, err := commonutil.GetClient() + if err != nil { + return errors.Wrap(err, "getting kubernetes client") + } + if err := commonutil.WaitForEvent(client, "default", "FailedCreatePodSandBox"); err != nil { + return errors.Wrap(err, "waiting for FailedCreatePodSandBox event") + } + return nil +} + func WaitForNginxRunning(t *testing.T) error { client, err := commonutil.GetClient() From 7bee8701b7bdba2ad410f76a43dc7cf625d88138 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 5 Dec 2018 12:32:11 -0800 Subject: [PATCH 12/23] Show gvisor pod logs upon test failure --- test/integration/addons_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 11bc96172a40..e6cdc10cf690 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -190,7 +190,12 @@ func testGvisor(t *testing.T) { minikubeRunner.RunCommand("addons enable gvisor", true) if err := util.WaitForGvisorControllerRunning(); err != nil { - t.Fatalf("waiting for gvisor controller to be up: %v", err) + // Print out logs from controller + out, err := kubectlRunner.RunCommand([]string{"logs", "gvisor", "-n", "kube-system"}) + if err != nil { + t.Errorf("error getting gvisor logs: %v", err) + } + t.Fatalf("waiting for gvisor controller to be up: %v \n %s", err, string(out)) } untrustedPath, _ := filepath.Abs("testdata/nginx-untrusted.yaml") From 765cc933207fd3f4cb30634609e76366902666f9 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 5 Dec 2018 13:59:22 -0800 Subject: [PATCH 13/23] Additional logs --- deploy/addons/gvisor/gvisor-pod.yaml | 2 +- test/integration/addons_test.go | 18 ++++++++++++++---- test/integration/util/util.go | 4 ++-- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/deploy/addons/gvisor/gvisor-pod.yaml b/deploy/addons/gvisor/gvisor-pod.yaml index 8b281d7390a4..50a54bb69336 100644 --- a/deploy/addons/gvisor/gvisor-pod.yaml +++ b/deploy/addons/gvisor/gvisor-pod.yaml @@ -18,8 +18,8 @@ metadata: name: gvisor namespace: kube-system labels: - minikube-addon: gvisor addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/minikube-addons: gvisor spec: hostPID: true containers: diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index e6cdc10cf690..f9e1e159dc54 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -188,39 +188,49 @@ func testGvisor(t *testing.T) { minikubeRunner := NewMinikubeRunner(t) kubectlRunner := util.NewKubectlRunner(t) + t.Log("enabling gvisor addon") minikubeRunner.RunCommand("addons enable gvisor", true) + t.Log("waiting for gvisor controller to come up") if err := util.WaitForGvisorControllerRunning(); err != nil { // Print out logs from controller - out, err := kubectlRunner.RunCommand([]string{"logs", "gvisor", "-n", "kube-system"}) - if err != nil { + if _, err := kubectlRunner.RunCommand([]string{"get", "pods", "--all-namespaces"}); err != nil { + t.Errorf("error getting all pods %v", err) + } + if _, err := kubectlRunner.RunCommand([]string{"logs", "gvisor", "-n", "kube-system"}); err != nil { t.Errorf("error getting gvisor logs: %v", err) } - t.Fatalf("waiting for gvisor controller to be up: %v \n %s", err, string(out)) + t.Fatalf("waiting for gvisor controller to be up: %v", err) } untrustedPath, _ := filepath.Abs("testdata/nginx-untrusted.yaml") + t.Log("creating pod with untrusted workload annotation") if _, err := kubectlRunner.RunCommand([]string{"create", "-f", untrustedPath}); err != nil { t.Fatalf("creating untrusted nginx resource: %v", err) } + t.Log("making sure untrusted workload is Running") if err := util.WaitForUntrustedNginxRunning(); err != nil { t.Fatalf("waiting for nginx to be up: %v", err) } + t.Log("disabling gvisor addon") minikubeRunner.RunCommand("addons disable gvisor", true) + t.Log("waiting for gvisor controller pod to be deleted") if err := util.WaitForGvisorControllerDeleted(); err != nil { t.Fatalf("waiting for gvisor controller to be deleted: %v", err) } + t.Log("recreating untrusted workload pod") if _, err := kubectlRunner.RunCommand([]string{"replace", "-f", untrustedPath, "--force"}); err != nil { t.Fatalf("replacing untrusted nginx resource: %v", err) } + t.Log("waiting for FailedCreatePodSandBox event") if err := util.WaitForFailedCreatePodSandBoxEvent(); err != nil { t.Fatalf("waiting for FailedCreatePodSandBox event: %v", err) } if _, err := kubectlRunner.RunCommand([]string{"delete", "-f", untrustedPath}); err != nil { - t.Logf("deleting untrusted nginx resource: %v", err) + t.Logf("error deleting untrusted nginx resource: %v", err) } } diff --git a/test/integration/util/util.go b/test/integration/util/util.go index e02226837124..db13af91e7fa 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -306,7 +306,7 @@ func WaitForGvisorControllerRunning() error { return errors.Wrap(err, "getting kubernetes client") } - selector := labels.SelectorFromSet(labels.Set(map[string]string{"minikube-addon": "gvisor"})) + selector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "gvisor"})) if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil { return errors.Wrap(err, "waiting for gvisor controller pod to stabilize") } @@ -320,7 +320,7 @@ func WaitForGvisorControllerDeleted() error { return errors.Wrap(err, "getting kubernetes client") } - selector := labels.SelectorFromSet(labels.Set(map[string]string{"minikube-addon": "gvisor"})) + selector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "gvisor"})) if err := commonutil.WaitForPodDelete(client, "kube-system", selector); err != nil { return errors.Wrap(err, "waiting for gvisor controller pod deletion") } From 6e6020f300bbfd51d84ac52b8039b5eaf805a06c Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 5 Dec 2018 15:40:25 -0800 Subject: [PATCH 14/23] Code review comments and added logs --- cmd/minikube/cmd/config/validations.go | 6 +++++- pkg/gvisor/disable.go | 2 +- test/integration/addons_test.go | 7 +++++-- test/integration/util/util.go | 6 +++--- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/cmd/minikube/cmd/config/validations.go b/cmd/minikube/cmd/config/validations.go index fe088cd7243c..3c1b28a64136 100644 --- a/cmd/minikube/cmd/config/validations.go +++ b/cmd/minikube/cmd/config/validations.go @@ -136,7 +136,11 @@ func IsContainerdRuntime(_, _ string) error { if config.KubernetesConfig.ContainerRuntime != constants.ContainerdRuntime { return fmt.Errorf(`This addon can only be enabled with the containerd runtime backend. -To enable this backend, please start minikube again with the following flags: +To enable this backend, please first stop minikube with: + +minikube stop + +and then start minikube again with the following flags: minikube start --container-runtime=containerd --docker-opt containerd=/var/run/containerd/containerd.sock --network-plugin=cni`) } diff --git a/pkg/gvisor/disable.go b/pkg/gvisor/disable.go index 881a4eca47ec..4c00bbaea0c9 100644 --- a/pkg/gvisor/disable.go +++ b/pkg/gvisor/disable.go @@ -32,7 +32,7 @@ func Disable() error { if err := os.Remove(filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil { return errors.Wrapf(err, "removing %s", constants.ContainerdConfigTomlPath) } - log.Printf("Storing default config.toml at %s", constants.ContainerdConfigTomlPath) + log.Printf("Restoring default config.toml at %s", constants.ContainerdConfigTomlPath) if err := mcnutils.CopyFile(filepath.Join(nodeDir, constants.StoredContainerdConfigTomlPath), filepath.Join(nodeDir, constants.ContainerdConfigTomlPath)); err != nil { return errors.Wrap(err, "reverting back to default config.toml") } diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index f9e1e159dc54..3830f6a6b6e9 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -22,6 +22,7 @@ import ( "bufio" "fmt" "io/ioutil" + "log" "net" "net/http" "net/url" @@ -191,11 +192,13 @@ func testGvisor(t *testing.T) { t.Log("enabling gvisor addon") minikubeRunner.RunCommand("addons enable gvisor", true) t.Log("waiting for gvisor controller to come up") - if err := util.WaitForGvisorControllerRunning(); err != nil { + if err := util.WaitForGvisorControllerRunning(t); err != nil { // Print out logs from controller - if _, err := kubectlRunner.RunCommand([]string{"get", "pods", "--all-namespaces"}); err != nil { + out, err := kubectlRunner.RunCommand([]string{"get", "pods", "--all-namespaces"}) + if err != nil { t.Errorf("error getting all pods %v", err) } + log.Print(string(out)) if _, err := kubectlRunner.RunCommand([]string{"logs", "gvisor", "-n", "kube-system"}); err != nil { t.Errorf("error getting gvisor logs: %v", err) } diff --git a/test/integration/util/util.go b/test/integration/util/util.go index db13af91e7fa..a54e96aee892 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -127,7 +127,7 @@ func (m *MinikubeRunner) Start() { switch r := m.Runtime; r { case constants.ContainerdRuntime: // TODO: priyawadhwa@ remove iso url once updated iso is being used in integration tests - containerdFlags := "--container-runtime=containerd --network-plugin=cni --docker-opt containerd=/var/run/containerd/containerd.sock --iso-url=https://storage.googleapis.com/minikube-iso/minikube.iso" + containerdFlags := "--container-runtime=containerd --network-plugin=cni --docker-opt containerd=/var/run/containerd/containerd.sock --iso-url=https://storage.googleapis.com/gvisor-preview/minikube.iso" m.RunCommand(fmt.Sprintf("start %s %s %s", m.StartArgs, m.Args, containerdFlags), true) default: m.RunCommand(fmt.Sprintf("start %s %s", m.StartArgs, m.Args), true) @@ -300,7 +300,7 @@ func WaitForIngressDefaultBackendRunning(t *testing.T) error { } // WaitForGvisorControllerRunning waits for the gvisor controller pod to be running -func WaitForGvisorControllerRunning() error { +func WaitForGvisorControllerRunning(t *testing.T) error { client, err := commonutil.GetClient() if err != nil { return errors.Wrap(err, "getting kubernetes client") @@ -341,7 +341,7 @@ func WaitForUntrustedNginxRunning() error { return nil } -// WaitForFailedCreatePodSandBoxEvent waits for a FailedCreatePodSandBox event to show appear +// WaitForFailedCreatePodSandBoxEvent waits for a FailedCreatePodSandBox event to appear func WaitForFailedCreatePodSandBoxEvent() error { client, err := commonutil.GetClient() if err != nil { From 1c01497f8eda480332b5b041ec05d802284ce4a1 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 5 Dec 2018 15:57:57 -0800 Subject: [PATCH 15/23] Remove default runsc debug as it's too verbose --- deploy/addons/gvisor/gvisor-containerd-shim.toml | 2 -- pkg/minikube/constants/constants.go | 1 - 2 files changed, 3 deletions(-) diff --git a/deploy/addons/gvisor/gvisor-containerd-shim.toml b/deploy/addons/gvisor/gvisor-containerd-shim.toml index 13da23b0fa8d..b4afb9c5150c 100644 --- a/deploy/addons/gvisor/gvisor-containerd-shim.toml +++ b/deploy/addons/gvisor/gvisor-containerd-shim.toml @@ -1,5 +1,3 @@ runc_shim = "/bin/containerd-shim" [runsc_config] - debug-log="/tmp/runsc/" - debug="true" user-log="/tmp/runsc/user-log-%ID%.log" \ No newline at end of file diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 3d05b99f437c..13c401d4251a 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -22,7 +22,6 @@ import ( "path/filepath" "runtime" "strings" - "time" "github.com/blang/semver" From 830f8f8301b5ab187f2c7e4591aa6df9a3ca4219 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 5 Dec 2018 16:03:57 -0800 Subject: [PATCH 16/23] change iso url --- test/integration/util/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/util/util.go b/test/integration/util/util.go index a54e96aee892..081a60fb0f7d 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -127,7 +127,7 @@ func (m *MinikubeRunner) Start() { switch r := m.Runtime; r { case constants.ContainerdRuntime: // TODO: priyawadhwa@ remove iso url once updated iso is being used in integration tests - containerdFlags := "--container-runtime=containerd --network-plugin=cni --docker-opt containerd=/var/run/containerd/containerd.sock --iso-url=https://storage.googleapis.com/gvisor-preview/minikube.iso" + containerdFlags := "--container-runtime=containerd --network-plugin=cni --docker-opt containerd=/var/run/containerd/containerd.sock --iso-url=https://storage.googleapis.com/k8s-minikube/gvisor-preview.iso" m.RunCommand(fmt.Sprintf("start %s %s %s", m.StartArgs, m.Args, containerdFlags), true) default: m.RunCommand(fmt.Sprintf("start %s %s", m.StartArgs, m.Args), true) From 2ec2e53c2d1d4468d7378dbc85a2b746e878b0f2 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Wed, 5 Dec 2018 16:52:58 -0800 Subject: [PATCH 17/23] More logs --- test/integration/addons_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 3830f6a6b6e9..ec7114f7d50f 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -195,10 +195,23 @@ func testGvisor(t *testing.T) { if err := util.WaitForGvisorControllerRunning(t); err != nil { // Print out logs from controller out, err := kubectlRunner.RunCommand([]string{"get", "pods", "--all-namespaces"}) + log.Print(string(out)) if err != nil { t.Errorf("error getting all pods %v", err) } + + out, err = kubectlRunner.RunCommand([]string{"logs", "kube-addon-manager-minikube", "-n", "kube-system"}) log.Print(string(out)) + if err != nil { + t.Errorf("error getting addon manager logs%v", err) + } + + output, err := minikubeRunner.CombinedOutput("addons list") + log.Print(output) + if err != nil { + t.Errorf("error getting addons list") + } + if _, err := kubectlRunner.RunCommand([]string{"logs", "gvisor", "-n", "kube-system"}); err != nil { t.Errorf("error getting gvisor logs: %v", err) } From a3cb54c4883c0624f803011ceee43c5dce37e36a Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 6 Dec 2018 11:06:40 -0800 Subject: [PATCH 18/23] Code review comments and additional logs --- deploy/addons/gvisor/README.md | 2 ++ pkg/gvisor/enable.go | 4 +--- test/integration/addons_test.go | 8 +++++--- test/integration/util/util.go | 1 + 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/deploy/addons/gvisor/README.md b/deploy/addons/gvisor/README.md index bac4e891cc29..4bae2b2a7388 100644 --- a/deploy/addons/gvisor/README.md +++ b/deploy/addons/gvisor/README.md @@ -50,6 +50,8 @@ spec: image: nginx ``` +_Note: this annotation will not be necessary once the RuntimeClass Kubernetes feature is available broadly._ + ### Disabling gVisor To disable gVisor, run: diff --git a/pkg/gvisor/enable.go b/pkg/gvisor/enable.go index 89f651196c20..7f5aa807fdeb 100644 --- a/pkg/gvisor/enable.go +++ b/pkg/gvisor/enable.go @@ -27,7 +27,6 @@ import ( "os/signal" "path/filepath" "syscall" - "time" "github.com/docker/machine/libmachine/mcnutils" "github.com/pkg/errors" @@ -70,8 +69,7 @@ func Enable() error { }() log.Print("gvisor successfully enabled in cluster") // sleep for one year so the pod continuously runs - time.Sleep(24 * 7 * 52 * time.Hour) - return nil + select {} } // makeGvisorDirs creates necessary directories on the node diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index ec7114f7d50f..4b156076c77b 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -190,7 +190,9 @@ func testGvisor(t *testing.T) { kubectlRunner := util.NewKubectlRunner(t) t.Log("enabling gvisor addon") - minikubeRunner.RunCommand("addons enable gvisor", true) + output := minikubeRunner.RunCommand("addons enable gvisor", true) + t.Log(output) + t.Log("waiting for gvisor controller to come up") if err := util.WaitForGvisorControllerRunning(t); err != nil { // Print out logs from controller @@ -206,10 +208,10 @@ func testGvisor(t *testing.T) { t.Errorf("error getting addon manager logs%v", err) } - output, err := minikubeRunner.CombinedOutput("addons list") + output, err := minikubeRunner.CombinedOutput("ls /etc/kubernetes/addons") log.Print(output) if err != nil { - t.Errorf("error getting addons list") + t.Errorf("getting files in /etc/kubernetes/addons") } if _, err := kubectlRunner.RunCommand([]string{"logs", "gvisor", "-n", "kube-system"}); err != nil { diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 081a60fb0f7d..eed43dcc493f 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -71,6 +71,7 @@ func (m *MinikubeRunner) RunCommand(command string, checkError bool) string { commandArr := strings.Split(command, " ") path, _ := filepath.Abs(m.BinaryPath) cmd := exec.Command(path, commandArr...) + m.T.Logf("Executing command: %s", cmd) stdout, err := cmd.Output() if checkError && err != nil { From 0e4a28c60e1108615d8309f316efaf1faae96cc7 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 6 Dec 2018 11:33:35 -0800 Subject: [PATCH 19/23] delete minikube in test before starting with containerd also remove extra logs --- test/integration/addons_test.go | 28 +--------------------------- test/integration/functional_test.go | 1 + test/integration/util/util.go | 1 - 3 files changed, 2 insertions(+), 28 deletions(-) diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 4b156076c77b..7d65b970222a 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -22,7 +22,6 @@ import ( "bufio" "fmt" "io/ioutil" - "log" "net" "net/http" "net/url" @@ -188,35 +187,10 @@ func testServicesList(t *testing.T) { func testGvisor(t *testing.T) { minikubeRunner := NewMinikubeRunner(t) kubectlRunner := util.NewKubectlRunner(t) - - t.Log("enabling gvisor addon") - output := minikubeRunner.RunCommand("addons enable gvisor", true) - t.Log(output) + minikubeRunner.RunCommand("addons enable gvisor", true) t.Log("waiting for gvisor controller to come up") if err := util.WaitForGvisorControllerRunning(t); err != nil { - // Print out logs from controller - out, err := kubectlRunner.RunCommand([]string{"get", "pods", "--all-namespaces"}) - log.Print(string(out)) - if err != nil { - t.Errorf("error getting all pods %v", err) - } - - out, err = kubectlRunner.RunCommand([]string{"logs", "kube-addon-manager-minikube", "-n", "kube-system"}) - log.Print(string(out)) - if err != nil { - t.Errorf("error getting addon manager logs%v", err) - } - - output, err := minikubeRunner.CombinedOutput("ls /etc/kubernetes/addons") - log.Print(output) - if err != nil { - t.Errorf("getting files in /etc/kubernetes/addons") - } - - if _, err := kubectlRunner.RunCommand([]string{"logs", "gvisor", "-n", "kube-system"}); err != nil { - t.Errorf("error getting gvisor logs: %v", err) - } t.Fatalf("waiting for gvisor controller to be up: %v", err) } diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 4dcc0aa01585..2e47b5965086 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -56,6 +56,7 @@ func TestFunctionalContainerd(t *testing.T) { t.Skip("Can't run containerd backend with none driver") } + minikubeRunner.RunCommand("delete", true) minikubeRunner.SetRuntime(constants.ContainerdRuntime) minikubeRunner.EnsureRunning() diff --git a/test/integration/util/util.go b/test/integration/util/util.go index eed43dcc493f..081a60fb0f7d 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -71,7 +71,6 @@ func (m *MinikubeRunner) RunCommand(command string, checkError bool) string { commandArr := strings.Split(command, " ") path, _ := filepath.Abs(m.BinaryPath) cmd := exec.Command(path, commandArr...) - m.T.Logf("Executing command: %s", cmd) stdout, err := cmd.Output() if checkError && err != nil { From c976e11b263da2fff5f8d10d088b8638abb2515e Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 6 Dec 2018 13:06:56 -0800 Subject: [PATCH 20/23] delete after finishing up containerd --- test/integration/functional_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 2e47b5965086..5a761a70e5c7 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -61,6 +61,7 @@ func TestFunctionalContainerd(t *testing.T) { minikubeRunner.EnsureRunning() t.Run("Gvisor", testGvisor) + minikubeRunner.RunCommand("delete", true) } // usingNoneDriver returns true if using the none driver From 1073ea070267e442db1d55858cd85389ff700f55 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 6 Dec 2018 14:14:20 -0800 Subject: [PATCH 21/23] Add minikube User-Agent to requests for shim and runsc --- pkg/gvisor/enable.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/gvisor/enable.go b/pkg/gvisor/enable.go index 7f5aa807fdeb..2268c7381d2e 100644 --- a/pkg/gvisor/enable.go +++ b/pkg/gvisor/enable.go @@ -120,7 +120,13 @@ func runsc() error { // downloadFileToDest downlaods the given file to the dest // if something already exists at dest, first remove it func downloadFileToDest(url, dest string) error { - resp, err := http.Get(url) + client := &http.Client{} + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return errors.Wrapf(err, "creating request for %s", url) + } + req.Header.Set("User-Agent", "minikube") + resp, err := client.Do(req) if err != nil { return err } From af6d30bdfefbde06226fff07b8c9b8d210f0913a Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Thu, 6 Dec 2018 14:51:46 -0800 Subject: [PATCH 22/23] Only delete minikube if there is already one running --- test/integration/functional_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 5a761a70e5c7..08ca9e171b8c 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -22,6 +22,7 @@ import ( "strings" "testing" + "github.com/docker/machine/libmachine/state" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/test/integration/util" ) @@ -56,7 +57,10 @@ func TestFunctionalContainerd(t *testing.T) { t.Skip("Can't run containerd backend with none driver") } - minikubeRunner.RunCommand("delete", true) + if minikubeRunner.GetStatus() != state.None.String() { + minikubeRunner.RunCommand("delete", true) + } + minikubeRunner.SetRuntime(constants.ContainerdRuntime) minikubeRunner.EnsureRunning() From c7e53416eb76772a83b705bc9307422887380fb8 Mon Sep 17 00:00:00 2001 From: Priya Wadhwa Date: Fri, 7 Dec 2018 14:31:53 -0800 Subject: [PATCH 23/23] update gvisor url --- pkg/minikube/constants/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 13c401d4251a..c278dcbefc2a 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -280,5 +280,5 @@ const ( // GvisorContainerdShimURL is the url to download gvisor-containerd-shim GvisorContainerdShimURL = "https://github.com/google/gvisor-containerd-shim/releases/download/v0.0.1-rc.0/gvisor-containerd-shim-v0.0.1-rc.0.linux-amd64" // GvisorURL is the url to download gvisor - GvisorURL = "https://storage.googleapis.com/gvisor/releases/nightly/2018-11-08/runsc" + GvisorURL = "https://storage.googleapis.com/gvisor/releases/nightly/2018-12-07/runsc" )