From d982c4bf74e14adc3219a0502b062ddb1cc55e83 Mon Sep 17 00:00:00 2001 From: Martynas Pumputis Date: Tue, 11 Dec 2018 14:30:46 +0100 Subject: [PATCH 1/2] Do not include the default CNI config by default Previously, minikube has been shipped with the default CNI config (/etc/cni/net.d/k8s.conf) in its rootfs. This complicated a lot when using a custom CNI plugin, as the default config was picked by kubelet before the custom CNI plugin has installed its own CNI config. So, the end result was that some Pods were attached to a network defined in the default config, and some got managed by the custom plugin. This commit introduces the flag "--enable-default-cni" to "minikube start" to trigger the provisioning of the default CNI config. Signed-off-by: Martynas Pumputis --- cmd/minikube/cmd/start.go | 3 ++ deploy/addons/gvisor/README.md | 2 +- .../rootfs-overlay/etc/cni/net.d/k8s.conf | 1 - .../rootfs-overlay/etc/rkt/net.d/k8s.conf | 19 -------- docs/alternative_runtimes.md | 5 +++ docs/contributors/minikube_iso.md | 1 + .../bootstrapper/kubeadm/default_cni.go | 43 +++++++++++++++++++ pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 9 ++++ pkg/minikube/config/types.go | 1 + pkg/minikube/constants/constants.go | 8 ++-- test/integration/util/util.go | 2 +- 11 files changed, 69 insertions(+), 25 deletions(-) delete mode 120000 deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/cni/net.d/k8s.conf delete mode 100644 deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/rkt/net.d/k8s.conf create mode 100644 pkg/minikube/bootstrapper/kubeadm/default_cni.go diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 683929d8e196..38e666793a0c 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -60,6 +60,7 @@ const ( containerRuntime = "container-runtime" criSocket = "cri-socket" networkPlugin = "network-plugin" + enableDefaultCNI = "enable-default-cni" hypervVirtualSwitch = "hyperv-virtual-switch" kvmNetwork = "kvm-network" keepContext = "keep-context" @@ -238,6 +239,7 @@ func runStart(cmd *cobra.Command, args []string) { ServiceCIDR: viper.GetString(serviceCIDR), ExtraOptions: extraOptions, ShouldLoadCachedImages: shouldCacheImages, + EnableDefaultCNI: viper.GetBool(enableDefaultCNI), } k8sBootstrapper, err := GetClusterBootstrapper(api, clusterBootstrapper) @@ -481,6 +483,7 @@ func init() { startCmd.Flags().String(criSocket, "", "The cri socket path to be used") startCmd.Flags().String(kubernetesVersion, constants.DefaultKubernetesVersion, "The kubernetes version that the minikube VM will use (ex: v1.2.3)") startCmd.Flags().String(networkPlugin, "", "The name of the network plugin") + startCmd.Flags().Bool(enableDefaultCNI, false, "Enable the default CNI plugin (/etc/cni/net.d/k8s.conf). Used in conjunction with \"--network-plugin=cni\"") startCmd.Flags().String(featureGates, "", "A set of key=value pairs that describe feature gates for alpha/experimental features.") startCmd.Flags().Bool(cacheImages, false, "If true, cache docker images for the current bootstrapper and load them into the machine.") startCmd.Flags().Var(&extraOptions, "extra-config", diff --git a/deploy/addons/gvisor/README.md b/deploy/addons/gvisor/README.md index 4bae2b2a7388..1a2a2a735b75 100644 --- a/deploy/addons/gvisor/README.md +++ b/deploy/addons/gvisor/README.md @@ -8,7 +8,7 @@ When starting minikube, specify the following flags, along with any additional d ```shell $ minikube start --container-runtime=containerd \ --docker-opt containerd=/var/run/containerd/containerd.sock \ - --network-plugin=cni + --network-plugin=cni --enable-default-cni ``` ### Enabling gVisor diff --git a/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/cni/net.d/k8s.conf b/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/cni/net.d/k8s.conf deleted file mode 120000 index 816796081d88..000000000000 --- a/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/cni/net.d/k8s.conf +++ /dev/null @@ -1 +0,0 @@ -../../../usr/libexec/kubernetes/kubelet-plugins/net/exec/k8s.conf \ No newline at end of file diff --git a/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/rkt/net.d/k8s.conf b/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/rkt/net.d/k8s.conf deleted file mode 100644 index 7b5404e53bf2..000000000000 --- a/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/rkt/net.d/k8s.conf +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "rkt.kubernetes.io", - "type": "bridge", - "bridge": "mybridge", - "mtu": 1460, - "addIf": "true", - "isGateway": true, - "ipMasq": true, - "ipam": { - "type": "host-local", - "subnet": "10.1.0.0/16", - "gateway": "10.1.0.1", - "routes": [ - { - "dst": "0.0.0.0/0" - } - ] - } -} diff --git a/docs/alternative_runtimes.md b/docs/alternative_runtimes.md index 6e87525e048b..dbf42f07823f 100644 --- a/docs/alternative_runtimes.md +++ b/docs/alternative_runtimes.md @@ -5,6 +5,7 @@ To use [rkt](https://github.com/coreos/rkt) as the container runtime run: ```shell $ minikube start \ --network-plugin=cni \ + --enable-default-cni \ --container-runtime=rkt ``` @@ -16,6 +17,7 @@ To use [CRI-O](https://github.com/kubernetes-incubator/cri-o) as the container r ```shell $ minikube start \ --network-plugin=cni \ + --enable-default-cni \ --container-runtime=cri-o ``` @@ -24,6 +26,7 @@ Or you can use the extended version: ```shell $ minikube start \ --network-plugin=cni \ + --enable-default-cni \ --cri-socket=/var/run/crio/crio.sock \ --extra-config=kubelet.container-runtime=remote \ --extra-config=kubelet.container-runtime-endpoint=unix:///var/run/crio/crio.sock \ @@ -37,6 +40,7 @@ To use [containerd](https://github.com/containerd/containerd) as the container r ```shell $ minikube start \ --network-plugin=cni \ + --enable-default-cni \ --container-runtime=containerd ``` @@ -45,6 +49,7 @@ Or you can use the extended version: ```shell $ minikube start \ --network-plugin=cni \ + --enable-default-cni \ --cri-socket=/run/containerd/containerd.sock \ --extra-config=kubelet.container-runtime=remote \ --extra-config=kubelet.container-runtime-endpoint=unix:///run/containerd/containerd.sock \ diff --git a/docs/contributors/minikube_iso.md b/docs/contributors/minikube_iso.md index 1dd5c967c2f0..6d63491b6850 100644 --- a/docs/contributors/minikube_iso.md +++ b/docs/contributors/minikube_iso.md @@ -40,6 +40,7 @@ The bootable ISO image will be available in `out/minikube.iso`. $ ./out/minikube start \ --container-runtime=rkt \ --network-plugin=cni \ + --enable-default-cni \ --iso-url=file://$GOPATH/src/k8s.io/minikube/out/minikube.iso ``` diff --git a/pkg/minikube/bootstrapper/kubeadm/default_cni.go b/pkg/minikube/bootstrapper/kubeadm/default_cni.go new file mode 100644 index 000000000000..dfd6cf61c515 --- /dev/null +++ b/pkg/minikube/bootstrapper/kubeadm/default_cni.go @@ -0,0 +1,43 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeadm + +// defaultCNIConfig is the CNI config which is provisioned when --enable-default-cni +// has been passed to `minikube start`. +// +// The config is being written to /etc/cni/net.d/k8s.conf and /etc/rkt/net.d/k8s.conf. +const defaultCNIConfig = ` +{ + "name": "rkt.kubernetes.io", + "type": "bridge", + "bridge": "mybridge", + "mtu": 1460, + "addIf": "true", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "10.1.0.0/16", + "gateway": "10.1.0.1", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ] + } +} +` diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 34cb39c4b93c..18315c473c4a 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -358,6 +358,15 @@ func (k *KubeadmBootstrapper) UpdateCluster(cfg config.KubernetesConfig) error { assets.NewMemoryAssetTarget([]byte(kubeadmCfg), constants.KubeadmConfigFile, "0640"), } + // Copy the default CNI config (k8s.conf), so that kubelet can successfully + // start a Pod in the case a user hasn't manually installed any CNI plugin + // and minikube was started with "--extra-config=kubelet.network-plugin=cni". + if cfg.EnableDefaultCNI { + files = append(files, + assets.NewMemoryAssetTarget([]byte(defaultCNIConfig), constants.DefaultCNIConfigPath, "0644"), + assets.NewMemoryAssetTarget([]byte(defaultCNIConfig), constants.DefaultRktNetConfigPath, "0644")) + } + var g errgroup.Group for _, bin := range []string{"kubelet", "kubeadm"} { bin := bin diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 36e17bf75e5f..657ed5ae305d 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -71,4 +71,5 @@ type KubernetesConfig struct { ExtraOptions util.ExtraOptionSlice ShouldLoadCachedImages bool + EnableDefaultCNI bool } diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 253773c6f1ff..5dd47d464f25 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -146,9 +146,11 @@ const AddonsPath = "/etc/kubernetes/addons" const FilesPath = "/files" const ( - KubeletServiceFile = "/lib/systemd/system/kubelet.service" - KubeletSystemdConfFile = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" - KubeadmConfigFile = "/var/lib/kubeadm.yaml" + KubeletServiceFile = "/lib/systemd/system/kubelet.service" + KubeletSystemdConfFile = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" + KubeadmConfigFile = "/var/lib/kubeadm.yaml" + DefaultCNIConfigPath = "/etc/cni/net.d/k8s.conf" + DefaultRktNetConfigPath = "/etc/rkt/net.d/k8s.conf" ) var Preflights = []string{ diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 7a4c11d77072..95d9166bdafe 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -174,7 +174,7 @@ func (m *MinikubeRunner) SSH(command string) (string, error) { func (m *MinikubeRunner) Start() { switch r := m.Runtime; r { case constants.ContainerdRuntime: - containerdFlags := "--container-runtime=containerd --network-plugin=cni --docker-opt containerd=/var/run/containerd/containerd.sock" + containerdFlags := "--container-runtime=containerd --network-plugin=cni --enable-default-cni --docker-opt containerd=/var/run/containerd/containerd.sock" m.RunCommand(fmt.Sprintf("start %s %s %s --alsologtostderr --v=5", m.StartArgs, m.Args, containerdFlags), true) default: m.RunCommand(fmt.Sprintf("start %s %s --alsologtostderr --v=5", m.StartArgs, m.Args), true) From 1252ad9a6d21af0b77916a3849e1a4a397fdabb7 Mon Sep 17 00:00:00 2001 From: Martynas Pumputis Date: Tue, 11 Dec 2018 14:43:53 +0100 Subject: [PATCH 2/2] Get rid of /usr/libexec/kubernetes/kubelet-plugins/net/exec/k8s.conf As per https://github.com/kubernetes/kubernetes/pull/32151, kubelet doesn't search for CNI configs in /usr/libexec/kubernetes/kubelet-plugins/net/exec/ by default anymore. Signed-off-by: Martynas Pumputis --- .../kubelet-plugins/net/exec/k8s.conf | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/usr/libexec/kubernetes/kubelet-plugins/net/exec/k8s.conf diff --git a/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/usr/libexec/kubernetes/kubelet-plugins/net/exec/k8s.conf b/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/usr/libexec/kubernetes/kubelet-plugins/net/exec/k8s.conf deleted file mode 100644 index 7b5404e53bf2..000000000000 --- a/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/usr/libexec/kubernetes/kubelet-plugins/net/exec/k8s.conf +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "rkt.kubernetes.io", - "type": "bridge", - "bridge": "mybridge", - "mtu": 1460, - "addIf": "true", - "isGateway": true, - "ipMasq": true, - "ipam": { - "type": "host-local", - "subnet": "10.1.0.0/16", - "gateway": "10.1.0.1", - "routes": [ - { - "dst": "0.0.0.0/0" - } - ] - } -}