From 28beac4edb0bfcd1e8dd47866ca017b39b1a5df2 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Wed, 9 Nov 2022 22:56:29 +0000 Subject: [PATCH] fix kubelet localStorageCapacityIsolation option for different k8s versions --- cmd/minikube/cmd/start_flags.go | 13 ++++++++-- .../bootstrapper/bsutil/ktmpl/v1beta3.go | 3 +++ pkg/minikube/bootstrapper/bsutil/kubeadm.go | 16 +++++++++++++ .../bootstrapper/bsutil/kubeadm_test.go | 24 +++++++++++++++++++ pkg/minikube/bootstrapper/bsutil/kubelet.go | 10 ++++++++ 5 files changed, 64 insertions(+), 2 deletions(-) diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index 4fd838fcf8f1..409d74d2b965 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -620,9 +620,18 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str } out.Styled(style.Notice, "Using {{.driver_name}} driver with root privileges", out.V{"driver_name": driver.FullName(drvName)}) } + // for btrfs: if k8s < v1.25.0-beta.0 set kubelet's LocalStorageCapacityIsolation feature gate flag to false, + // and if k8s >= v1.25.0-beta.0 (when it went ga and removed as feature gate), set kubelet's localStorageCapacityIsolation option (via kubeadm config) to false. + // ref: https://github.com/kubernetes/minikube/issues/14728#issue-1327885840 if si.StorageDriver == "btrfs" { - klog.Info("auto-setting LocalStorageCapacityIsolation to false because using btrfs storage driver") - cc.KubernetesConfig.FeatureGates = addFeatureGate(cc.KubernetesConfig.FeatureGates, "LocalStorageCapacityIsolation=false") + if semver.MustParse(strings.TrimPrefix(k8sVersion, version.VersionPrefix)).LT(semver.MustParse("1.25.0-beta.0")) { + klog.Info("auto-setting LocalStorageCapacityIsolation to false because using btrfs storage driver") + cc.KubernetesConfig.FeatureGates = addFeatureGate(cc.KubernetesConfig.FeatureGates, "LocalStorageCapacityIsolation=false") + } else if !cc.KubernetesConfig.ExtraOptions.Exists("kubelet.localStorageCapacityIsolation=false") { + if err := cc.KubernetesConfig.ExtraOptions.Set("kubelet.localStorageCapacityIsolation=false"); err != nil { + exit.Error(reason.InternalConfigSet, "failed to set extra option", err) + } + } } if runtime.GOOS == "linux" && si.DockerOS == "Docker Desktop" { out.WarningT("For an improved experience it's recommended to use Docker Engine instead of Docker Desktop.\nDocker Engine installation instructions: https://docs.docker.com/engine/install/#server") diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta3.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta3.go index f6df7e499b48..c2b73c635081 100644 --- a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta3.go +++ b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta3.go @@ -78,6 +78,9 @@ authentication: x509: clientCAFile: {{.ClientCAFile}} cgroupDriver: {{.CgroupDriver}} +{{- range $key, $val := .KubeletConfigOpts}} +{{$key}}: {{$val}} +{{- end}} clusterDomain: "{{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}}" # disable disk resource management by default imageGCHighThresholdPercent: 100 diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go index cbe857d0821d..112a395de694 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubeadm.go +++ b/pkg/minikube/bootstrapper/bsutil/kubeadm.go @@ -109,6 +109,7 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana ControlPlaneAddress string KubeProxyOptions map[string]string ResolvConfSearchRegression bool + KubeletConfigOpts map[string]string }{ CertDir: vmpath.GuestKubernetesCertsDir, ServiceCIDR: constants.DefaultServiceCIDR, @@ -133,6 +134,7 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana ControlPlaneAddress: constants.ControlPlaneAlias, KubeProxyOptions: createKubeProxyOptions(k8s.ExtraOptions), ResolvConfSearchRegression: HasResolvConfSearchRegression(k8s.KubernetesVersion), + KubeletConfigOpts: kubeletConfigOpts(k8s.ExtraOptions), } if k8s.ServiceCIDR != "" { @@ -215,3 +217,17 @@ func HasResolvConfSearchRegression(k8sVersion string) bool { } return versionSemver.EQ(semver.Version{Major: 1, Minor: 25}) } + +// kubeletConfigOpts extracts only those kubelet extra options allowed by kubeletConfigParams. +func kubeletConfigOpts(extraOpts config.ExtraOptionSlice) map[string]string { + args := map[string]string{} + for _, eo := range extraOpts { + if eo.Component != Kubelet { + continue + } + if config.ContainsParam(kubeletConfigParams, eo.Key) { + args[eo.Key] = eo.Value + } + } + return args +} diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go b/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go index baa9cebe81f6..8a9548b3b358 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go +++ b/pkg/minikube/bootstrapper/bsutil/kubeadm_test.go @@ -307,3 +307,27 @@ func TestEtcdExtraArgs(t *testing.T) { t.Errorf("machines mismatch (-want +got):\n%s", diff) } } + +func TestKubeletConfig(t *testing.T) { + expected := map[string]string{ + "localStorageCapacityIsolation": "false", + } + extraOpts := append(getExtraOpts(), []config.ExtraOption{ + { + Component: Kubelet, + Key: "unsupported-config-option", + Value: "any", + }, { + Component: Kubelet, + Key: "localStorageCapacityIsolation", + Value: "false", + }, { + Component: Kubelet, + Key: "kubelet.cgroups-per-qos", + Value: "false", + }}...) + actual := kubeletConfigOpts(extraOpts) + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("machines mismatch (-want +got):\n%s", diff) + } +} diff --git a/pkg/minikube/bootstrapper/bsutil/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kubelet.go index dc3a0099968e..b358d0630016 100644 --- a/pkg/minikube/bootstrapper/bsutil/kubelet.go +++ b/pkg/minikube/bootstrapper/bsutil/kubelet.go @@ -36,6 +36,11 @@ import ( "k8s.io/minikube/pkg/util" ) +// kubeletConfigParams are the only allowed kubelet parameters for kubeadmin config file and not to be used as kubelet flags +var kubeletConfigParams = []string{ + "localStorageCapacityIsolation", +} + func extraKubeletOpts(mc config.ClusterConfig, nc config.Node, r cruntime.Manager) (map[string]string, error) { k8s := mc.KubernetesConfig version, err := util.ParseKubernetesVersion(k8s.KubernetesVersion) @@ -95,6 +100,11 @@ func extraKubeletOpts(mc config.ClusterConfig, nc config.Node, r cruntime.Manage extraOpts["feature-gates"] = kubeletFeatureArgs } + // filter out non-flag extra kubelet config options + for _, opt := range kubeletConfigParams { + delete(extraOpts, opt) + } + return extraOpts, nil }