KEP-4603: Node specific kubelet config for maximum backoff down to 1 second (kubernetes#128374)

* Add feature gate, API, and conflict validation tests for enablecrashloopbackoffmax

Signed-off-by: Laura Lorenz <[email protected]>

* Handle when current base is longer than node max

Signed-off-by: Laura Lorenz <[email protected]>

* Update pkg/features/kube_features.go

Co-authored-by: Tsubasa Nagasawa <[email protected]>

* Fix indentation

Signed-off-by: Laura Lorenz <[email protected]>

* Follow convention for success test

Signed-off-by: Laura Lorenz <[email protected]>

* Normalize casing, and change field to Duration

Signed-off-by: Laura Lorenz <[email protected]>

* Fix json name and some other casing errors

Signed-off-by: Laura Lorenz <[email protected]>

* Another one I missed before

Signed-off-by: Laura Lorenz <[email protected]>

* Don't clobber global max function

Signed-off-by: Laura Lorenz <[email protected]>

* Change to flat value in defaults.go

Signed-off-by: Laura Lorenz <[email protected]>

* Streamline validation and defaults

Signed-off-by: Laura Lorenz <[email protected]>

* Fix typecheck

Signed-off-by: Laura Lorenz <[email protected]>

* Lint

Signed-off-by: Laura Lorenz <[email protected]>

* Tighten up validation for subsecond values

Signed-off-by: Laura Lorenz <[email protected]>

* Rename field from MaxBackOffPeriod to MaxContainerRestartPeriod

Signed-off-by: Laura Lorenz <[email protected]>

* A few missed references to renames

Signed-off-by: Laura Lorenz <[email protected]>

* Only compare flags in flags test

Signed-off-by: Laura Lorenz <[email protected]>

* Don't mess with SetDefault signature

Nobody messes with SetDefault signature

Signed-off-by: Laura Lorenz <[email protected]>

* Fix stale signature change, and update test data

Signed-off-by: Laura Lorenz <[email protected]>

* Inspect current feature gates at defaulting time

Signed-off-by: Laura Lorenz <[email protected]>

* Don't use the global feature gate for temp usage

Signed-off-by: Laura Lorenz <[email protected]>

* Expose default error, and some comments

Signed-off-by: Laura Lorenz <[email protected]>

* Hint fuzzer for less arbitrary values to FeatureGates

Signed-off-by: Laura Lorenz <[email protected]>

---------

Signed-off-by: Laura Lorenz <[email protected]>
Co-authored-by: Tsubasa Nagasawa <[email protected]>

Author: lauralorenz

cmd/kubelet/app/options/options_test.go

@@ -100,8 +100,8 @@ func TestRoundTrip(t *testing.T) {
 			}
 			continue
 		}
-		if !reflect.DeepEqual(modifiedFlags, outputFlags) {
-			t.Errorf("%s: flags did not round trip: %s", testCase.name, cmp.Diff(modifiedFlags, outputFlags))
+		if !reflect.DeepEqual(modifiedFlags.KubeletFlags, outputFlags.KubeletFlags) {
+			t.Errorf("%s: flags did not round trip: %s", testCase.name, cmp.Diff(modifiedFlags.KubeletFlags, outputFlags.KubeletFlags))
 			continue
 		}
 	}

cmd/kubelet/app/server.go

@@ -367,6 +367,7 @@ func mergeKubeletConfigurations(kubeletConfig *kubeletconfiginternal.KubeletConf
 	}
 	// apply defaulting after decoding
 	kubeletconfigv1beta1conversion.SetDefaults_KubeletConfiguration(versionedConfig)
+
 	// convert back to internal config
 	if err := kubeletconfigv1beta1conversion.Convert_v1beta1_KubeletConfiguration_To_config_KubeletConfiguration(versionedConfig, kubeletConfig, nil); err != nil {
 		return fmt.Errorf("failed to convert merged config to internal kubelet configuration: %w", err)

pkg/features/kube_features.go

@@ -232,6 +232,15 @@ const (
 	// status from DRA drivers.
 	DRAResourceClaimDeviceStatus featuregate.Feature = "DRAResourceClaimDeviceStatus"
 
+	// owner: @lauralorenz
+	// kep: https://kep.k8s.io/4603
+	// owner: @lauralorenz
+	// kep: https://kep.k8s.io/4603
+	//
+	// Enables support for configurable per-node backoff maximums for restarting
+	// containers (aka containers in CrashLoopBackOff)
+	KubeletCrashLoopBackOffMax featuregate.Feature = "KubeletCrashLoopBackOffMax"
+
 	// owner: @harche
 	// kep: http://kep.k8s.io/3386
 	//

pkg/features/versioned_kube_features.go

@@ -187,6 +187,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 		{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
 	},
 
+	KubeletCrashLoopBackOffMax: {
+		{Version: version.MustParse("1.32"), Default: false, PreRelease: featuregate.Alpha},
+	},
+
 	ElasticIndexedJob: {
 		{Version: version.MustParse("1.27"), Default: true, PreRelease: featuregate.Beta},
 		{Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // GA in 1.31, remove in 1.32

pkg/generated/openapi/zz_generated.openapi.go

@@ -121,6 +121,10 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 			obj.EnableSystemLogHandler = true
 			obj.MemoryThrottlingFactor = ptr.To(rand.Float64())
 			obj.LocalStorageCapacityIsolation = true
+			obj.FeatureGates = map[string]bool{
+				"AllAlpha": false,
+				"AllBeta":  true,
+			}
 		},
 	}
 }

pkg/kubelet/apis/config/fuzzer/fuzzer.go

@@ -302,5 +302,6 @@ var (
 		"Tracing.SamplingRatePerMillion",
 		"LocalStorageCapacityIsolation",
 		"FailCgroupV1",
+		"CrashLoopBackOff.MaxContainerRestartPeriod",
 	)
 )

pkg/kubelet/apis/config/helpers_test.go

@@ -25,6 +25,7 @@ cpuCFSQuota: true
 cpuCFSQuotaPeriod: 100ms
 cpuManagerPolicy: none
 cpuManagerReconcilePeriod: 10s
+crashLoopBackOff: {}
 enableControllerAttachDetach: true
 enableDebugFlagsHandler: true
 enableDebuggingHandlers: true

pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml

@@ -25,6 +25,7 @@ cpuCFSQuota: true
 cpuCFSQuotaPeriod: 100ms
 cpuManagerPolicy: none
 cpuManagerReconcilePeriod: 10s
+crashLoopBackOff: {}
 enableControllerAttachDetach: true
 enableDebugFlagsHandler: true
 enableDebuggingHandlers: true

pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml

@@ -506,6 +506,12 @@ type KubeletConfiguration struct {
 	// option is explicitly enabled.
 	// +optional
 	FailCgroupV1 bool
+
+	// CrashLoopBackOff contains config to modify node-level parameters for
+	// container restart behavior
+	// +featureGate=KubeletCrashLoopBackoffMax
+	// +optional
+	CrashLoopBackOff CrashLoopBackOffConfig
 }
 
 // KubeletAuthorizationMode denotes the authorization mode for the kubelet
@@ -684,3 +690,13 @@ type MemorySwapConfiguration struct {
 	// +optional
 	SwapBehavior string
 }
+
+// CrashLoopBackOffConfig is used for setting configuration for this kubelet's
+// container restart behavior
+type CrashLoopBackOffConfig struct {
+	// MaxContainerRestartPeriod is the maximum duration the backoff delay can accrue
+	// to for container restarts, minimum 1 second, maximum 300 seconds.
+	// +featureGate=KubeletCrashLoopBackOffMax
+	// +optional
+	MaxContainerRestartPeriod *metav1.Duration
+}