diff --git a/cmd/csi-attacher/main.go b/cmd/csi-attacher/main.go index e2ab068cf..f1628c649 100644 --- a/cmd/csi-attacher/main.go +++ b/cmd/csi-attacher/main.go @@ -22,9 +22,11 @@ import ( "fmt" "net/http" "os" + "strings" "time" "k8s.io/apimachinery/pkg/runtime" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -49,6 +51,7 @@ import ( "github.com/kubernetes-csi/external-attacher/pkg/attacher" "github.com/kubernetes-csi/external-attacher/pkg/controller" "google.golang.org/grpc" + utilflag "k8s.io/component-base/cli/flag" ) const ( @@ -88,6 +91,8 @@ var ( kubeAPIBurst = flag.Int("kube-api-burst", 10, "Burst to use while communicating with the kubernetes apiserver. Defaults to 10.") maxGRPCLogLength = flag.Int("max-grpc-log-length", -1, "The maximum amount of characters logged for every grpc responses. Defaults to no limit") + + featureGates map[string]bool ) var ( @@ -95,6 +100,9 @@ var ( ) func main() { + flag.Var(utilflag.NewMapStringBool(&featureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features. "+ + "Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n")) + fg := featuregate.NewFeatureGate() logsapi.AddFeatureGates(fg) c := logsapi.NewLoggingConfiguration() @@ -108,6 +116,11 @@ func main() { klog.FlushAndExit(klog.ExitFlushTimeout, 1) } + if err := utilfeature.DefaultMutableFeatureGate.SetFromMap(featureGates); err != nil { + logger.Error(err, "failed to store flag gates", "featureGates", featureGates) + klog.FlushAndExit(klog.ExitFlushTimeout, 1) + } + if *showVersion { fmt.Println(os.Args[0], version) return diff --git a/go.mod b/go.mod index 049484207..2b5ea613f 100644 --- a/go.mod +++ b/go.mod @@ -13,10 +13,11 @@ require ( github.com/kubernetes-csi/csi-lib-utils v0.22.0 github.com/kubernetes-csi/csi-test/v5 v5.3.1 google.golang.org/grpc v1.72.1 - k8s.io/api v0.33.1 - k8s.io/apimachinery v0.33.1 - k8s.io/client-go v0.33.1 - k8s.io/component-base v0.33.1 + k8s.io/api v0.33.3 + k8s.io/apimachinery v0.33.3 + k8s.io/apiserver v0.33.3 + k8s.io/client-go v0.33.3 + k8s.io/component-base v0.33.3 k8s.io/csi-translation-lib v0.33.0 k8s.io/klog/v2 v2.130.1 ) diff --git a/go.sum b/go.sum index 6c7d10599..e6cd85d32 100644 --- a/go.sum +++ b/go.sum @@ -203,6 +203,8 @@ k8s.io/api v0.33.0 h1:yTgZVn1XEe6opVpP1FylmNrIFWuDqe2H0V8CT5gxfIU= k8s.io/api v0.33.0/go.mod h1:CTO61ECK/KU7haa3qq8sarQ0biLq2ju405IZAd9zsiM= k8s.io/apimachinery v0.33.0 h1:1a6kHrJxb2hs4t8EE5wuR/WxKDwGN1FKH3JvDtA0CIQ= k8s.io/apimachinery v0.33.0/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/apiserver v0.33.3 h1:Wv0hGc+QFdMJB4ZSiHrCgN3zL3QRatu56+rpccKC3J4= +k8s.io/apiserver v0.33.3/go.mod h1:05632ifFEe6TxwjdAIrwINHWE2hLwyADFk5mBsQa15E= k8s.io/client-go v0.33.0 h1:UASR0sAYVUzs2kYuKn/ZakZlcs2bEHaizrrHUZg0G98= k8s.io/client-go v0.33.0/go.mod h1:kGkd+l/gNGg8GYWAPr0xF1rRKvVWvzh9vmZAMXtaKOg= k8s.io/component-base v0.33.0 h1:Ot4PyJI+0JAD9covDhwLp9UNkUja209OzsJ4FzScBNk= diff --git a/pkg/controller/csi_handler.go b/pkg/controller/csi_handler.go index b96e86a52..1f6a0e10e 100644 --- a/pkg/controller/csi_handler.go +++ b/pkg/controller/csi_handler.go @@ -26,11 +26,14 @@ import ( "github.com/kubernetes-csi/csi-lib-utils/connection" "github.com/kubernetes-csi/external-attacher/pkg/attacher" + "github.com/kubernetes-csi/external-attacher/pkg/features" + "google.golang.org/grpc/status" v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/kubernetes" corelisters "k8s.io/client-go/listers/core/v1" storagelisters "k8s.io/client-go/listers/storage/v1" @@ -610,11 +613,21 @@ func (h *csiHandler) saveAttachError(ctx context.Context, va *storage.VolumeAtta logger := klog.FromContext(ctx) logger.V(4).Info("Saving attach error") clone := va.DeepCopy() - clone.Status.AttachError = &storage.VolumeError{ + + volumeError := &storage.VolumeError{ Message: err.Error(), Time: metav1.Now(), } + if utilfeature.DefaultFeatureGate.Enabled(features.MutableCSINodeAllocatableCount) { + if st, ok := status.FromError(err); ok { + errorCode := int32(st.Code()) + volumeError.ErrorCode = &errorCode + } + } + + clone.Status.AttachError = volumeError + var newVa *storage.VolumeAttachment if newVa, err = h.patchVA(ctx, va, clone, "status"); err != nil { return va, err diff --git a/pkg/controller/csi_handler_test.go b/pkg/controller/csi_handler_test.go index c0a3b9ef8..ecb1a5777 100644 --- a/pkg/controller/csi_handler_test.go +++ b/pkg/controller/csi_handler_test.go @@ -23,9 +23,12 @@ import ( "testing" "time" + "google.golang.org/grpc/codes" + "github.com/kubernetes-csi/csi-lib-utils/connection" "github.com/kubernetes-csi/external-attacher/pkg/attacher" - + "github.com/kubernetes-csi/external-attacher/pkg/features" + "google.golang.org/grpc/status" v1 "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -33,9 +36,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" core "k8s.io/client-go/testing" + featuregatetesting "k8s.io/component-base/featuregate/testing" csitranslator "k8s.io/csi-translation-lib" "k8s.io/klog/v2" _ "k8s.io/klog/v2/ktesting/init" @@ -269,6 +274,16 @@ func patch(original, new interface{}) []byte { return patch } +func vaWithAttachErrorAndCode(va *storage.VolumeAttachment, message string, code codes.Code) *storage.VolumeAttachment { + errorCode := int32(code) + va.Status.AttachError = &storage.VolumeError{ + Message: message, + Time: metav1.Time{}, + ErrorCode: &errorCode, + } + return va +} + func TestCSIHandler(t *testing.T) { vaGroupResourceVersion := schema.GroupVersionResource{ Group: storage.GroupName, @@ -1403,6 +1418,61 @@ func TestCSIHandler(t *testing.T) { runTests(t, csiHandlerFactory, tests) } +func TestVolumeAttachmentWithErrorCode(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MutableCSINodeAllocatableCount, true) + + vaGroupResourceVersion := schema.GroupVersionResource{ + Group: storage.GroupName, + Version: "v1", + Resource: "volumeattachments", + } + + var noMetadata map[string]string + var noAttrs map[string]string + var noSecrets map[string]string + var notDetached = false + var success error + var readWrite = false + + test := testCase{ + name: "CSI attach fails with gRPC error -> controller saves ErrorCode and retries", + initialObjects: []runtime.Object{pvWithFinalizer(), csiNode()}, + addedVA: va(false, "", nil), + expectedActions: []core.Action{ + core.NewPatchAction(vaGroupResourceVersion, metav1.NamespaceNone, testPVName+"-"+testNodeName, + types.MergePatchType, patch(va(false, "", nil), va(false, fin, ann))), + + // The CSI call fails, so the controller saves the error status. + core.NewPatchSubresourceAction(vaGroupResourceVersion, metav1.NamespaceNone, + testPVName+"-"+testNodeName, + types.MergePatchType, patch(va(false, fin, ann), + vaWithAttachErrorAndCode(va(false, fin, ann), "rpc error: code = ResourceExhausted desc = mock rpc error", codes.ResourceExhausted)), "status"), + + // On retry, the controller reads the original VA again and tries to re-apply the finalizer/annotation. + core.NewPatchAction(vaGroupResourceVersion, metav1.NamespaceNone, testPVName+"-"+testNodeName, + types.MergePatchType, patch( + vaWithAttachErrorAndCode(va(false, "", nil), "rpc error: code = ResourceExhausted desc = mock rpc error", codes.ResourceExhausted), + vaWithAttachErrorAndCode(va(false, fin, ann), "rpc error: code = ResourceExhausted desc = mock rpc error", codes.ResourceExhausted), + )), + + // The CSI call succeeds now, and the controller clears the error and marks the VA as attached. + core.NewPatchSubresourceAction(vaGroupResourceVersion, metav1.NamespaceNone, + testPVName+"-"+testNodeName, + types.MergePatchType, patch( + vaWithAttachErrorAndCode(va(false, fin, ann), "rpc error: code = ResourceExhausted desc = mock rpc error", codes.ResourceExhausted), + va(true /*attached*/, fin, ann), + ), + "status"), + }, + expectedCSICalls: []csiCall{ + {"attach", testVolumeHandle, testNodeID, noAttrs, noSecrets, readWrite, status.Error(codes.ResourceExhausted, "mock rpc error"), notDetached, noMetadata, 0}, + {"attach", testVolumeHandle, testNodeID, noAttrs, noSecrets, readWrite, success, notDetached, noMetadata, 0}, + }, + } + + runTests(t, csiHandlerFactory, []testCase{test}) +} + func TestCSIHandlerReconcileVA(t *testing.T) { nID := map[string]string{ vaNodeIDAnnotation: testNodeID, diff --git a/pkg/features/features.go b/pkg/features/features.go new file mode 100644 index 000000000..deb03a41a --- /dev/null +++ b/pkg/features/features.go @@ -0,0 +1,44 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package features + +import ( + "k8s.io/apiserver/pkg/util/feature" + "k8s.io/component-base/featuregate" +) + +const ( + // owner: @torredil @gnufied @msau42 + // kep: https://kep.k8s.io/4876 + // alpha: v1.33 + // beta: v1.34 + // + // Makes CSINode.Spec.Drivers[*].Allocatable.Count mutable, allowing CSI drivers to + // update the number of volumes that can be allocated on a node. Additionally, enables + // setting ErrorCode field in VolumeAttachment status. + MutableCSINodeAllocatableCount featuregate.Feature = "MutableCSINodeAllocatableCount" +) + +func init() { + feature.DefaultMutableFeatureGate.Add(defaultKubernetesFeatureGates) +} + +// defaultKubernetesFeatureGates consists of all known feature keys specific to external-attacher. +// To add a new feature, define a key for it above and add it here. +var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ + MutableCSINodeAllocatableCount: {Default: false, PreRelease: featuregate.Beta}, +} diff --git a/vendor/k8s.io/apiserver/LICENSE b/vendor/k8s.io/apiserver/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/vendor/k8s.io/apiserver/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/k8s.io/apiserver/pkg/util/feature/feature_gate.go b/vendor/k8s.io/apiserver/pkg/util/feature/feature_gate.go new file mode 100644 index 000000000..00a9e099b --- /dev/null +++ b/vendor/k8s.io/apiserver/pkg/util/feature/feature_gate.go @@ -0,0 +1,33 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package feature + +import ( + "k8s.io/component-base/featuregate" +) + +var ( + // DefaultMutableFeatureGate is a mutable version of DefaultFeatureGate. + // Only top-level commands/options setup and the k8s.io/component-base/featuregate/testing package should make use of this. + // Tests that need to modify feature gates for the duration of their test should use: + // featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features., ) + DefaultMutableFeatureGate featuregate.MutableVersionedFeatureGate = featuregate.NewFeatureGate() + + // DefaultFeatureGate is a shared global FeatureGate. + // Top-level commands/options setup that needs to modify this feature gate should use DefaultMutableFeatureGate. + DefaultFeatureGate featuregate.FeatureGate = DefaultMutableFeatureGate +) diff --git a/vendor/k8s.io/component-base/featuregate/testing/feature_gate.go b/vendor/k8s.io/component-base/featuregate/testing/feature_gate.go new file mode 100644 index 000000000..e9d452fb8 --- /dev/null +++ b/vendor/k8s.io/component-base/featuregate/testing/feature_gate.go @@ -0,0 +1,174 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "strings" + "sync" + + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/component-base/featuregate" +) + +var ( + overrideLock sync.Mutex + featureFlagOverride map[featuregate.Feature]string + emulationVersionOverride string + emulationVersionOverrideValue *version.Version +) + +func init() { + featureFlagOverride = map[featuregate.Feature]string{} +} + +// SetFeatureGateDuringTest sets the specified gate to the specified value for duration of the test. +// Fails when it detects second call to the same flag or is unable to set or restore feature flag. +// +// WARNING: Can leak set variable when called in test calling t.Parallel(), however second attempt to set the same feature flag will cause fatal. +// +// Example use: +// +// featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features., true) +func SetFeatureGateDuringTest(tb TB, gate featuregate.FeatureGate, f featuregate.Feature, value bool) { + tb.Helper() + detectParallelOverrideCleanup := detectParallelOverride(tb, f) + originalValue := gate.Enabled(f) + originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion() + originalExplicitlySet := gate.(featuregate.MutableVersionedFeatureGate).ExplicitlySet(f) + + // Specially handle AllAlpha and AllBeta + if f == "AllAlpha" || f == "AllBeta" { + // Iterate over individual gates so their individual values get restored + for k, v := range gate.(featuregate.MutableFeatureGate).GetAll() { + if k == "AllAlpha" || k == "AllBeta" { + continue + } + if (f == "AllAlpha" && v.PreRelease == featuregate.Alpha) || (f == "AllBeta" && v.PreRelease == featuregate.Beta) { + SetFeatureGateDuringTest(tb, gate, k, value) + } + } + } + + if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, value)); err != nil { + tb.Errorf("error setting %s=%v: %v", f, value, err) + } + + tb.Cleanup(func() { + tb.Helper() + detectParallelOverrideCleanup() + emuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion() + if !emuVer.EqualTo(originalEmuVer) { + tb.Fatalf("change of feature gate emulation version from %s to %s in the chain of SetFeatureGateDuringTest is not allowed\nuse SetFeatureGateEmulationVersionDuringTest to change emulation version in tests", + originalEmuVer.String(), emuVer.String()) + } + if originalExplicitlySet { + if err := gate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", f, originalValue)); err != nil { + tb.Errorf("error restoring %s=%v: %v", f, originalValue, err) + } + } else { + if err := gate.(featuregate.MutableVersionedFeatureGate).ResetFeatureValueToDefault(f); err != nil { + tb.Errorf("error restoring %s=%v: %v", f, originalValue, err) + } + } + }) +} + +// SetFeatureGateEmulationVersionDuringTest sets the specified gate to the specified emulation version for duration of the test. +// Fails when it detects second call to set a different emulation version or is unable to set or restore emulation version. +// WARNING: Can leak set variable when called in test calling t.Parallel(), however second attempt to set a different emulation version will cause fatal. +// Example use: + +// featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.31")) +func SetFeatureGateEmulationVersionDuringTest(tb TB, gate featuregate.FeatureGate, ver *version.Version) { + tb.Helper() + detectParallelOverrideCleanup := detectParallelOverrideEmulationVersion(tb, ver) + originalEmuVer := gate.(featuregate.MutableVersionedFeatureGate).EmulationVersion() + if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(ver); err != nil { + tb.Fatalf("failed to set emulation version to %s during test: %v", ver.String(), err) + } + tb.Cleanup(func() { + tb.Helper() + detectParallelOverrideCleanup() + if err := gate.(featuregate.MutableVersionedFeatureGate).SetEmulationVersion(originalEmuVer); err != nil { + tb.Fatalf("failed to restore emulation version to %s during test", originalEmuVer.String()) + } + }) +} + +func detectParallelOverride(tb TB, f featuregate.Feature) func() { + tb.Helper() + overrideLock.Lock() + defer overrideLock.Unlock() + beforeOverrideTestName := featureFlagOverride[f] + if beforeOverrideTestName != "" && !sameTestOrSubtest(tb, beforeOverrideTestName) { + tb.Fatalf("Detected parallel setting of a feature gate by both %q and %q", beforeOverrideTestName, tb.Name()) + } + featureFlagOverride[f] = tb.Name() + + return func() { + tb.Helper() + overrideLock.Lock() + defer overrideLock.Unlock() + if afterOverrideTestName := featureFlagOverride[f]; afterOverrideTestName != tb.Name() { + tb.Fatalf("Detected parallel setting of a feature gate between both %q and %q", afterOverrideTestName, tb.Name()) + } + featureFlagOverride[f] = beforeOverrideTestName + } +} + +func detectParallelOverrideEmulationVersion(tb TB, ver *version.Version) func() { + tb.Helper() + overrideLock.Lock() + defer overrideLock.Unlock() + beforeOverrideTestName := emulationVersionOverride + beforeOverrideValue := emulationVersionOverrideValue + if ver.EqualTo(beforeOverrideValue) { + return func() {} + } + if beforeOverrideTestName != "" && !sameTestOrSubtest(tb, beforeOverrideTestName) { + tb.Fatalf("Detected parallel setting of a feature gate emulation version by both %q and %q", beforeOverrideTestName, tb.Name()) + } + emulationVersionOverride = tb.Name() + emulationVersionOverrideValue = ver + + return func() { + tb.Helper() + overrideLock.Lock() + defer overrideLock.Unlock() + if afterOverrideTestName := emulationVersionOverride; afterOverrideTestName != tb.Name() { + tb.Fatalf("Detected parallel setting of a feature gate emulation version between both %q and %q", afterOverrideTestName, tb.Name()) + } + emulationVersionOverride = beforeOverrideTestName + emulationVersionOverrideValue = beforeOverrideValue + } +} + +func sameTestOrSubtest(tb TB, testName string) bool { + // Assumes that "/" is not used in test names. + return tb.Name() == testName || strings.HasPrefix(tb.Name(), testName+"/") +} + +type TB interface { + Cleanup(func()) + Error(args ...any) + Errorf(format string, args ...any) + Fatal(args ...any) + Fatalf(format string, args ...any) + Helper() + Name() string +} diff --git a/vendor/modules.txt b/vendor/modules.txt index a23b5f8bb..a7b9ee30f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -341,7 +341,7 @@ gopkg.in/inf.v0 # gopkg.in/yaml.v3 v3.0.1 ## explicit gopkg.in/yaml.v3 -# k8s.io/api v0.33.1 => k8s.io/api v0.33.0 +# k8s.io/api v0.33.3 => k8s.io/api v0.33.0 ## explicit; go 1.24.0 k8s.io/api/admissionregistration/v1 k8s.io/api/admissionregistration/v1alpha1 @@ -401,7 +401,7 @@ k8s.io/api/storage/v1 k8s.io/api/storage/v1alpha1 k8s.io/api/storage/v1beta1 k8s.io/api/storagemigration/v1alpha1 -# k8s.io/apimachinery v0.33.1 => k8s.io/apimachinery v0.33.0 +# k8s.io/apimachinery v0.33.3 => k8s.io/apimachinery v0.33.0 ## explicit; go 1.24.0 k8s.io/apimachinery/pkg/api/equality k8s.io/apimachinery/pkg/api/errors @@ -457,7 +457,10 @@ k8s.io/apimachinery/pkg/version k8s.io/apimachinery/pkg/watch k8s.io/apimachinery/third_party/forked/golang/json k8s.io/apimachinery/third_party/forked/golang/reflect -# k8s.io/client-go v0.33.1 => k8s.io/client-go v0.33.0 +# k8s.io/apiserver v0.33.3 +## explicit; go 1.24.0 +k8s.io/apiserver/pkg/util/feature +# k8s.io/client-go v0.33.3 => k8s.io/client-go v0.33.0 ## explicit; go 1.24.0 k8s.io/client-go/applyconfigurations k8s.io/client-go/applyconfigurations/admissionregistration/v1 @@ -789,10 +792,11 @@ k8s.io/client-go/util/homedir k8s.io/client-go/util/keyutil k8s.io/client-go/util/watchlist k8s.io/client-go/util/workqueue -# k8s.io/component-base v0.33.1 => k8s.io/component-base v0.33.0 +# k8s.io/component-base v0.33.3 => k8s.io/component-base v0.33.0 ## explicit; go 1.24.0 k8s.io/component-base/cli/flag k8s.io/component-base/featuregate +k8s.io/component-base/featuregate/testing k8s.io/component-base/logs k8s.io/component-base/logs/api/v1 k8s.io/component-base/logs/internal/setverbositylevel