From c0c93b3c2d8afc8cad32d893469ee4e6e2fdd5dd Mon Sep 17 00:00:00 2001 From: Roman Bednar Date: Wed, 17 May 2023 14:12:09 +0200 Subject: [PATCH 1/6] make azure_client_secret optional Due to addition of Azure workload identity feature, ccoctl will no longer provide azure_client_secret in all configurations. If the feature is enabled no client secret will be set. --- assets/controller.yaml | 1 + assets/node.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/assets/controller.yaml b/assets/controller.yaml index fe5a1ef8..2f29f0c6 100644 --- a/assets/controller.yaml +++ b/assets/controller.yaml @@ -64,6 +64,7 @@ spec: secretKeyRef: name: azure-disk-credentials key: azure_client_secret + optional: true volumeMounts: - name: host-cloud-config mountPath: /etc/cloud-config diff --git a/assets/node.yaml b/assets/node.yaml index 6f6801be..f6022b10 100644 --- a/assets/node.yaml +++ b/assets/node.yaml @@ -50,6 +50,7 @@ spec: secretKeyRef: name: azure-disk-credentials key: azure_client_secret + optional: true volumeMounts: - name: host-cloud-config mountPath: /etc/cloud-config From dbb28388dcfe8f77c4c9023e9e4358a8603db8e0 Mon Sep 17 00:00:00 2001 From: Roman Bednar Date: Wed, 17 May 2023 14:12:52 +0200 Subject: [PATCH 2/6] load optional tenant id and token path from azure credentials secret If Azure workload identity is enabled two new secrets will be provided by ccoctl for tenant id and path to federated token file. Those have to be optional because if the feature is disabled those values will not be set. --- assets/controller.yaml | 12 ++++++++++++ assets/node.yaml | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/assets/controller.yaml b/assets/controller.yaml index 2f29f0c6..409d37f4 100644 --- a/assets/controller.yaml +++ b/assets/controller.yaml @@ -65,6 +65,18 @@ spec: name: azure-disk-credentials key: azure_client_secret optional: true + - name: AZURE_TENANT_ID + valueFrom: + secretKeyRef: + name: azure-disk-credentials + key: azure_tenant_id + optional: true + - name: AZURE_FEDERATED_TOKEN_FILE + valueFrom: + secretKeyRef: + name: azure-disk-credentials + key: azure_federated_token_file + optional: true volumeMounts: - name: host-cloud-config mountPath: /etc/cloud-config diff --git a/assets/node.yaml b/assets/node.yaml index f6022b10..ebc1c594 100644 --- a/assets/node.yaml +++ b/assets/node.yaml @@ -51,6 +51,18 @@ spec: name: azure-disk-credentials key: azure_client_secret optional: true + - name: AZURE_TENANT_ID + valueFrom: + secretKeyRef: + name: azure-disk-credentials + key: azure_tenant_id + optional: true + - name: AZURE_FEDERATED_TOKEN_FILE + valueFrom: + secretKeyRef: + name: azure-disk-credentials + key: azure_federated_token_file + optional: true volumeMounts: - name: host-cloud-config mountPath: /etc/cloud-config From a76c32cffbfdcc26657ecac61f81dc2b45f17d60 Mon Sep 17 00:00:00 2001 From: Roman Bednar Date: Tue, 23 May 2023 13:30:46 +0200 Subject: [PATCH 3/6] make asset functions more dynamic We need to make asset functions more dynamic. Currently we replace only one value but in next patches we will need to also set arguments for azure credential injector. This argument will have to change based on feature gate state. --- pkg/operator/starter.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/pkg/operator/starter.go b/pkg/operator/starter.go index bd5bb177..51002ccf 100644 --- a/pkg/operator/starter.go +++ b/pkg/operator/starter.go @@ -101,6 +101,9 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller go azureStackConfigSyncer.Run(ctx, 1) } + deploymentAsset := &assetWithReplacement{} + deploymentAsset.Replace("${CLUSTER_CLOUD_CONTROLLER_MANAGER_OPERATOR_IMAGE}", os.Getenv(ccmOperatorImageEnvName)) + csiControllerSet := csicontrollerset.NewCSIControllerSet( operatorClient, controllerConfig.EventRecorder, @@ -160,7 +163,7 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller configInformers, ).WithCSIDriverControllerService( "AzureDiskDriverControllerServiceController", - assetWithImageReplaced(), + deploymentAsset.GetAssetFunc(), "controller.yaml", kubeClient, kubeInformersForNamespaces.InformersFor(defaultNamespace), @@ -181,7 +184,7 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller csidrivercontrollerservicecontroller.WithSecretHashAnnotationHook(defaultNamespace, secretName, secretInformer), ).WithCSIDriverNodeService( "AzureDiskDriverNodeServiceController", - assetWithImageReplaced(), + deploymentAsset.GetAssetFunc(), "node.yaml", kubeClient, kubeInformersForNamespaces.InformersFor(defaultNamespace), @@ -227,14 +230,22 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller return fmt.Errorf("stopped") } -func assetWithImageReplaced() func(name string) ([]byte, error) { +type assetWithReplacement []string + +func (r *assetWithReplacement) Replace(old, new string) { + *r = append(*r, old, new) +} + +func (r *assetWithReplacement) GetAssetFunc() func(name string) ([]byte, error) { return func(name string) ([]byte, error) { assetBytes, err := assets.ReadFile(name) if err != nil { return assetBytes, err } - asset := string(assetBytes) - asset = strings.ReplaceAll(asset, "${CLUSTER_CLOUD_CONTROLLER_MANAGER_OPERATOR_IMAGE}", os.Getenv(ccmOperatorImageEnvName)) + + replacer := strings.NewReplacer(*r...) + asset := replacer.Replace(string(assetBytes)) + return []byte(asset), nil } } From ff5eb9b3341470202ff474adf0833db21dcd3acd Mon Sep 17 00:00:00 2001 From: Roman Bednar Date: Thu, 1 Jun 2023 12:17:24 +0200 Subject: [PATCH 4/6] enable Azure workload identity based on featuregate Operator needs to get a featuregate state and enable Azure workload identity feature if the featuregate is set. We do this by adding a placeholder string to --enable-azure-workload-identity injector flag and replacing it's string value to "true" if the feature should be enabled, and "false" otherwise. --- assets/controller.yaml | 1 + assets/node.yaml | 1 + pkg/operator/starter.go | 41 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/assets/controller.yaml b/assets/controller.yaml index 409d37f4..d7be0eed 100644 --- a/assets/controller.yaml +++ b/assets/controller.yaml @@ -53,6 +53,7 @@ spec: - --output-file-path=/etc/merged-cloud-config/cloud.conf # Force disable node's managed identity, azure-disk-credentials Secret should be used. - --disable-identity-extension-auth + - --enable-azure-workload-identity=${ENABLE_AZURE_WORKLOAD_IDENTITY} env: - name: AZURE_CLIENT_ID valueFrom: diff --git a/assets/node.yaml b/assets/node.yaml index ebc1c594..797cd17c 100644 --- a/assets/node.yaml +++ b/assets/node.yaml @@ -39,6 +39,7 @@ spec: - --output-file-path=/etc/merged-cloud-config/cloud.conf # Force disable node's managed identity, azure-disk-credentials Secret should be used. - --disable-identity-extension-auth + - --enable-azure-workload-identity=${ENABLE_AZURE_WORKLOAD_IDENTITY} env: - name: AZURE_CLIENT_ID valueFrom: diff --git a/pkg/operator/starter.go b/pkg/operator/starter.go index 51002ccf..c1f6aaf0 100644 --- a/pkg/operator/starter.go +++ b/pkg/operator/starter.go @@ -3,6 +3,7 @@ package operator import ( "context" "fmt" + configv1 "github.com/openshift/api/config/v1" "os" "strings" "time" @@ -29,6 +30,8 @@ import ( "github.com/openshift/library-go/pkg/operator/csi/csidrivernodeservicecontroller" goc "github.com/openshift/library-go/pkg/operator/genericoperatorclient" "github.com/openshift/library-go/pkg/operator/v1helpers" + + "github.com/openshift/library-go/pkg/operator/configobserver/featuregates" ) const ( @@ -40,8 +43,9 @@ const ( trustedCAConfigMap = "azure-disk-csi-driver-trusted-ca-bundle" resync = 20 * time.Minute - ccmOperatorImageEnvName = "CLUSTER_CLOUD_CONTROLLER_MANAGER_OPERATOR_IMAGE" - diskEncryptionSetID = "diskEncryptionSetID" + ccmOperatorImageEnvName = "CLUSTER_CLOUD_CONTROLLER_MANAGER_OPERATOR_IMAGE" + diskEncryptionSetID = "diskEncryptionSetID" + operatorImageVersionEnvVarName = "OPERATOR_IMAGE_VERSION" ) func RunOperator(ctx context.Context, controllerConfig *controllercmd.ControllerContext) error { @@ -101,7 +105,40 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller go azureStackConfigSyncer.Run(ctx, 1) } + desiredVersion := os.Getenv(operatorImageVersionEnvVarName) + missingVersion := "0.0.1-snapshot" + + featureGateAccessor := featuregates.NewFeatureGateAccess( + desiredVersion, + missingVersion, + configInformers.Config().V1().ClusterVersions(), + configInformers.Config().V1().FeatureGates(), + controllerConfig.EventRecorder, + ) + go featureGateAccessor.Run(ctx) + go configInformers.Start(ctx.Done()) + + select { + case <-featureGateAccessor.InitialFeatureGatesObserved(): + featureGates, _ := featureGateAccessor.CurrentFeatureGates() + klog.Info("FeatureGates initialized", "knownFeatures", featureGates.KnownFeatures()) + case <-time.After(1 * time.Minute): + klog.Error(nil, "timed out waiting for FeatureGate detection") + return fmt.Errorf("timed out waiting for FeatureGate detection") + } + + featureGates, err := featureGateAccessor.CurrentFeatureGates() + if err != nil { + return err + } + deploymentAsset := &assetWithReplacement{} + if featureGates.Enabled(configv1.FeatureGateAzureWorkloadIdentity) { + deploymentAsset.Replace("${ENABLE_AZURE_WORKLOAD_IDENTITY}", "true") + } else { + deploymentAsset.Replace("${ENABLE_AZURE_WORKLOAD_IDENTITY}", "false") + } + deploymentAsset.Replace("${CLUSTER_CLOUD_CONTROLLER_MANAGER_OPERATOR_IMAGE}", os.Getenv(ccmOperatorImageEnvName)) csiControllerSet := csicontrollerset.NewCSIControllerSet( From f5ee4f9fee3d5408f9f9a4b6010babd13400a2ee Mon Sep 17 00:00:00 2001 From: Jeremiah Stuever Date: Thu, 20 Apr 2023 13:03:26 -0700 Subject: [PATCH 5/6] Mount serviceaccount token into csi-driver container --- assets/controller.yaml | 9 +++++++++ pkg/azurestackhub/azure_stack_hub_test.go | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/assets/controller.yaml b/assets/controller.yaml index d7be0eed..dd688962 100644 --- a/assets/controller.yaml +++ b/assets/controller.yaml @@ -123,6 +123,9 @@ spec: - name: msi mountPath: /var/lib/waagent/ManagedIdentity-Settings readOnly: true + - name: bound-sa-token + mountPath: /var/run/secrets/openshift/serviceaccount + readOnly: true resources: requests: memory: 50Mi @@ -366,3 +369,9 @@ spec: secretName: azure-disk-csi-driver-controller-metrics-serving-cert - name: merged-cloud-config emptydir: + - name: bound-sa-token + projected: + sources: + - serviceAccountToken: + path: token + audience: openshift diff --git a/pkg/azurestackhub/azure_stack_hub_test.go b/pkg/azurestackhub/azure_stack_hub_test.go index 6ff06078..1142b1bd 100644 --- a/pkg/azurestackhub/azure_stack_hub_test.go +++ b/pkg/azurestackhub/azure_stack_hub_test.go @@ -41,7 +41,7 @@ func TestInjectPodSpecHappyPath(t *testing.T) { assert.Nil(t, yaml.Unmarshal(file, dep)) injectEnvAndMounts(&dep.Spec.Template.Spec) - assert.Len(t, dep.Spec.Template.Spec.Volumes, 6) + assert.Len(t, dep.Spec.Template.Spec.Volumes, 7) foundCfgVolume := false for _, v := range dep.Spec.Template.Spec.Volumes { if v.Name == azureCfgName { @@ -59,7 +59,7 @@ func TestInjectPodSpecHappyPath(t *testing.T) { } } assert.NotNil(t, csiDriver, "no csi-driver container found") - assert.Len(t, csiDriver.VolumeMounts, 4) + assert.Len(t, csiDriver.VolumeMounts, 5) foundCfgVolumeMount := false for _, v := range csiDriver.VolumeMounts { if v.Name == azureCfgName { From ba6fd96a211f35c7d4e15264f4575d00e5d9987a Mon Sep 17 00:00:00 2001 From: Roman Bednar Date: Tue, 20 Jun 2023 09:14:46 +0200 Subject: [PATCH 6/6] tidy && vendor --- .../featuregates/featuregate.go | 47 +++ .../hardcoded_featuregate_reader.go | 78 +++++ .../featuregates/observe_featuregates.go | 118 +++++++ .../featuregates/simple_featuregate_reader.go | 318 ++++++++++++++++++ vendor/modules.txt | 1 + 5 files changed, 562 insertions(+) create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/featuregate.go create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/hardcoded_featuregate_reader.go create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/observe_featuregates.go create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/simple_featuregate_reader.go diff --git a/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/featuregate.go b/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/featuregate.go new file mode 100644 index 00000000..5ff0f3af --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/featuregate.go @@ -0,0 +1,47 @@ +package featuregates + +import ( + "fmt" + configv1 "github.com/openshift/api/config/v1" + "k8s.io/apimachinery/pkg/util/sets" +) + +// FeatureGate indicates whether a given feature is enabled or not +// This interface is heavily influenced by k8s.io/component-base, but not exactly compatible. +type FeatureGate interface { + // Enabled returns true if the key is enabled. + Enabled(key configv1.FeatureGateName) bool + // KnownFeatures returns a slice of strings describing the FeatureGate's known features. + KnownFeatures() []configv1.FeatureGateName +} + +type featureGate struct { + enabled sets.Set[configv1.FeatureGateName] + disabled sets.Set[configv1.FeatureGateName] +} + +func NewFeatureGate(enabled, disabled []configv1.FeatureGateName) FeatureGate { + return &featureGate{ + enabled: sets.New[configv1.FeatureGateName](enabled...), + disabled: sets.New[configv1.FeatureGateName](disabled...), + } +} + +func (f *featureGate) Enabled(key configv1.FeatureGateName) bool { + if f.enabled.Has(key) { + return true + } + if f.disabled.Has(key) { + return false + } + + panic(fmt.Errorf("feature %q is not registered in FeatureGates %v", key, f.KnownFeatures())) +} + +func (f *featureGate) KnownFeatures() []configv1.FeatureGateName { + allKnown := sets.NewString() + allKnown.Insert(FeatureGateNamesToStrings(f.enabled.UnsortedList())...) + allKnown.Insert(FeatureGateNamesToStrings(f.disabled.UnsortedList())...) + + return StringsToFeatureGateNames(allKnown.List()) +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/hardcoded_featuregate_reader.go b/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/hardcoded_featuregate_reader.go new file mode 100644 index 00000000..58ae7176 --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/hardcoded_featuregate_reader.go @@ -0,0 +1,78 @@ +package featuregates + +import ( + "context" + "fmt" + + configv1 "github.com/openshift/api/config/v1" +) + +type hardcodedFeatureGateAccess struct { + enabled []configv1.FeatureGateName + disabled []configv1.FeatureGateName + readErr error + + initialFeatureGatesObserved chan struct{} +} + +// NewHardcodedFeatureGateAccess returns a FeatureGateAccess that is always initialized and always +// returns the provided feature gates. +func NewHardcodedFeatureGateAccess(enabled, disabled []configv1.FeatureGateName) FeatureGateAccess { + initialFeatureGatesObserved := make(chan struct{}) + close(initialFeatureGatesObserved) + c := &hardcodedFeatureGateAccess{ + enabled: enabled, + disabled: disabled, + initialFeatureGatesObserved: initialFeatureGatesObserved, + } + + return c +} + +// NewHardcodedFeatureGateAccessForTesting returns a FeatureGateAccess that returns stub responses +// using caller-supplied values. +func NewHardcodedFeatureGateAccessForTesting(enabled, disabled []configv1.FeatureGateName, initialFeatureGatesObserved chan struct{}, readErr error) FeatureGateAccess { + return &hardcodedFeatureGateAccess{ + enabled: enabled, + disabled: disabled, + initialFeatureGatesObserved: initialFeatureGatesObserved, + readErr: readErr, + } +} + +func (c *hardcodedFeatureGateAccess) SetChangeHandler(featureGateChangeHandlerFn FeatureGateChangeHandlerFunc) { + // ignore +} + +func (c *hardcodedFeatureGateAccess) Run(ctx context.Context) { + // ignore +} + +func (c *hardcodedFeatureGateAccess) InitialFeatureGatesObserved() <-chan struct{} { + return c.initialFeatureGatesObserved +} + +func (c *hardcodedFeatureGateAccess) AreInitialFeatureGatesObserved() bool { + select { + case <-c.InitialFeatureGatesObserved(): + return true + default: + return false + } +} + +func (c *hardcodedFeatureGateAccess) CurrentFeatureGates() (FeatureGate, error) { + return NewFeatureGate(c.enabled, c.disabled), c.readErr +} + +// NewHardcodedFeatureGateAccessFromFeatureGate returns a FeatureGateAccess that is static and initialised from +// a populated FeatureGate status. +// If the desired version is missing, this will return an error. +func NewHardcodedFeatureGateAccessFromFeatureGate(featureGate *configv1.FeatureGate, desiredVersion string) (FeatureGateAccess, error) { + features, err := featuresFromFeatureGate(featureGate, desiredVersion) + if err != nil { + return nil, fmt.Errorf("unable to determine features: %w", err) + } + + return NewHardcodedFeatureGateAccess(features.Enabled, features.Disabled), nil +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/observe_featuregates.go b/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/observe_featuregates.go new file mode 100644 index 00000000..0f2cb85f --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/observe_featuregates.go @@ -0,0 +1,118 @@ +package featuregates + +import ( + "fmt" + "reflect" + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/sets" + + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/library-go/pkg/operator/configobserver" + "github.com/openshift/library-go/pkg/operator/events" +) + +// NewObserveFeatureFlagsFunc produces a configobserver for feature gates. If non-nil, the featureWhitelist filters +// feature gates to a known subset (instead of everything). The featureBlacklist will stop certain features from making +// it through the list. The featureBlacklist should be empty, but for a brief time, some featuregates may need to skipped. +// @smarterclayton will live forever in shame for being the first to require this for "IPv6DualStack". +func NewObserveFeatureFlagsFunc(featureWhitelist sets.Set[configv1.FeatureGateName], featureBlacklist sets.Set[configv1.FeatureGateName], configPath []string, featureGateAccess FeatureGateAccess) configobserver.ObserveConfigFunc { + return (&featureFlags{ + allowAll: len(featureWhitelist) == 0, + featureWhitelist: featureWhitelist, + featureBlacklist: featureBlacklist, + configPath: configPath, + featureGateAccess: featureGateAccess, + }).ObserveFeatureFlags +} + +type featureFlags struct { + allowAll bool + featureWhitelist sets.Set[configv1.FeatureGateName] + // we add a forceDisableFeature list because we've now had bad featuregates break individual operators. Awesome. + featureBlacklist sets.Set[configv1.FeatureGateName] + configPath []string + featureGateAccess FeatureGateAccess +} + +// ObserveFeatureFlags fills in --feature-flags for the kube-apiserver +func (f *featureFlags) ObserveFeatureFlags(genericListers configobserver.Listers, recorder events.Recorder, existingConfig map[string]interface{}) (map[string]interface{}, []error) { + prunedExistingConfig := configobserver.Pruned(existingConfig, f.configPath) + + errs := []error{} + + if !f.featureGateAccess.AreInitialFeatureGatesObserved() { + // if we haven't observed featuregates yet, return the existing + return prunedExistingConfig, nil + } + + featureGates, err := f.featureGateAccess.CurrentFeatureGates() + if err != nil { + return prunedExistingConfig, append(errs, err) + } + observedConfig := map[string]interface{}{} + newConfigValue := f.getWhitelistedFeatureNames(featureGates) + + currentConfigValue, _, err := unstructured.NestedStringSlice(existingConfig, f.configPath...) + if err != nil { + errs = append(errs, err) + // keep going on read error from existing config + } + if !reflect.DeepEqual(currentConfigValue, newConfigValue) { + recorder.Eventf("ObserveFeatureFlagsUpdated", "Updated %v to %s", strings.Join(f.configPath, "."), strings.Join(newConfigValue, ",")) + } + + if err := unstructured.SetNestedStringSlice(observedConfig, newConfigValue, f.configPath...); err != nil { + recorder.Warningf("ObserveFeatureFlags", "Failed setting %v: %v", strings.Join(f.configPath, "."), err) + return prunedExistingConfig, append(errs, err) + } + + return configobserver.Pruned(observedConfig, f.configPath), errs +} + +func (f *featureFlags) getWhitelistedFeatureNames(featureGates FeatureGate) []string { + newConfigValue := []string{} + formatEnabledFunc := func(fs configv1.FeatureGateName) string { + return fmt.Sprintf("%v=true", fs) + } + formatDisabledFunc := func(fs configv1.FeatureGateName) string { + return fmt.Sprintf("%v=false", fs) + } + + for _, knownFeatureGate := range featureGates.KnownFeatures() { + if f.featureBlacklist.Has(knownFeatureGate) { + continue + } + // only add whitelisted feature flags + if !f.allowAll && !f.featureWhitelist.Has(knownFeatureGate) { + continue + } + + if featureGates.Enabled(knownFeatureGate) { + newConfigValue = append(newConfigValue, formatEnabledFunc(knownFeatureGate)) + } else { + newConfigValue = append(newConfigValue, formatDisabledFunc(knownFeatureGate)) + } + } + + return newConfigValue +} + +func StringsToFeatureGateNames(in []string) []configv1.FeatureGateName { + out := []configv1.FeatureGateName{} + for _, curr := range in { + out = append(out, configv1.FeatureGateName(curr)) + } + + return out +} + +func FeatureGateNamesToStrings(in []configv1.FeatureGateName) []string { + out := []string{} + for _, curr := range in { + out = append(out, string(curr)) + } + + return out +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/simple_featuregate_reader.go b/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/simple_featuregate_reader.go new file mode 100644 index 00000000..4b2caccd --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/configobserver/featuregates/simple_featuregate_reader.go @@ -0,0 +1,318 @@ +package featuregates + +import ( + "context" + "fmt" + "os" + "reflect" + "sync" + "time" + + configv1 "github.com/openshift/api/config/v1" + + v1 "github.com/openshift/client-go/config/informers/externalversions/config/v1" + configlistersv1 "github.com/openshift/client-go/config/listers/config/v1" + "github.com/openshift/library-go/pkg/operator/events" + apierrors "k8s.io/apimachinery/pkg/api/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" +) + +type FeatureGateChangeHandlerFunc func(featureChange FeatureChange) + +// FeatureGateAccess is used to get a list of enabled and disabled featuregates. +// Create a new instance using NewFeatureGateAccess. +// To create one for unit testing, use NewHardcodedFeatureGateAccess. +type FeatureGateAccess interface { + // SetChangeHandler can only be called before Run. + // The default change handler will exit 0 when the set of featuregates changes. + // That is usually the easiest and simplest thing for an *operator* to do. + // This also discourages direct operand reading since all operands restarting simultaneously is bad. + // This function allows changing that default behavior to something else (perhaps a channel notification for + // all impacted controllers in an operator. + // I doubt this will be worth the effort in the majority of cases. + SetChangeHandler(featureGateChangeHandlerFn FeatureGateChangeHandlerFunc) + + // Run starts a go func that continously watches the set of featuregates enabled in the cluster. + Run(ctx context.Context) + // InitialFeatureGatesObserved returns a channel that is closed once the featuregates have + // been observed. Once closed, the CurrentFeatureGates method will return the current set of + // featuregates and will never return a non-nil error. + InitialFeatureGatesObserved() <-chan struct{} + // CurrentFeatureGates returns the list of enabled and disabled featuregates. + // It returns an error if the current set of featuregates is not known. + CurrentFeatureGates() (FeatureGate, error) + // AreInitialFeatureGatesObserved returns true if the initial featuregates have been observed. + AreInitialFeatureGatesObserved() bool +} + +type Features struct { + Enabled []configv1.FeatureGateName + Disabled []configv1.FeatureGateName +} + +type FeatureChange struct { + Previous *Features + New Features +} + +type defaultFeatureGateAccess struct { + desiredVersion string + missingVersionMarker string + clusterVersionLister configlistersv1.ClusterVersionLister + featureGateLister configlistersv1.FeatureGateLister + initialFeatureGatesObserved chan struct{} + + featureGateChangeHandlerFn FeatureGateChangeHandlerFunc + + lock sync.Mutex + started bool + initialFeatures Features + currentFeatures Features + + queue workqueue.RateLimitingInterface + eventRecorder events.Recorder +} + +// NewFeatureGateAccess returns a controller that keeps the list of enabled/disabled featuregates up to date. +// desiredVersion is the version of this operator that would be set on the clusteroperator.status.versions. +// missingVersionMarker is the stub version provided by the operator. If that is also the desired version, +// then the most either the desired clusterVersion or most recent version will be used. +// clusterVersionInformer is used when desiredVersion and missingVersionMarker are the same to derive the "best" version +// of featuregates to use. +// featureGateInformer is used to track changes to the featureGates once they are initially set. +// By default, when the enabled/disabled list of featuregates changes, os.Exit is called. This behavior can be +// overridden by calling SetChangeHandler to whatever you wish the behavior to be. +// A common construct is: +/* go +featureGateAccessor := NewFeatureGateAccess(args) +go featureGateAccessor.Run(ctx) + +select{ +case <- featureGateAccessor.InitialFeatureGatesObserved(): + featureGates, _ := featureGateAccessor.CurrentFeatureGates() + klog.Infof("FeatureGates initialized: knownFeatureGates=%v", featureGates.KnownFeatures()) +case <- time.After(1*time.Minute): + klog.Errorf("timed out waiting for FeatureGate detection") + return fmt.Errorf("timed out waiting for FeatureGate detection") +} + +// whatever other initialization you have to do, at this point you have FeatureGates to drive your behavior. +*/ +// That construct is easy. It is better to use the .spec.observedConfiguration construct common in library-go operators +// to avoid gating your general startup on FeatureGate determination, but if you haven't already got that mechanism +// this construct is easy. +func NewFeatureGateAccess( + desiredVersion, missingVersionMarker string, + clusterVersionInformer v1.ClusterVersionInformer, + featureGateInformer v1.FeatureGateInformer, + eventRecorder events.Recorder) FeatureGateAccess { + c := &defaultFeatureGateAccess{ + desiredVersion: desiredVersion, + missingVersionMarker: missingVersionMarker, + clusterVersionLister: clusterVersionInformer.Lister(), + featureGateLister: featureGateInformer.Lister(), + initialFeatureGatesObserved: make(chan struct{}), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "feature-gate-detector"), + eventRecorder: eventRecorder, + } + c.SetChangeHandler(ForceExit) + + // we aren't expecting many + clusterVersionInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.queue.Add("cluster") + }, + UpdateFunc: func(old, cur interface{}) { + c.queue.Add("cluster") + }, + DeleteFunc: func(uncast interface{}) { + c.queue.Add("cluster") + }, + }) + featureGateInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.queue.Add("cluster") + }, + UpdateFunc: func(old, cur interface{}) { + c.queue.Add("cluster") + }, + DeleteFunc: func(uncast interface{}) { + c.queue.Add("cluster") + }, + }) + + return c +} + +func ForceExit(featureChange FeatureChange) { + if featureChange.Previous != nil { + os.Exit(0) + } +} + +func (c *defaultFeatureGateAccess) SetChangeHandler(featureGateChangeHandlerFn FeatureGateChangeHandlerFunc) { + c.lock.Lock() + defer c.lock.Unlock() + + if c.started { + panic("programmer error, cannot update the change handler after starting") + } + c.featureGateChangeHandlerFn = featureGateChangeHandlerFn +} + +func (c *defaultFeatureGateAccess) Run(ctx context.Context) { + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + + klog.Infof("Starting feature-gate-detector") + defer klog.Infof("Shutting down feature-gate-detector") + + go wait.UntilWithContext(ctx, c.runWorker, time.Second) + + <-ctx.Done() +} + +func (c *defaultFeatureGateAccess) syncHandler(ctx context.Context) error { + desiredVersion := c.desiredVersion + if c.missingVersionMarker == c.desiredVersion { + clusterVersion, err := c.clusterVersionLister.Get("version") + if apierrors.IsNotFound(err) { + return nil // we will be re-triggered when it is created + } + if err != nil { + return err + } + + desiredVersion = clusterVersion.Status.Desired.Version + if len(desiredVersion) == 0 && len(clusterVersion.Status.History) > 0 { + desiredVersion = clusterVersion.Status.History[0].Version + } + } + + featureGate, err := c.featureGateLister.Get("cluster") + if apierrors.IsNotFound(err) { + return nil // we will be re-triggered when it is created + } + if err != nil { + return err + } + + features, err := featuresFromFeatureGate(featureGate, desiredVersion) + if err != nil { + return fmt.Errorf("unable to determine features: %w", err) + } + + c.setFeatureGates(features) + + return nil +} + +func (c *defaultFeatureGateAccess) setFeatureGates(features Features) { + c.lock.Lock() + defer c.lock.Unlock() + + var previousFeatures *Features + if c.AreInitialFeatureGatesObserved() { + t := c.currentFeatures + previousFeatures = &t + } + + c.currentFeatures = features + + if !c.AreInitialFeatureGatesObserved() { + c.initialFeatures = features + close(c.initialFeatureGatesObserved) + c.eventRecorder.Eventf("FeatureGatesInitialized", "FeatureGates updated to %#v", c.currentFeatures) + } + + if previousFeatures == nil || !reflect.DeepEqual(*previousFeatures, c.currentFeatures) { + if previousFeatures != nil { + c.eventRecorder.Eventf("FeatureGatesModified", "FeatureGates updated to %#v", c.currentFeatures) + } + + c.featureGateChangeHandlerFn(FeatureChange{ + Previous: previousFeatures, + New: c.currentFeatures, + }) + } +} + +func (c *defaultFeatureGateAccess) InitialFeatureGatesObserved() <-chan struct{} { + return c.initialFeatureGatesObserved +} + +func (c *defaultFeatureGateAccess) AreInitialFeatureGatesObserved() bool { + select { + case <-c.InitialFeatureGatesObserved(): + return true + default: + return false + } +} + +func (c *defaultFeatureGateAccess) CurrentFeatureGates() (FeatureGate, error) { + c.lock.Lock() + defer c.lock.Unlock() + + if !c.AreInitialFeatureGatesObserved() { + return nil, fmt.Errorf("featureGates not yet observed") + } + retEnabled := make([]configv1.FeatureGateName, len(c.currentFeatures.Enabled)) + retDisabled := make([]configv1.FeatureGateName, len(c.currentFeatures.Disabled)) + copy(retEnabled, c.currentFeatures.Enabled) + copy(retDisabled, c.currentFeatures.Disabled) + + return NewFeatureGate(retEnabled, retDisabled), nil +} + +func (c *defaultFeatureGateAccess) runWorker(ctx context.Context) { + for c.processNextWorkItem(ctx) { + } +} + +func (c *defaultFeatureGateAccess) processNextWorkItem(ctx context.Context) bool { + dsKey, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(dsKey) + + err := c.syncHandler(ctx) + if err == nil { + c.queue.Forget(dsKey) + return true + } + + utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err)) + c.queue.AddRateLimited(dsKey) + + return true +} + +func featuresFromFeatureGate(featureGate *configv1.FeatureGate, desiredVersion string) (Features, error) { + found := false + features := Features{} + for _, featureGateValues := range featureGate.Status.FeatureGates { + if featureGateValues.Version != desiredVersion { + continue + } + found = true + for _, enabled := range featureGateValues.Enabled { + features.Enabled = append(features.Enabled, enabled.Name) + } + for _, disabled := range featureGateValues.Disabled { + features.Disabled = append(features.Disabled, disabled.Name) + } + break + } + + if !found { + return Features{}, fmt.Errorf("missing desired version %q in featuregates.config.openshift.io/cluster", desiredVersion) + } + + return features, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 1039803a..38d6a3ae 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -288,6 +288,7 @@ github.com/openshift/library-go/pkg/crypto github.com/openshift/library-go/pkg/network github.com/openshift/library-go/pkg/operator/condition github.com/openshift/library-go/pkg/operator/configobserver +github.com/openshift/library-go/pkg/operator/configobserver/featuregates github.com/openshift/library-go/pkg/operator/configobserver/proxy github.com/openshift/library-go/pkg/operator/csi/credentialsrequestcontroller github.com/openshift/library-go/pkg/operator/csi/csiconfigobservercontroller