diff --git a/pkg/config/leaderelection/leaderelection.go b/pkg/config/leaderelection/leaderelection.go index edbd10fb1d..1f6cff9d86 100644 --- a/pkg/config/leaderelection/leaderelection.go +++ b/pkg/config/leaderelection/leaderelection.go @@ -29,8 +29,20 @@ import ( // See https://github.com/kubernetes/kubernetes/issues/107454 for // details on how to migrate to "leases" leader election. // Don't forget the callbacks! -// TODO: In the next version we should switch to using "leases" func ToLeaderElectionWithConfigmapLease(clientConfig *rest.Config, config configv1.LeaderElection, component, identity string) (leaderelection.LeaderElectionConfig, error) { + return toLeaderElection(clientConfig, config, component, identity, resourcelock.ConfigMapsLeasesResourceLock) +} + +// ToLeaderElectionWithLease returns a "leases" based leader +// election config that you just need to fill in the Callback for. +// Don't forget the callbacks! +func ToLeaderElectionWithLease(clientConfig *rest.Config, config configv1.LeaderElection, component, identity string) (leaderelection.LeaderElectionConfig, error) { + return toLeaderElection(clientConfig, config, component, identity, resourcelock.LeasesResourceLock) +} + +// toLeaderElection returns a leader election config that you just need to fill in the Callback for. The input +// resourceLock must be a value from the k8s.io/client-go/tools/leaderelection/resourcelock package. +func toLeaderElection(clientConfig *rest.Config, config configv1.LeaderElection, component, identity, resourceLock string) (leaderelection.LeaderElectionConfig, error) { kubeClient, err := kubernetes.NewForConfig(clientConfig) if err != nil { return leaderelection.LeaderElectionConfig{}, err @@ -57,7 +69,7 @@ func ToLeaderElectionWithConfigmapLease(clientConfig *rest.Config, config config eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events("")}) eventRecorder := eventBroadcaster.NewRecorder(clientgoscheme.Scheme, corev1.EventSource{Component: component}) rl, err := resourcelock.New( - resourcelock.ConfigMapsLeasesResourceLock, + resourceLock, config.Namespace, config.Name, kubeClient.CoreV1(), diff --git a/pkg/controller/controllercmd/builder.go b/pkg/controller/controllercmd/builder.go index 977ca34a21..5ed852a25d 100644 --- a/pkg/controller/controllercmd/builder.go +++ b/pkg/controller/controllercmd/builder.go @@ -2,6 +2,7 @@ package controllercmd import ( "context" + "errors" "fmt" "io/ioutil" "os" @@ -29,6 +30,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/tools/record" "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/legacyregistry" @@ -59,19 +61,29 @@ type ControllerContext struct { OperatorNamespace string } +type leaderElectionResourceLock string + +const ( + LeasesResourceLock leaderElectionResourceLock = resourcelock.LeasesResourceLock + ConfigMapsLeasesResourceLock leaderElectionResourceLock = resourcelock.ConfigMapsLeasesResourceLock +) + +var ErrInvalidResourceLock = errors.New("the leader election resource lock is not supported") + // defaultObserverInterval specifies the default interval that file observer will do rehash the files it watches and react to any changes // in those files. var defaultObserverInterval = 5 * time.Second // ControllerBuilder allows the construction of an controller in optional pieces. type ControllerBuilder struct { - kubeAPIServerConfigFile *string - clientOverrides *client.ClientConnectionOverrides - leaderElection *configv1.LeaderElection - fileObserver fileobserver.Observer - fileObserverReactorFn func(file string, action fileobserver.ActionType) error - eventRecorderOptions record.CorrelatorOptions - componentOwnerReference *corev1.ObjectReference + kubeAPIServerConfigFile *string + clientOverrides *client.ClientConnectionOverrides + leaderElection *configv1.LeaderElection + leaderElectionResourceLock leaderElectionResourceLock + fileObserver fileobserver.Observer + fileObserverReactorFn func(file string, action fileobserver.ActionType) error + eventRecorderOptions record.CorrelatorOptions + componentOwnerReference *corev1.ObjectReference startFunc StartFunc componentName string @@ -157,6 +169,14 @@ func (b *ControllerBuilder) WithLeaderElection(leaderElection configv1.LeaderEle return b } +// WithLeaderElectionResourceLock sets the resource lock. If not set, controllercmd.ConfigMapsLeasesResourceLock will +// be used for backwards compatibility. +func (b *ControllerBuilder) WithLeaderElectionResourceLock(resourceLock leaderElectionResourceLock) *ControllerBuilder { + b.leaderElectionResourceLock = resourceLock + + return b +} + // WithVersion accepts a getting that provide binary version information that is used to report build_info information to prometheus func (b *ControllerBuilder) WithVersion(info version.Info) *ControllerBuilder { b.versionInfo = &info @@ -328,8 +348,19 @@ func (b *ControllerBuilder) Run(ctx context.Context, config *unstructured.Unstru // ensure blocking TCP connections don't block the leader election leaderConfig := rest.CopyConfig(protoConfig) leaderConfig.Timeout = b.leaderElection.RenewDeadline.Duration + var leaderElection leaderelection.LeaderElectionConfig + + switch b.leaderElectionResourceLock { + // default to configmapsleases for leader election + // TODO: In the next version we should switch to using "leases" by default + case "", ConfigMapsLeasesResourceLock: + leaderElection, err = leaderelectionconverter.ToLeaderElectionWithConfigmapLease(leaderConfig, *b.leaderElection, b.componentName, b.instanceIdentity) + case LeasesResourceLock: + leaderElection, err = leaderelectionconverter.ToLeaderElectionWithLease(leaderConfig, *b.leaderElection, b.componentName, b.instanceIdentity) + default: + return fmt.Errorf("%w: %s", ErrInvalidResourceLock, b.leaderElectionResourceLock) + } - leaderElection, err := leaderelectionconverter.ToLeaderElectionWithConfigmapLease(leaderConfig, *b.leaderElection, b.componentName, b.instanceIdentity) if err != nil { return err } diff --git a/pkg/controller/controllercmd/cmd.go b/pkg/controller/controllercmd/cmd.go index fb5bd2afbe..b999841809 100644 --- a/pkg/controller/controllercmd/cmd.go +++ b/pkg/controller/controllercmd/cmd.go @@ -17,6 +17,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/version" "k8s.io/apiserver/pkg/server" + "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/component-base/logs" "k8s.io/klog/v2" @@ -43,6 +44,10 @@ type ControllerCommandConfig struct { // DisableServing disables serving metrics, debug and health checks and so on. DisableServing bool + // LeaderElectionResourceLock sets the resource lock. If not set, controllercmd.ConfigMapsLeasesResourceLock will + // be used for backwards compatibility. + LeaderElectionResourceLock leaderElectionResourceLock + // DisableLeaderElection allows leader election to be suspended DisableLeaderElection bool @@ -59,8 +64,9 @@ func NewControllerCommandConfig(componentName string, version version.Info, star basicFlags: NewControllerFlags(), - DisableServing: false, - DisableLeaderElection: false, + LeaderElectionResourceLock: resourcelock.ConfigMapsLeasesResourceLock, + DisableServing: false, + DisableLeaderElection: false, } } @@ -282,6 +288,7 @@ func (c *ControllerCommandConfig) StartController(ctx context.Context) error { WithKubeConfigFile(c.basicFlags.KubeConfigFile, nil). WithComponentNamespace(c.basicFlags.Namespace). WithLeaderElection(config.LeaderElection, c.basicFlags.Namespace, c.componentName+"-lock"). + WithLeaderElectionResourceLock(c.LeaderElectionResourceLock). WithVersion(c.version). WithEventRecorderOptions(events.RecommendedClusterSingletonCorrelatorOptions()). WithRestartOnChange(exitOnChangeReactorCh, startingFileContent, observedFiles...).