diff --git a/bindata/assets/config/defaultconfig.yaml b/bindata/assets/config/defaultconfig.yaml index cc085931b..dac76167a 100644 --- a/bindata/assets/config/defaultconfig.yaml +++ b/bindata/assets/config/defaultconfig.yaml @@ -20,7 +20,7 @@ extendedArguments: leader-elect-retry-period: - "3s" leader-elect-resource-lock: - - "configmaps" + - "leases" controllers: - "*" - "-ttl" # TODO: this is excluded in kube-core, but not in #21092 diff --git a/bindata/assets/kube-controller-manager/leader-election-cluster-policy-controller-role.yaml b/bindata/assets/kube-controller-manager/leader-election-cluster-policy-controller-role.yaml index 48e089dc7..77266dcce 100644 --- a/bindata/assets/kube-controller-manager/leader-election-cluster-policy-controller-role.yaml +++ b/bindata/assets/kube-controller-manager/leader-election-cluster-policy-controller-role.yaml @@ -13,6 +13,7 @@ rules: - "" resources: - configmaps + - leases verbs: - get - list diff --git a/bindata/assets/kube-controller-manager/leader-election-kube-controller-manager-role-kube-system.yaml b/bindata/assets/kube-controller-manager/leader-election-kube-controller-manager-role-kube-system.yaml index 87c56e430..05b24a114 100644 --- a/bindata/assets/kube-controller-manager/leader-election-kube-controller-manager-role-kube-system.yaml +++ b/bindata/assets/kube-controller-manager/leader-election-kube-controller-manager-role-kube-system.yaml @@ -13,5 +13,6 @@ rules: - "" resources: - configmaps + - leases verbs: - create diff --git a/go.mod b/go.mod index c5a347027..93399ec74 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/openshift/api v0.0.0-20220110171111-997c316db5e1 github.com/openshift/build-machinery-go v0.0.0-20211213093930-7e33a7eb4ce3 github.com/openshift/client-go v0.0.0-20211209144617-7385dd6338e3 - github.com/openshift/library-go v0.0.0-20220111125907-7f25b9c7ad22 + github.com/openshift/library-go v0.0.0-20220117141918-17ce0045e35e github.com/prometheus/common v0.28.0 github.com/spf13/cobra v1.2.1 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index c7015be1f..88deca37c 100644 --- a/go.sum +++ b/go.sum @@ -522,8 +522,8 @@ github.com/openshift/build-machinery-go v0.0.0-20211213093930-7e33a7eb4ce3 h1:65 github.com/openshift/build-machinery-go v0.0.0-20211213093930-7e33a7eb4ce3/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE= github.com/openshift/client-go v0.0.0-20211209144617-7385dd6338e3 h1:SG1aqwleU6bGD0X4mhkTNupjVnByMYYuW4XbnCPavQU= github.com/openshift/client-go v0.0.0-20211209144617-7385dd6338e3/go.mod h1:cwhyki5lqBmrT0m8Im+9I7PGFaraOzcYPtEz93RcsGY= -github.com/openshift/library-go v0.0.0-20220111125907-7f25b9c7ad22 h1:yi4NoYekLpqHqatGMwashmyxui0mI3AcoWMPozuCZfA= -github.com/openshift/library-go v0.0.0-20220111125907-7f25b9c7ad22/go.mod h1:4UQ9snU1vg53fyTpHQw3vLPiAxI8ub5xrc+y8KPQQFs= +github.com/openshift/library-go v0.0.0-20220117141918-17ce0045e35e h1:O3M+vt90vglXRKyb9At4S9Z8MpKHwlRJlc1HbokZlt4= +github.com/openshift/library-go v0.0.0-20220117141918-17ce0045e35e/go.mod h1:4UQ9snU1vg53fyTpHQw3vLPiAxI8ub5xrc+y8KPQQFs= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= diff --git a/pkg/cmd/render/render_test.go b/pkg/cmd/render/render_test.go index 7e2ef2a8a..450e3afa8 100644 --- a/pkg/cmd/render/render_test.go +++ b/pkg/cmd/render/render_test.go @@ -181,7 +181,7 @@ func TestRenderCommand(t *testing.T) { "--flex-volume-plugin-dir=/etc/kubernetes/kubelet-plugins/volume/exec", "--kube-api-burst=300", "--kube-api-qps=150", - "--leader-elect-resource-lock=configmaps", + "--leader-elect-resource-lock=leases", "--leader-elect-retry-period=3s", "--leader-elect=true", "--pv-recycler-pod-template-filepath-hostpath=", diff --git a/test/e2e/operator_test.go b/test/e2e/operator_test.go index d3774517e..5d3b35888 100644 --- a/test/e2e/operator_test.go +++ b/test/e2e/operator_test.go @@ -264,15 +264,15 @@ func TestKCMRecovery(t *testing.T) { ctx := context.Background() - // Try to delete the kube controller manager's configmap in kube-system - err = kubeClient.CoreV1().ConfigMaps("kube-system").Delete(ctx, "kube-controller-manager", metav1.DeleteOptions{}) + // Try to delete the kube controller manager's lease object in kube-system + err = kubeClient.CoordinationV1().Leases("kube-system").Delete(ctx, "kube-controller-manager", metav1.DeleteOptions{}) if err != nil { t.Fatal(err) } - // Check to see that the configmap then gets recreated + // Check to see that the lease object then gets recreated err = wait.Poll(time.Second*5, time.Second*300, func() (bool, error) { - _, err := kubeClient.CoreV1().ConfigMaps("kube-system").Get(ctx, "kube-controller-manager", metav1.GetOptions{}) + _, err := kubeClient.CoordinationV1().Leases("kube-system").Get(ctx, "kube-controller-manager", metav1.GetOptions{}) if errors.IsNotFound(err) { return false, nil } diff --git a/vendor/github.com/openshift/library-go/pkg/config/leaderelection/leaderelection.go b/vendor/github.com/openshift/library-go/pkg/config/leaderelection/leaderelection.go index 5cec68257..edbd10fb1 100644 --- a/vendor/github.com/openshift/library-go/pkg/config/leaderelection/leaderelection.go +++ b/vendor/github.com/openshift/library-go/pkg/config/leaderelection/leaderelection.go @@ -22,8 +22,15 @@ import ( configv1 "github.com/openshift/api/config/v1" ) -// ToConfigMapLeaderElection returns a leader election config that you just need to fill in the Callback for. Don't forget the callbacks! -func ToConfigMapLeaderElection(clientConfig *rest.Config, config configv1.LeaderElection, component, identity string) (leaderelection.LeaderElectionConfig, error) { +// ToLeaderElectionWithConfigmapLease returns a "configmapsleases" based leader +// election config that you just need to fill in the Callback for. +// It is compatible with a "configmaps" based leader election and +// paves the way toward using "leases" based leader election. +// See https://github.com/kubernetes/kubernetes/issues/107454 for +// details on how to migrate to "leases" leader election. +// Don't forget the callbacks! +// TODO: In the next version we should switch to using "leases" +func ToLeaderElectionWithConfigmapLease(clientConfig *rest.Config, config configv1.LeaderElection, component, identity string) (leaderelection.LeaderElectionConfig, error) { kubeClient, err := kubernetes.NewForConfig(clientConfig) if err != nil { return leaderelection.LeaderElectionConfig{}, err @@ -50,7 +57,7 @@ func ToConfigMapLeaderElection(clientConfig *rest.Config, config configv1.Leader eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events("")}) eventRecorder := eventBroadcaster.NewRecorder(clientgoscheme.Scheme, corev1.EventSource{Component: component}) rl, err := resourcelock.New( - resourcelock.ConfigMapsResourceLock, + resourcelock.ConfigMapsLeasesResourceLock, config.Namespace, config.Name, kubeClient.CoreV1(), diff --git a/vendor/github.com/openshift/library-go/pkg/controller/controllercmd/builder.go b/vendor/github.com/openshift/library-go/pkg/controller/controllercmd/builder.go index f98b86f3d..820892a17 100644 --- a/vendor/github.com/openshift/library-go/pkg/controller/controllercmd/builder.go +++ b/vendor/github.com/openshift/library-go/pkg/controller/controllercmd/builder.go @@ -329,7 +329,7 @@ func (b *ControllerBuilder) Run(ctx context.Context, config *unstructured.Unstru leaderConfig := rest.CopyConfig(protoConfig) leaderConfig.Timeout = b.leaderElection.RenewDeadline.Duration - leaderElection, err := leaderelectionconverter.ToConfigMapLeaderElection(leaderConfig, *b.leaderElection, b.componentName, b.instanceIdentity) + leaderElection, err := leaderelectionconverter.ToLeaderElectionWithConfigmapLease(leaderConfig, *b.leaderElection, b.componentName, b.instanceIdentity) if err != nil { return err } diff --git a/vendor/github.com/openshift/library-go/pkg/operator/staticresourcecontroller/static_resource_controller.go b/vendor/github.com/openshift/library-go/pkg/operator/staticresourcecontroller/static_resource_controller.go index 7c3f3e691..8ef536e66 100644 --- a/vendor/github.com/openshift/library-go/pkg/operator/staticresourcecontroller/static_resource_controller.go +++ b/vendor/github.com/openshift/library-go/pkg/operator/staticresourcecontroller/static_resource_controller.go @@ -51,10 +51,17 @@ func init() { utilruntime.Must(admissionregistrationv1.AddToScheme(genericScheme)) } +// StaticResourcesPreconditionsFuncType checks if the precondition is met (is true) and then proceeds with the sync. +// When the requirement is not met, the controller reports degraded status. +// +// In case, the returned ready flag is false, a proper error with a valid description is recommended. +type StaticResourcesPreconditionsFuncType func(ctx context.Context) (bool, error) + type StaticResourceController struct { name string manifests []conditionalManifests ignoreNotFoundOnCreate bool + preconfitions []StaticResourcesPreconditionsFuncType operatorClient v1helpers.OperatorClient clients *resourceapply.ClientHolder @@ -99,6 +106,8 @@ func NewStaticResourceController( operatorClient: operatorClient, clients: clients, + preconfitions: []StaticResourcesPreconditionsFuncType{defaultStaticResourcesPreconditionsFunc}, + eventRecorder: eventRecorder.WithComponentSuffix(strings.ToLower(name)), factory: factory.New().WithInformers(operatorClient.Informer()).ResyncEvery(1 * time.Minute), @@ -119,6 +128,18 @@ func (c *StaticResourceController) WithIgnoreNotFoundOnCreate() *StaticResourceC return c } +// WithPrecondition adds a precondition, which blocks the sync method from being executed. Preconditions might be chained using: +// WithPrecondition(a).WithPrecondition(b).WithPrecondition(c). +// If any of the preconditions is false, the sync will result in an error. +// +// The requirement parameter should follow the convention described in the StaticResourcesPreconditionsFuncType. +// +// When the requirement is not met, the controller reports degraded status. +func (c *StaticResourceController) WithPrecondition(precondition StaticResourcesPreconditionsFuncType) *StaticResourceController { + c.preconfitions = append(c.preconfitions, precondition) + return c +} + // WithConditionalResources adds a set of manifests to be created when the shouldCreateFnArg is true and should be // deleted when the shouldDeleteFnArg is true. // If shouldCreateFnArg is nil, then it is always create. @@ -262,6 +283,28 @@ func (c *StaticResourceController) Sync(ctx context.Context, syncContext factory return nil } + for _, precondition := range c.preconfitions { + ready, err := precondition(ctx) + // We don't care about the other preconditions, we just stop on the first one. + if !ready { + var message string + if err != nil { + message = err.Error() + } else { + message = "the operator didn't specify what preconditions are missing" + } + if _, _, updateErr := v1helpers.UpdateStatus(ctx, c.operatorClient, v1helpers.UpdateConditionFn(operatorv1.OperatorCondition{ + Type: fmt.Sprintf("%sDegraded", c.name), + Status: operatorv1.ConditionTrue, + Reason: "PreconditionNotReady", + Message: message, + })); updateErr != nil { + return updateErr + } + return err + } + } + errors := []error{} var notFoundErrorsCount int for _, conditionalManifest := range c.manifests { @@ -393,3 +436,7 @@ func (c *StaticResourceController) RelatedObjects() ([]configv1.ObjectReference, func (c *StaticResourceController) Run(ctx context.Context, workers int) { c.factory.WithSync(c.Sync).ToController(c.Name(), c.eventRecorder).Run(ctx, workers) } + +func defaultStaticResourcesPreconditionsFunc(_ context.Context) (bool, error) { + return true, nil +} diff --git a/vendor/github.com/openshift/library-go/test/library/metrics/query.go b/vendor/github.com/openshift/library-go/test/library/metrics/query.go index de9f34625..8a993215f 100644 --- a/vendor/github.com/openshift/library-go/test/library/metrics/query.go +++ b/vendor/github.com/openshift/library-go/test/library/metrics/query.go @@ -10,15 +10,13 @@ import ( "strings" "time" + routeclient "github.com/openshift/client-go/route/clientset/versioned" + prometheusapi "github.com/prometheus/client_golang/api" + prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/transport" - - prometheusapi "github.com/prometheus/client_golang/api" - prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1" - - routeclient "github.com/openshift/client-go/route/clientset/versioned" ) // NewPrometheusClient returns Prometheus API or error @@ -27,7 +25,7 @@ import ( // `ALERTS{alertname="PodDisruptionBudgetAtLimit",alertstate="pending",namespace="pdbnamespace",poddisruptionbudget="pdbname",prometheus="openshift-monitoring/k8s",service="kube-state-metrics",severity="warning"}==1` // Example query: // `scheduler_scheduling_duration_seconds_sum` -func NewPrometheusClient(ctx context.Context, kclient *kubernetes.Clientset, rc *routeclient.Clientset) (prometheusv1.API, error) { +func NewPrometheusClient(ctx context.Context, kclient kubernetes.Interface, rc routeclient.Interface) (prometheusv1.API, error) { _, err := kclient.CoreV1().Services("openshift-monitoring").Get(ctx, "prometheus-k8s", metav1.GetOptions{}) if err != nil { return nil, err @@ -58,7 +56,7 @@ func NewPrometheusClient(ctx context.Context, kclient *kubernetes.Clientset, rc return createClient(ctx, kclient, host, bearerToken) } -func createClient(ctx context.Context, kclient *kubernetes.Clientset, host, bearerToken string) (prometheusv1.API, error) { +func createClient(ctx context.Context, kclient kubernetes.Interface, host, bearerToken string) (prometheusv1.API, error) { // retrieve router CA routerCAConfigMap, err := kclient.CoreV1().ConfigMaps("openshift-config-managed").Get(ctx, "default-ingress-cert", metav1.GetOptions{}) if err != nil { diff --git a/vendor/modules.txt b/vendor/modules.txt index 064517d4b..e18550cfc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -254,7 +254,7 @@ github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1 github.com/openshift/client-go/route/clientset/versioned github.com/openshift/client-go/route/clientset/versioned/scheme github.com/openshift/client-go/route/clientset/versioned/typed/route/v1 -# github.com/openshift/library-go v0.0.0-20220111125907-7f25b9c7ad22 +# github.com/openshift/library-go v0.0.0-20220117141918-17ce0045e35e ## explicit; go 1.17 github.com/openshift/library-go/pkg/assets github.com/openshift/library-go/pkg/authorization/hardcodedauthorizer