From 7bd9773cdfd012c04b4ba509e7c51f57921252a3 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Sun, 7 Apr 2019 11:09:46 +0200 Subject: [PATCH 1/4] pkg/operator: wait for cache on mcp/cc/mcp Signed-off-by: Antonio Murdaca --- pkg/operator/operator.go | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index fb6a65504e..16a4f996f3 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -3,6 +3,7 @@ package operator import ( "encoding/base64" "encoding/json" + "errors" "fmt" "io/ioutil" "os" @@ -88,9 +89,14 @@ type Operator struct { daemonsetListerSynced cache.InformerSynced infraListerSynced cache.InformerSynced networkListerSynced cache.InformerSynced + mcpListerSynced cache.InformerSynced + ccListerSynced cache.InformerSynced + mcListerSynced cache.InformerSynced // queue only ever has one item, but it has nice error handling backoff/retry semantics queue workqueue.RateLimitingInterface + + stopCh <-chan struct{} } // New returns a new machine config operator. @@ -152,8 +158,11 @@ func New( optr.mcoconfigLister = mcoconfigInformer.Lister() optr.mcoconfigListerSynced = mcoconfigInformer.Informer().HasSynced optr.mcpLister = mcpInformer.Lister() + optr.mcpListerSynced = mcpInformer.Informer().HasSynced optr.ccLister = ccInformer.Lister() + optr.ccListerSynced = ccInformer.Informer().HasSynced optr.mcLister = mcInformer.Lister() + optr.mcListerSynced = mcInformer.Informer().HasSynced optr.deployLister = deployInformer.Lister() optr.deployListerSynced = deployInformer.Informer().HasSynced optr.daemonsetLister = daemonsetInformer.Lister() @@ -189,7 +198,6 @@ func (optr *Operator) Run(workers int, stopCh <-chan struct{}) { if !cache.WaitForCacheSync(stopCh, optr.crdListerSynced, - optr.mcoconfigListerSynced, optr.deployListerSynced, optr.daemonsetListerSynced, optr.infraListerSynced, @@ -198,6 +206,20 @@ func (optr *Operator) Run(workers int, stopCh <-chan struct{}) { return } + // these can only be synced after CRDs are installed + if !optr.inClusterBringup { + if !cache.WaitForCacheSync(stopCh, + optr.mcpListerSynced, + optr.ccListerSynced, + optr.mcListerSynced, + ) { + glog.Error("failed to sync caches") + return + } + } + + optr.stopCh = stopCh + for i := 0; i < workers; i++ { go wait.Until(optr.worker, time.Second, stopCh) } @@ -261,6 +283,13 @@ func (optr *Operator) sync(key string) error { return err } + if optr.inClusterBringup { + // sync now our own informers after having installed the CRDs + if !cache.WaitForCacheSync(optr.stopCh, optr.mcpListerSynced, optr.mcListerSynced, optr.ccListerSynced) { + return errors.New("failed to sync caches for informers") + } + } + namespace, _, err := cache.SplitMetaNamespaceKey(key) if err != nil { return err From b9e3eb67b6ba3c391f4b45de0450b86df0de8cf3 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Sun, 7 Apr 2019 11:34:57 +0200 Subject: [PATCH 2/4] pkg/operator: use informers for ConfigMaps Signed-off-by: Antonio Murdaca --- cmd/machine-config-operator/start.go | 1 + pkg/operator/operator.go | 22 +++++++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/cmd/machine-config-operator/start.go b/cmd/machine-config-operator/start.go index 4ff9b55a3a..52a16369b3 100644 --- a/cmd/machine-config-operator/start.go +++ b/cmd/machine-config-operator/start.go @@ -67,6 +67,7 @@ func runStartCmd(cmd *cobra.Command, args []string) { ctrlctx.KubeNamespacedInformerFactory.Rbac().V1().ClusterRoles(), ctrlctx.KubeNamespacedInformerFactory.Rbac().V1().ClusterRoleBindings(), ctrlctx.KubeNamespacedInformerFactory.Core().V1().ConfigMaps(), + ctrlctx.KubeInformerFactory.Core().V1().ConfigMaps(), ctrlctx.ConfigInformerFactory.Config().V1().Infrastructures(), ctrlctx.ConfigInformerFactory.Config().V1().Networks(), ctrlctx.ClientBuilder.MachineConfigClientOrDie(componentName), diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 16a4f996f3..eec33346c9 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -12,7 +12,7 @@ import ( "github.com/golang/glog" configclientset "github.com/openshift/client-go/config/clientset/versioned" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" apiextclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apiextinformersv1beta1 "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions/apiextensions/v1beta1" apiextlistersv1beta1 "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1beta1" @@ -26,6 +26,7 @@ import ( "k8s.io/client-go/kubernetes" coreclientsetv1 "k8s.io/client-go/kubernetes/typed/core/v1" appslisterv1 "k8s.io/client-go/listers/apps/v1" + corelisterv1 "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" @@ -82,6 +83,8 @@ type Operator struct { daemonsetLister appslisterv1.DaemonSetLister infraLister configlistersv1.InfrastructureLister networkLister configlistersv1.NetworkLister + mcoCmLister corelisterv1.ConfigMapLister + clusterCmLister corelisterv1.ConfigMapLister crdListerSynced cache.InformerSynced mcoconfigListerSynced cache.InformerSynced @@ -92,6 +95,8 @@ type Operator struct { mcpListerSynced cache.InformerSynced ccListerSynced cache.InformerSynced mcListerSynced cache.InformerSynced + mcoCmListerSynced cache.InformerSynced + clusterCmListerSynced cache.InformerSynced // queue only ever has one item, but it has nice error handling backoff/retry semantics queue workqueue.RateLimitingInterface @@ -114,7 +119,8 @@ func New( daemonsetInformer appsinformersv1.DaemonSetInformer, clusterRoleInformer rbacinformersv1.ClusterRoleInformer, clusterRoleBindingInformer rbacinformersv1.ClusterRoleBindingInformer, - cmInformer coreinformersv1.ConfigMapInformer, + mcoCmInformer coreinformersv1.ConfigMapInformer, + clusterCmInfomer coreinformersv1.ConfigMapInformer, infraInformer configinformersv1.InfrastructureInformer, networkInformer configinformersv1.NetworkInformer, client mcfgclientset.Interface, @@ -147,12 +153,16 @@ func New( daemonsetInformer.Informer().AddEventHandler(optr.eventHandler()) clusterRoleInformer.Informer().AddEventHandler(optr.eventHandler()) clusterRoleBindingInformer.Informer().AddEventHandler(optr.eventHandler()) - cmInformer.Informer().AddEventHandler(optr.eventHandler()) + mcoCmInformer.Informer().AddEventHandler(optr.eventHandler()) infraInformer.Informer().AddEventHandler(optr.eventHandler()) networkInformer.Informer().AddEventHandler(optr.eventHandler()) optr.syncHandler = optr.sync + optr.clusterCmLister = clusterCmInfomer.Lister() + optr.clusterCmListerSynced = clusterCmInfomer.Informer().HasSynced + optr.mcoCmLister = mcoCmInformer.Lister() + optr.mcoCmListerSynced = mcoCmInformer.Informer().HasSynced optr.crdLister = crdInformer.Lister() optr.crdListerSynced = crdInformer.Informer().HasSynced optr.mcoconfigLister = mcoconfigInformer.Lister() @@ -201,6 +211,8 @@ func (optr *Operator) Run(workers int, stopCh <-chan struct{}) { optr.deployListerSynced, optr.daemonsetListerSynced, optr.infraListerSynced, + optr.mcoCmListerSynced, + optr.clusterCmListerSynced, optr.networkListerSynced) { glog.Error("failed to sync caches") return @@ -371,7 +383,7 @@ func (optr *Operator) sync(key string) error { } func (optr *Operator) getOsImageURL(namespace string) (string, error) { - cm, err := optr.kubeClient.CoreV1().ConfigMaps(namespace).Get(osImageConfigMapName, metav1.GetOptions{}) + cm, err := optr.mcoCmLister.ConfigMaps(namespace).Get(osImageConfigMapName) if err != nil { return "", err } @@ -379,7 +391,7 @@ func (optr *Operator) getOsImageURL(namespace string) (string, error) { } func (optr *Operator) getCAsFromConfigMap(namespace, name, key string) ([]byte, error) { - cm, err := optr.kubeClient.CoreV1().ConfigMaps(namespace).Get(name, metav1.GetOptions{}) + cm, err := optr.clusterCmLister.ConfigMaps(namespace).Get(name) if err != nil { return nil, err } From 4f95b4280ef7c3c569a65f6956431fa07f922012 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Sun, 7 Apr 2019 19:34:39 +0200 Subject: [PATCH 3/4] pkg/operator: loop to add even handler to informers Signed-off-by: Antonio Murdaca --- pkg/operator/operator.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index eec33346c9..bd483e54ef 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -145,17 +145,21 @@ func New( queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machineconfigoperator"), } - mcoconfigInformer.Informer().AddEventHandler(optr.eventHandler()) - controllerConfigInformer.Informer().AddEventHandler(optr.eventHandler()) - serviceAccountInfomer.Informer().AddEventHandler(optr.eventHandler()) - crdInformer.Informer().AddEventHandler(optr.eventHandler()) - deployInformer.Informer().AddEventHandler(optr.eventHandler()) - daemonsetInformer.Informer().AddEventHandler(optr.eventHandler()) - clusterRoleInformer.Informer().AddEventHandler(optr.eventHandler()) - clusterRoleBindingInformer.Informer().AddEventHandler(optr.eventHandler()) - mcoCmInformer.Informer().AddEventHandler(optr.eventHandler()) - infraInformer.Informer().AddEventHandler(optr.eventHandler()) - networkInformer.Informer().AddEventHandler(optr.eventHandler()) + for _, i := range []cache.SharedIndexInformer{ + mcoconfigInformer.Informer(), + controllerConfigInformer.Informer(), + serviceAccountInfomer.Informer(), + crdInformer.Informer(), + deployInformer.Informer(), + daemonsetInformer.Informer(), + clusterRoleInformer.Informer(), + clusterRoleBindingInformer.Informer(), + mcoCmInformer.Informer(), + infraInformer.Informer(), + networkInformer.Informer(), + } { + i.AddEventHandler(optr.eventHandler()) + } optr.syncHandler = optr.sync From b9545cc5644bb9f67a7f71980abf8b04094ecdf4 Mon Sep 17 00:00:00 2001 From: Antonio Murdaca Date: Sun, 7 Apr 2019 22:57:20 +0200 Subject: [PATCH 4/4] pkg/operator: remove unused informer/lister Signed-off-by: Antonio Murdaca --- cmd/machine-config-operator/start.go | 1 - pkg/operator/operator.go | 6 ------ 2 files changed, 7 deletions(-) diff --git a/cmd/machine-config-operator/start.go b/cmd/machine-config-operator/start.go index 52a16369b3..a6dda17124 100644 --- a/cmd/machine-config-operator/start.go +++ b/cmd/machine-config-operator/start.go @@ -55,7 +55,6 @@ func runStartCmd(cmd *cobra.Command, args []string) { controller := operator.New( componentNamespace, componentName, startOpts.imagesFile, - ctrlctx.NamespacedInformerFactory.Machineconfiguration().V1().MCOConfigs(), ctrlctx.NamespacedInformerFactory.Machineconfiguration().V1().MachineConfigPools(), ctrlctx.NamespacedInformerFactory.Machineconfiguration().V1().ControllerConfigs(), ctrlctx.NamespacedInformerFactory.Machineconfiguration().V1().MachineConfigs(), diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index bd483e54ef..e3c5cd5bfd 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -75,7 +75,6 @@ type Operator struct { syncHandler func(ic string) error crdLister apiextlistersv1beta1.CustomResourceDefinitionLister - mcoconfigLister mcfglistersv1.MCOConfigLister mcpLister mcfglistersv1.MachineConfigPoolLister ccLister mcfglistersv1.ControllerConfigLister mcLister mcfglistersv1.MachineConfigLister @@ -87,7 +86,6 @@ type Operator struct { clusterCmLister corelisterv1.ConfigMapLister crdListerSynced cache.InformerSynced - mcoconfigListerSynced cache.InformerSynced deployListerSynced cache.InformerSynced daemonsetListerSynced cache.InformerSynced infraListerSynced cache.InformerSynced @@ -108,7 +106,6 @@ type Operator struct { func New( namespace, name string, imagesFile string, - mcoconfigInformer mcfginformersv1.MCOConfigInformer, mcpInformer mcfginformersv1.MachineConfigPoolInformer, ccInformer mcfginformersv1.ControllerConfigInformer, mcInformer mcfginformersv1.MachineConfigInformer, @@ -146,7 +143,6 @@ func New( } for _, i := range []cache.SharedIndexInformer{ - mcoconfigInformer.Informer(), controllerConfigInformer.Informer(), serviceAccountInfomer.Informer(), crdInformer.Informer(), @@ -169,8 +165,6 @@ func New( optr.mcoCmListerSynced = mcoCmInformer.Informer().HasSynced optr.crdLister = crdInformer.Lister() optr.crdListerSynced = crdInformer.Informer().HasSynced - optr.mcoconfigLister = mcoconfigInformer.Lister() - optr.mcoconfigListerSynced = mcoconfigInformer.Informer().HasSynced optr.mcpLister = mcpInformer.Lister() optr.mcpListerSynced = mcpInformer.Informer().HasSynced optr.ccLister = ccInformer.Lister()