controller: Wait for template and kubelet before starting renderer

cmd/machine-config-controller/start.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"flag"
+	"sync"
 
 	"github.com/golang/glog"
 	"github.com/openshift/machine-config-operator/cmd/common"
@@ -81,16 +82,22 @@ func runStartCmd(cmd *cobra.Command, args []string) {
 }
 
 func startControllers(ctx *common.ControllerContext) error {
+	var readyFlag sync.WaitGroup
+
 	// Our primary MCs come from here
+	readyFlag.Add(1)
 	go template.New(
 		rootOpts.templates,
 		ctx.InformerFactory.Machineconfiguration().V1().ControllerConfigs(),
 		ctx.InformerFactory.Machineconfiguration().V1().MachineConfigs(),
 		ctx.ClientBuilder.KubeClientOrDie("template-controller"),
 		ctx.ClientBuilder.MachineConfigClientOrDie("template-controller"),
+		&readyFlag,
 	).Run(2, ctx.Stop)
 
-	// Add all "sub-renderers here"
+	// Add all "sub-renderers here".  If any of them are *required*, then
+	// you should add a readyFlag for them and notify when their fragment is ready.
+	// By default there's no `kubeletconfig`, so we don't need to block on this today.
 	go kubeletconfig.New(
 		rootOpts.templates,
 		ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(),
@@ -111,22 +118,27 @@ func startControllers(ctx *common.ControllerContext) error {
 		ctx.ClientBuilder.ConfigClientOrDie("container-runtime-config-controller"),
 	).Run(2, ctx.Stop)
 
-	// The renderer creates "rendered" MCs from the MC fragments generated by
-	// the above sub-controllers, which are then consumed by the node controller
-	go render.New(
-		ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(),
-		ctx.InformerFactory.Machineconfiguration().V1().MachineConfigs(),
-		ctx.ClientBuilder.KubeClientOrDie("render-controller"),
-		ctx.ClientBuilder.MachineConfigClientOrDie("render-controller"),
-	).Run(2, ctx.Stop)
-
-	// The node controller consumes data written by the above
-	go node.New(
-		ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(),
-		ctx.KubeInformerFactory.Core().V1().Nodes(),
-		ctx.ClientBuilder.KubeClientOrDie("node-update-controller"),
-		ctx.ClientBuilder.MachineConfigClientOrDie("node-update-controller"),
-	).Run(2, ctx.Stop)
+	// Async wait for the MC fragments before starting the renderer
+	go func() {
+		readyFlag.Wait()
+
+		// The renderer creates "rendered" MCs from the MC fragments generated by
+		// the above sub-controllers, which are then consumed by the node controller
+		go render.New(
+			ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(),
+			ctx.InformerFactory.Machineconfiguration().V1().MachineConfigs(),
+			ctx.ClientBuilder.KubeClientOrDie("render-controller"),
+			ctx.ClientBuilder.MachineConfigClientOrDie("render-controller"),
+		).Run(2, ctx.Stop)
+
+		// The node controller consumes data written by the above
+		go node.New(
+			ctx.InformerFactory.Machineconfiguration().V1().MachineConfigPools(),
+			ctx.KubeInformerFactory.Core().V1().Nodes(),
+			ctx.ClientBuilder.KubeClientOrDie("node-update-controller"),
+			ctx.ClientBuilder.MachineConfigClientOrDie("node-update-controller"),
+		).Run(2, ctx.Stop)
+	}()
 
 	return nil
 }

pkg/controller/template/template_controller.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"sort"
+	"sync"
 	"time"
 
 	"github.com/golang/glog"
@@ -47,6 +48,13 @@ type Controller struct {
 	kubeClient    clientset.Interface
 	eventRecorder record.EventRecorder
 
+	// firstSync is in charge of closing the readyFlag WaitGroup the first time
+	// the controller is initialized on startup
+	firstSync sync.Once
+	// readyFlag is in charge of the very first sync of the controller
+	// with the others
+	readyFlag *sync.WaitGroup
+
 	syncHandler             func(ccKey string) error
 	enqueueControllerConfig func(*mcfgv1.ControllerConfig)
 
@@ -66,6 +74,7 @@ func New(
 	mcInformer mcfginformersv1.MachineConfigInformer,
 	kubeClient clientset.Interface,
 	mcfgClient mcfgclientset.Interface,
+	readyFlag *sync.WaitGroup,
 ) *Controller {
 	eventBroadcaster := record.NewBroadcaster()
 	eventBroadcaster.StartLogging(glog.Infof)
@@ -100,6 +109,8 @@ func New(
 	ctrl.ccListerSynced = ccInformer.Informer().HasSynced
 	ctrl.mcListerSynced = mcInformer.Informer().HasSynced
 
+	ctrl.readyFlag = readyFlag
+
 	return ctrl
 }
 
@@ -365,6 +376,11 @@ func (ctrl *Controller) syncControllerConfig(key string) error {
 		}
 	}
 
+	ctrl.firstSync.Do(func() {
+		glog.Infof("Initial template sync done")
+		ctrl.readyFlag.Done()
+	})
+
 	return ctrl.syncCompletedStatus(cfg)
 }
 

pkg/controller/template/template_controller_test.go

@@ -5,6 +5,7 @@ import (
 	"reflect"
 	"testing"
 	"time"
+	"sync"
 
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
@@ -82,9 +83,11 @@ func (f *fixture) newController() *Controller {
 	f.kubeclient = k8sfake.NewSimpleClientset(f.kubeobjects...)
 
 	i := informers.NewSharedInformerFactory(f.client, noResyncPeriodFunc())
+	var readyFlag sync.WaitGroup
+	readyFlag.Add(1)
 	c := New(templateDir,
 		i.Machineconfiguration().V1().ControllerConfigs(), i.Machineconfiguration().V1().MachineConfigs(),
-		f.kubeclient, f.client)
+		f.kubeclient, f.client, &readyFlag)
 
 	c.ccListerSynced = alwaysReady
 	c.mcListerSynced = alwaysReady