From 78d0026e89398c77bf14cd5b761278f5739ff2e8 Mon Sep 17 00:00:00 2001 From: Vince Prignano Date: Tue, 15 Sep 2020 12:52:49 -0700 Subject: [PATCH] :bug: Controller.Watch() should not store watches if already started The controller internal struct holds a list of watches (as []watchDescription) when someone calls .Watch() to then start the watches and informers once we're ready to call Start(). This behavior caused a memory leak in the case Watch was called after a controller has already been started and if the source.Kind's cache was either stopped or not available any longer. The leak was caused by the watches internal slice holding on to all references to each watch ever issued (and their respective caches). Signed-off-by: Vince Prignano --- pkg/internal/controller/controller.go | 27 ++++++++++++++-------- pkg/internal/controller/controller_test.go | 4 ++-- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/pkg/internal/controller/controller.go b/pkg/internal/controller/controller.go index bb782d2fad..07d51cd237 100644 --- a/pkg/internal/controller/controller.go +++ b/pkg/internal/controller/controller.go @@ -71,8 +71,8 @@ type Controller struct { // TODO(community): Consider initializing a logger with the Controller Name as the tag - // watches maintains a list of sources, handlers, and predicates to start when the controller is started. - watches []watchDescription + // startWatches maintains a list of sources, handlers, and predicates to start when the controller is started. + startWatches []watchDescription // Log is used to log messages to users during reconciliation, or for example when a watch is started. Log logr.Logger @@ -108,13 +108,16 @@ func (c *Controller) Watch(src source.Source, evthdler handler.EventHandler, prc } } - c.watches = append(c.watches, watchDescription{src: src, handler: evthdler, predicates: prct}) - if c.Started { - c.Log.Info("Starting EventSource", "source", src) - return src.Start(evthdler, c.Queue, prct...) + // Controller hasn't started yet, store the watches locally and return. + // + // These watches are going to be held on the controller struct until the manager or user calls Start(...). + if !c.Started { + c.startWatches = append(c.startWatches, watchDescription{src: src, handler: evthdler, predicates: prct}) + return nil } - return nil + c.Log.Info("Starting EventSource", "source", src) + return src.Start(evthdler, c.Queue, prct...) } // Start implements controller.Controller @@ -135,7 +138,7 @@ func (c *Controller) Start(stop <-chan struct{}) error { // NB(directxman12): launch the sources *before* trying to wait for the // caches to sync so that they have a chance to register their intendeded // caches. - for _, watch := range c.watches { + for _, watch := range c.startWatches { c.Log.Info("Starting EventSource", "source", watch.src) if err := watch.src.Start(watch.handler, c.Queue, watch.predicates...); err != nil { return err @@ -145,7 +148,7 @@ func (c *Controller) Start(stop <-chan struct{}) error { // Start the SharedIndexInformer factories to begin populating the SharedIndexInformer caches c.Log.Info("Starting Controller") - for _, watch := range c.watches { + for _, watch := range c.startWatches { syncingSource, ok := watch.src.(source.SyncingSource) if !ok { continue @@ -159,6 +162,12 @@ func (c *Controller) Start(stop <-chan struct{}) error { } } + // All the watches have been started, we can reset the local slice. + // + // We should never hold watches more than necessary, each watch source can hold a backing cache, + // which won't be garbage collected if we hold a reference to it. + c.startWatches = nil + if c.JitterPeriod == 0 { c.JitterPeriod = 1 * time.Second } diff --git a/pkg/internal/controller/controller_test.go b/pkg/internal/controller/controller_test.go index b57c542770..c936031500 100644 --- a/pkg/internal/controller/controller_test.go +++ b/pkg/internal/controller/controller_test.go @@ -91,7 +91,7 @@ var _ = Describe("controller", func() { Describe("Start", func() { It("should return an error if there is an error waiting for the informers", func(done Done) { f := false - ctrl.watches = []watchDescription{{ + ctrl.startWatches = []watchDescription{{ src: source.NewKindWithCache(&corev1.Pod{}, &informertest.FakeInformers{Synced: &f}), }} ctrl.Name = "foo" @@ -115,7 +115,7 @@ var _ = Describe("controller", func() { Expect(err).NotTo(HaveOccurred()) _, err = c.GetInformer(context.TODO(), &appsv1.ReplicaSet{}) Expect(err).NotTo(HaveOccurred()) - ctrl.watches = []watchDescription{{ + ctrl.startWatches = []watchDescription{{ src: source.NewKindWithCache(&appsv1.Deployment{}, &informertest.FakeInformers{}), }}