feat: add nodepool scoped maestro readonly bundles controllers

backend/pkg/app/backend.go

@@ -495,14 +495,25 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context,
 		activeOperationLister,
 		backendInformers,
 	)
-	maestroCreateReadonlyBundlesController := controllers.NewCreateClusterScopedMaestroReadonlyBundlesController(
+
+	maestroCreateClusterScopedReadonlyBundlesController := controllers.NewCreateClusterScopedMaestroReadonlyBundlesController(
 		activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient,
 		backendInformers, b.options.MaestroSourceEnvironmentIdentifier, maestroClientBuilder,
 	)
-	maestroReadAndPersistReadonlyBundlesContentController := controllers.NewReadAndPersistClusterScopedMaestroReadonlyBundlesContentController(
+	maestroReadAndPersistClusterScopedReadonlyBundlesContentController := controllers.NewReadAndPersistClusterScopedMaestroReadonlyBundlesContentController(
+		activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient,
+		backendInformers, b.options.MaestroSourceEnvironmentIdentifier, maestroClientBuilder,
+	)
+
+	maestroCreateNodePoolScopedReadonlyBundlesController := controllers.NewCreateNodePoolScopedMaestroReadonlyBundlesController(
 		activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient,
 		backendInformers, b.options.MaestroSourceEnvironmentIdentifier, maestroClientBuilder,
 	)
+	maestroReadAndPersistNodePoolScopedReadonlyBundlesContentController := controllers.NewReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentController(
+		activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient,
+		backendInformers, b.options.MaestroSourceEnvironmentIdentifier, maestroClientBuilder,
+	)
+
 	maestroDeleteOrphanedReadonlyBundlesController := controllers.NewDeleteOrphanedMaestroReadonlyBundlesController(
 		b.options.CosmosDBClient,
 		b.options.ClustersServiceClient,
@@ -609,8 +620,10 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context,
 				go azureClusterResourceGroupExistenceValidationController.Run(ctx, 20)
 				go azureClusterManagedIdentitiesExistenceValidationController.Run(ctx, 20)
 				go nodePoolVersionController.Run(ctx, 20)
-				go maestroCreateReadonlyBundlesController.Run(ctx, 20)
-				go maestroReadAndPersistReadonlyBundlesContentController.Run(ctx, 20)
+				go maestroCreateClusterScopedReadonlyBundlesController.Run(ctx, 20)
+				go maestroReadAndPersistClusterScopedReadonlyBundlesContentController.Run(ctx, 20)
+				go maestroCreateNodePoolScopedReadonlyBundlesController.Run(ctx, 20)
+				go maestroReadAndPersistNodePoolScopedReadonlyBundlesContentController.Run(ctx, 20)
 				go maestroDeleteOrphanedReadonlyBundlesController.Run(ctx, 20)
 				go triggerNodePoolUpgradeController.Run(ctx, 20)
 				go nodePoolPropertiesSyncController.Run(ctx, 20)

backend/pkg/controllers/controllerutils/cluster_watching_controller.go

@@ -64,8 +64,13 @@ func NewClusterWatchingController(
 	if informers != nil {
 		clusterInformer, _ := informers.Clusters()
 		serviceProviderInformer, _ := informers.ServiceProviderClusters()
+		err := clusterController.QueueForInformers(resyncDuration, clusterInformer, serviceProviderInformer)
+		if err != nil {
+			panic(err) // coding error
+		}
 		managementClusterContentInformer, _ := informers.ManagementClusterContents()
-		err := clusterController.QueueForInformers(resyncDuration, clusterInformer, serviceProviderInformer, managementClusterContentInformer)
+		// Limit the max depth of ManagementClusterContent to 1 to only consider the cluster-scoped ManagementClusterContents
+		err = clusterController.QueueForInformersWithMaxDepth(resyncDuration, 1, managementClusterContentInformer)
 		if err != nil {
 			panic(err) // coding error
 		}

backend/pkg/controllers/controllerutils/generic_watching_controller.go

@@ -143,13 +143,25 @@ func (c *genericWatchingController[T]) processNextWorkItem(ctx context.Context)
 	return true
 }
 
+// QueueForInformers is equivalent to calling QueueForInformersWithMaxDepth with maxDepth of -1.
+// See QueueForInformersWithMaxDepth for more details.
 func (c *genericWatchingController[T]) QueueForInformers(resyncDuration time.Duration, notifiers ...Notifier) error {
+	return c.QueueForInformersWithMaxDepth(resyncDuration, -1, notifiers...)
+}
+
+// QueueForInformersWithMaxDepth adds event handlers to the notifiers for the controller with a given max depth.
+// maxDepth is the maximum number of parent hops to traverse when searching for a resourceID whose type is c.resourceType. Each
+// walk to Parent consumes one level.
+// maxDepth 0 means only the resourceID itself is considered.
+// maxDepth -1 (or any negative value) means no limit. The parent walk continues until a match or nil parent is reached.
+// It is exposed so that individual controllers can add other items to requeue based on easily.
+func (c *genericWatchingController[T]) QueueForInformersWithMaxDepth(resyncDuration time.Duration, maxDepth int, notifiers ...Notifier) error {
 	errs := []error{}
 	for _, notifier := range notifiers {
 		_, err := notifier.AddEventHandlerWithOptions(
 			cache.ResourceEventHandlerFuncs{
-				AddFunc:    c.EnqueueCosmosAdd,
-				UpdateFunc: c.EnqueueCosmosUpdate,
+				AddFunc:    c.enqueueCosmosAddFunc(maxDepth),
+				UpdateFunc: c.enqueueCosmosUpdateFunc(maxDepth),
 			},
 			cache.HandlerOptions{
 				ResyncPeriod: ptr.To(resyncDuration),
@@ -159,14 +171,34 @@ func (c *genericWatchingController[T]) QueueForInformers(resyncDuration time.Dur
 	return errors.Join(errs...)
 }
 
-// EnqueueResourceIDAdd traverses to find a resourceID that is an hcpcluster and adds it if found.
+// enqueueResourceIDAdd is equivalent to calling EnqueueResourceIDAddWithMaxDepth with a maxDepth of -1.
+// See EnqueueResourceIDAddWithMaxDepth for more details.
 // It is exposed so that individual controllers can add other items to requeue based on easily.
 func (c *genericWatchingController[T]) EnqueueResourceIDAdd(resourceID *azcorearm.ResourceID, changed bool) {
+	c.EnqueueResourceIDAddWithMaxDepth(resourceID, changed, -1)
+}
+
+// enqueueResourceIDAddWithMaxDepth traverses resourceID and its parents according to maxDepth until it
+// finds a resourceID that is of the resource type of c.resourceType and adds it if found. Each walk to Parent consumes one level.
+// maxDepth is the maximum number of parent hops to traverse when searching for a resourceID of type c.resourceType.
+// maxDepth 0 means only the resourceID itself is considered.
+// maxDepth -1 (or any negative value) means no limit. The parent walk continues until a match or nil parent is reached.
+// It is exposed so that individual controllers can add other items to requeue based on easily.
+// When there's a match of resourceType: when changed is true, the resourceID is added to the queue immediately. Otherwise, the resourceID is
+// added to the queue only if the cooldown checker allows it.
+func (c *genericWatchingController[T]) EnqueueResourceIDAddWithMaxDepth(resourceID *azcorearm.ResourceID, changed bool, maxDepth int) {
 	if resourceID == nil {
 		return
 	}
 	if !armhelpers.ResourceTypeEqual(resourceID.ResourceType, c.resourceType) {
-		c.EnqueueResourceIDAdd(resourceID.Parent, changed)
+		if maxDepth == 0 {
+			return
+		}
+		nextDepth := maxDepth
+		if maxDepth > 0 {
+			nextDepth = maxDepth - 1
+		}
+		c.EnqueueResourceIDAddWithMaxDepth(resourceID.Parent, changed, nextDepth)
 		return
 	}
 
@@ -191,11 +223,23 @@ func (c *genericWatchingController[T]) EnqueueResourceIDAdd(resourceID *azcorear
 	c.queue.Add(key)
 }
 
-func (c *genericWatchingController[T]) EnqueueCosmosAdd(newObj any) {
-	c.EnqueueResourceIDAdd(newObj.(arm.CosmosPersistable).GetCosmosData().GetResourceID(), true)
+func (c *genericWatchingController[T]) enqueueCosmosAddFunc(maxDepth int) func(any) {
+	return func(newObj any) {
+		c.enqueueCosmosAddWithMaxDepth(newObj, maxDepth)
+	}
+}
+
+func (c *genericWatchingController[T]) enqueueCosmosAddWithMaxDepth(newObj any, maxDepth int) {
+	c.EnqueueResourceIDAddWithMaxDepth(newObj.(arm.CosmosPersistable).GetCosmosData().GetResourceID(), true, maxDepth)
+}
+
+func (c *genericWatchingController[T]) enqueueCosmosUpdateFunc(maxDepth int) func(any, any) {
+	return func(oldObj, newObj any) {
+		c.enqueueCosmosUpdateWithMaxDepth(oldObj, newObj, maxDepth)
+	}
 }
 
-func (c *genericWatchingController[T]) EnqueueCosmosUpdate(oldObj, newObj any) {
+func (c *genericWatchingController[T]) enqueueCosmosUpdateWithMaxDepth(oldObj, newObj any, maxDepth int) {
 	changed := oldObj.(arm.CosmosPersistable).GetCosmosData().GetEtag() != newObj.(arm.CosmosPersistable).GetCosmosData().GetEtag()
-	c.EnqueueResourceIDAdd(newObj.(arm.CosmosPersistable).GetCosmosData().GetResourceID(), changed)
+	c.EnqueueResourceIDAddWithMaxDepth(newObj.(arm.CosmosPersistable).GetCosmosData().GetResourceID(), changed, maxDepth)
 }