From c1f3b39e9a70fb28437ee215c057101b5d6d6dc6 Mon Sep 17 00:00:00 2001 From: Miguel Soriano Date: Mon, 23 Mar 2026 10:19:09 +0100 Subject: [PATCH 01/16] feat: create nodepool scoped maestro readonly bundles controllers --- backend/pkg/app/backend.go | 20 +- ...ped_maestro_readonly_bundles_controller.go | 515 ++++++++++++++++++ ...ned_maestro_readonly_bundles_controller.go | 275 +++++++++- ...aestro_readonly_bundles_controller_test.go | 452 ++++++++++++++- ...tro_readonly_bundles_content_controller.go | 370 +++++++++++++ .../api/types_serviceprovider_nodepool.go | 14 + internal/api/zz_generated.deepcopy.go | 11 + 7 files changed, 1610 insertions(+), 47 deletions(-) create mode 100644 backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go create mode 100644 backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go diff --git a/backend/pkg/app/backend.go b/backend/pkg/app/backend.go index fd56c2cc880..e719b3e04e8 100644 --- a/backend/pkg/app/backend.go +++ b/backend/pkg/app/backend.go @@ -495,14 +495,24 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, activeOperationLister, backendInformers, ) - maestroCreateReadonlyBundlesController := controllers.NewCreateClusterScopedMaestroReadonlyBundlesController( + maestroCreateClusterScopedReadonlyBundlesController := controllers.NewCreateClusterScopedMaestroReadonlyBundlesController( activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient, backendInformers, b.options.MaestroSourceEnvironmentIdentifier, maestroClientBuilder, ) - maestroReadAndPersistReadonlyBundlesContentController := controllers.NewReadAndPersistClusterScopedMaestroReadonlyBundlesContentController( + maestroReadAndPersistClusterScopedReadonlyBundlesContentController := controllers.NewReadAndPersistClusterScopedMaestroReadonlyBundlesContentController( activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient, backendInformers, b.options.MaestroSourceEnvironmentIdentifier, maestroClientBuilder, ) + + maestroCreateNodePoolScopedReadonlyBundlesController := controllers.NewCreateNodePoolScopedMaestroReadonlyBundlesController( + activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient, + backendInformers, b.options.MaestroSourceEnvironmentIdentifier, maestroClientBuilder, + ) + maestroReadAndPersistNodePoolScopedReadonlyBundlesContentController := controllers.NewReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentController( + activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient, + backendInformers, b.options.MaestroSourceEnvironmentIdentifier, maestroClientBuilder, + ) + maestroDeleteOrphanedReadonlyBundlesController := controllers.NewDeleteOrphanedMaestroReadonlyBundlesController( b.options.CosmosDBClient, b.options.ClustersServiceClient, @@ -609,8 +619,10 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, go azureClusterResourceGroupExistenceValidationController.Run(ctx, 20) go azureClusterManagedIdentitiesExistenceValidationController.Run(ctx, 20) go nodePoolVersionController.Run(ctx, 20) - go maestroCreateReadonlyBundlesController.Run(ctx, 20) - go maestroReadAndPersistReadonlyBundlesContentController.Run(ctx, 20) + go maestroCreateClusterScopedReadonlyBundlesController.Run(ctx, 20) + go maestroReadAndPersistClusterScopedReadonlyBundlesContentController.Run(ctx, 20) + go maestroCreateNodePoolScopedReadonlyBundlesController.Run(ctx, 20) + go maestroReadAndPersistNodePoolScopedReadonlyBundlesContentController.Run(ctx, 20) go maestroDeleteOrphanedReadonlyBundlesController.Run(ctx, 20) go triggerNodePoolUpgradeController.Run(ctx, 20) go nodePoolPropertiesSyncController.Run(ctx, 20) diff --git a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go new file mode 100644 index 00000000000..67b43e83f4e --- /dev/null +++ b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go @@ -0,0 +1,515 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package controllers + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "github.com/google/uuid" + workv1 "open-cluster-management.io/api/work/v1" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + + arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" + hsv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + + "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" + "github.com/Azure/ARO-HCP/backend/pkg/informers" + "github.com/Azure/ARO-HCP/backend/pkg/listers" + "github.com/Azure/ARO-HCP/backend/pkg/maestro" + "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/database" + "github.com/Azure/ARO-HCP/internal/ocm" + "github.com/Azure/ARO-HCP/internal/utils" +) + +const ( + // readonlyBundleManagedByK8sLabelValueClusterScoped is the K8s label associated to the readonlyBundleManagedByK8sLabelKey + // key that indicates that the readonly Maestro bundle is managed by the create + // nodepool scoped maestro readonly bundles controller. + readonlyBundleManagedByK8sLabelValueNodePoolScoped = "create-nodepool-scoped-maestro-readonly-bundles-controller" +) + +// createClusterScopedMaestroReadonlyBundlesSyncer is a controller that creates Maestro readonly bundles for the clusters. +// It is responsible for creating the Maestro readonly bundles and storing a reference to them in Cosmos. It does +// not persist the content of the Maestro bundles themselves. That is the responsibility of the +// readAndPersistMaestroReadonlyBundlesContentSyncer controller. +// As of now we support the creation of a Maestro readonly bundle for the Hypershift's NodePool CRs associated to +// the Cluster. +type createNodePoolScopedMaestroReadonlyBundlesSyncer struct { + cooldownChecker controllerutils.CooldownChecker + + activeOperationLister listers.ActiveOperationLister + + cosmosClient database.DBClient + + clusterServiceClient ocm.ClusterServiceClientSpec + + maestroSourceEnvironmentIdentifier string + + maestroClientBuilder maestro.MaestroClientBuilder + + // uuidV4Generator is used to generate a new UUIDv4. It must be provided. + // We define it as a dependency to enable deterministic testing in some + // scenarios. + uuidV4Generator func() (uuid.UUID, error) +} + +var _ controllerutils.NodePoolSyncer = (*createNodePoolScopedMaestroReadonlyBundlesSyncer)(nil) + +func NewCreateNodePoolScopedMaestroReadonlyBundlesController( + activeOperationLister listers.ActiveOperationLister, + cosmosClient database.DBClient, + clusterServiceClient ocm.ClusterServiceClientSpec, + informers informers.BackendInformers, + maestroSourceEnvironmentIdentifier string, + maestroClientBuilder maestro.MaestroClientBuilder, +) controllerutils.Controller { + + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + cooldownChecker: controllerutils.DefaultActiveOperationPrioritizingCooldown(activeOperationLister), + cosmosClient: cosmosClient, + clusterServiceClient: clusterServiceClient, + activeOperationLister: activeOperationLister, + maestroSourceEnvironmentIdentifier: maestroSourceEnvironmentIdentifier, + maestroClientBuilder: maestroClientBuilder, + uuidV4Generator: uuid.NewRandom, + } + + controller := controllerutils.NewNodePoolWatchingController( + "CreateNodePoolScopedMaestroReadonlyBundles", + cosmosClient, + informers, + 1*time.Minute, + syncer, + ) + + return controller +} + +// generateMaestroBundleInternalName generates a Maestro Bundle internal name for a given NodePool. +// Used to generate a Maestro Bundle internal name for a new Maestro Bundle reference. +// The generated name is a concatenation of the prefix with the lowercased name of the NodePool. +// Example: "readonlyHypershiftNodePool-my-nodepool" +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) generateMaestroBundleInternalName(nodePool *api.HCPOpenShiftClusterNodePool) api.MaestroBundleInternalName { + return api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + strings.ToLower(nodePool.Name)) +} + +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) SyncOnce(ctx context.Context, key controllerutils.HCPNodePoolKey) error { + existingNodePool, err := c.cosmosClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName).Get(ctx, key.HCPClusterName) + if database.IsResponseError(err, http.StatusNotFound) { + return nil // nodepool doesn't exist, no work to do + } + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get NodePool: %w", err)) + } + + existingServiceProviderNodePool, err := database.GetOrCreateServiceProviderNodePool(ctx, c.cosmosClient, key.GetResourceID()) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get or create ServiceProviderNodePool: %w", err)) + } + + // The list of Maestro Bundle internal names that are recognized by the controller. + // Any Maestro Bundle internal name that is not in this list will not be synced by the + // controller and reported as an error. + recognizedMaestroBundles := []api.MaestroBundleInternalName{ + c.generateMaestroBundleInternalName(existingNodePool), + } + + var maestroBundlesToSync []api.MaestroBundleInternalName + // We first check if there's any recognized Maestro Bundle reference that needs to be synced. + for _, maestroBundleInternalName := range recognizedMaestroBundles { + currentMaestroBundleReference, err := existingServiceProviderNodePool.Status.MaestroReadonlyBundles.Get(maestroBundleInternalName) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get Maestro Bundle reference: %w", err)) + } + + if currentMaestroBundleReference == nil { + maestroBundlesToSync = append(maestroBundlesToSync, maestroBundleInternalName) + continue + } + if len(currentMaestroBundleReference.MaestroAPIMaestroBundleName) == 0 { + maestroBundlesToSync = append(maestroBundlesToSync, maestroBundleInternalName) + continue + } + if len(currentMaestroBundleReference.MaestroAPIMaestroBundleID) == 0 { + maestroBundlesToSync = append(maestroBundlesToSync, maestroBundleInternalName) + continue + } + } + if len(maestroBundlesToSync) == 0 { + return nil + } + + serviceProviderNodePoolsDBClient := c.cosmosClient.ServiceProviderNodePools( + key.SubscriptionID, + key.ResourceGroupName, + key.HCPClusterName, + key.HCPNodePoolName, + ) + + // We get the provision shard (management cluster) the CS cluster is allocated to. + // As of now in CS the shard allocation occurs synchronously during aro-hcp cluster creation call in CS API so + // we are guaranteed to have a shard allocated for the cluster. If this changes in the future + // we would need to change the logic in controllers to check that the retrieved cluster has a + // shard allocated. + clusterProvisionShard, err := c.clusterServiceClient.GetClusterProvisionShard(ctx, existingNodePool.ServiceProviderProperties.ClusterServiceID) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get Cluster Provision Shard from Cluster Service: %w", err)) + } + + // We create a new context with a cancel function so we can cancel the Maestro client when the sync is done. + // This is important to avoid leaking resources when the sync is done. + ctx, cancel := context.WithCancel(ctx) + defer cancel() + maestroClient, err := c.createMaestroClientFromProvisionShard(ctx, clusterProvisionShard) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) + } + + csCluster, err := c.clusterServiceClient.GetCluster(ctx, existingNodePool.ServiceProviderProperties.ClusterServiceID) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get Cluster from Cluster Service: %w", err)) + } + csClusterDomainPrefix := csCluster.DomainPrefix() + + // We sync the Maestro Bundles that need to be synced. + // We pass the latest existingServiceProviderCluster into each iteration and use the returned + // updated SPC for the next, so that multiple bundles see persisted updates from previous iterations. + // We always apply updatedSPC (even on error) so in-memory state stays in sync with Cosmos + // when syncMaestroBundle persisted a partial change before failing. + var syncErrors []error + for _, maestroBundleInternalName := range maestroBundlesToSync { + updatedSPNP, syncErr := c.syncMaestroBundle( + ctx, maestroBundleInternalName, existingServiceProviderNodePool, existingNodePool, maestroClient, + serviceProviderNodePoolsDBClient, clusterProvisionShard, csClusterDomainPrefix, + ) + existingServiceProviderNodePool = updatedSPNP + if syncErr != nil { + syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to sync Maestro Bundle %q: %w", maestroBundleInternalName, syncErr))) + } + } + + return utils.TrackError(errors.Join(syncErrors...)) +} + +// syncMaestroBundle ensures the given Maestro bundle exists in Maestro, as well as a reference to it in ServiceProviderCluster. +// It returns the updated ServiceProviderCluster (after any Replace calls) so the caller can pass it into the next sync. +// On error, the first return value is always the lastest persisted ServiceProviderClass SPC, so the +// caller can keep in-memory state in sync and subsequent bundle syncs in the same run never see stale data. +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) syncMaestroBundle( + ctx context.Context, + maestroBundleInternalName api.MaestroBundleInternalName, + existingServiceProviderNodePool *api.ServiceProviderNodePool, + existingNodePool *api.HCPOpenShiftClusterNodePool, + maestroClient maestro.Client, + serviceProviderNodePoolsDBClient database.ServiceProviderNodePoolCRUD, + clusterProvisionShard *arohcpv1alpha1.ProvisionShard, + csClusterDomainPrefix string, +) (*api.ServiceProviderNodePool, error) { + lastPersistedSPNP := existingServiceProviderNodePool + + existingMaestroBundleRef, err := existingServiceProviderNodePool.Status.MaestroReadonlyBundles.Get(maestroBundleInternalName) + if err != nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle reference: %w", err)) + } + // If the Maestro Bundle reference does not exist, we create a new Maestro Bundle + // reference for the Maestro API Maestro Bundle name. When this occurs we also immediately + // store the content in Cosmos. This ensures that we have the name reserved for it + // and it makes it resistant to crashes/reboots. + if existingMaestroBundleRef == nil { + var err error + existingMaestroBundleRef, err = c.buildInitialMaestroBundleReference(maestroBundleInternalName) + if err != nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to build initial Maestro Bundle reference: %w", err)) + } + err = existingServiceProviderNodePool.Status.MaestroReadonlyBundles.Set(existingMaestroBundleRef) + if err != nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to set internal Maestro Bundle reference: %w", err)) + } + existingServiceProviderNodePool, err = serviceProviderNodePoolsDBClient.Replace(ctx, existingServiceProviderNodePool, nil) + if err != nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to replace ServiceProviderCluster in database: %w", err)) + } + lastPersistedSPNP = existingServiceProviderNodePool + existingMaestroBundleRef, err = existingServiceProviderNodePool.Status.MaestroReadonlyBundles.Get(maestroBundleInternalName) + if err != nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle reference: %w", err)) + } + if existingMaestroBundleRef == nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("maestro Bundle reference %q not found in ServiceProviderCluster", maestroBundleInternalName)) + } + } + + // We ensure that the Maestro Bundle exists using the Maestro API + maestroBundleNamespacedName := types.NamespacedName{ + Name: existingMaestroBundleRef.MaestroAPIMaestroBundleName, + Namespace: clusterProvisionShard.MaestroConfig().ConsumerName(), + } + + var desiredMaestroBundle *workv1.ManifestWork + switch maestroBundleInternalName { + case c.generateMaestroBundleInternalName(existingNodePool): + desiredMaestroBundle = c.buildInitialReadonlyMaestroBundleForNodePool(existingNodePool, csClusterDomainPrefix, maestroBundleNamespacedName) + default: + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("unrecognized Maestro Bundle internal name: %s", maestroBundleInternalName)) + } + + resultMaestroBundle, err := c.getOrCreateMaestroBundle(ctx, maestroClient, desiredMaestroBundle) + if err != nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to get or create Maestro Bundle: %w", err)) + } + + // If the Maestro API MaestroBundle ID is not set we store the returned Maestro Bundle ID in the corresponding Maestro Bundle reference of the ServiceProviderCluster in Cosmos. + if len(existingMaestroBundleRef.MaestroAPIMaestroBundleID) == 0 { + bundleID := string(resultMaestroBundle.UID) + existingMaestroBundleRef.MaestroAPIMaestroBundleID = bundleID + err = existingServiceProviderNodePool.Status.MaestroReadonlyBundles.Set(existingMaestroBundleRef) + if err != nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to set Maestro Bundle reference: %w", err)) + } + existingServiceProviderNodePool, err = serviceProviderNodePoolsDBClient.Replace(ctx, existingServiceProviderNodePool, nil) + if err != nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to replace ServiceProviderCluster in database: %w", err)) + } + lastPersistedSPNP = existingServiceProviderNodePool + } + + return lastPersistedSPNP, nil +} + +// buildClusterEmptyNodePool returns an empty hosted cluster representing a Cluster's Hypershift NodePool resource. +// It strictly contains the type information and the object meta information necessary to identify the resource in the management cluster. +// It can be used to provide as the input of a Maestro resource bundle. +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildClusterEmptyNodePool(csClusterID string, csClusterDomainPrefix string, csNodePoolID string) *hsv1beta1.NodePool { + // TODO To calculate the NodePool namespace we pass the maestro source ID because it turns out to have the same + // value as the envName in CS. This is not accurate but it is good enough. + // I would decouple what is the maestro source ID envname part from the envname. The reason being that they are + // conceptually different things, they just happen to have the same value for the envName part. + // I am hesitant to provide a generic "environment name" deployment parameter to backend because people might introduce conditional logic based + // on the environment name which is fragile. The options I see are: + // * Provide a deployment parameter to backend that is named something concrete like "k8s-names-calculations-env-name" or similar to indicate + // that is something that is used to calculate names/namespaces of some k8s resources. + // * Expose in the CS API Cluster payload the "CDNamespace" associated to the cluster and start storing it in cosmos. This would allow to fully + // decouple from this concept of CDNamespace and we would use the stored value when needed. However, if we want to + // create resources in the same namespace as the old ones then we would still need to keep forever the concept of "env name part used to calculate + // some k8s resource names/namespaces". + nodePoolNamespace := c.getNodePoolNamespace(c.maestroSourceEnvironmentIdentifier, csClusterID, csNodePoolID) + nodePoolName := csClusterDomainPrefix + + // We first build the resource (manifest) that we want to put within the Maestro Bundle. + // The resource is empty and it only has the type information and the object meta + // information necessary to identify the resource in the management cluster. + nodePool := &hsv1beta1.NodePool{ + TypeMeta: metav1.TypeMeta{ + Kind: "NodePool", + APIVersion: hsv1beta1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: nodePoolName, + Namespace: nodePoolNamespace, + }, + } + + return nodePool +} + +// buildInitialReadonlyMaestroBundleForNodePool builds an initial readonly Maestro Bundle a the Cluster's Hypershift NodePool. +// Used to create the readonly Maestro bundle associated to it. +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildInitialReadonlyMaestroBundleForNodePool(nodePool *api.HCPOpenShiftClusterNodePool, csClusterDomainPrefix string, maestroBundleNamespacedName types.NamespacedName) *workv1.ManifestWork { + hypershiftNodePool := c.buildClusterEmptyNodePool(nodePool.ServiceProviderProperties.ClusterServiceID.ID(), csClusterDomainPrefix, nodePool.ID.Name) + maestroBundleResourceIdentifier := workv1.ResourceIdentifier{ + Group: hsv1beta1.SchemeGroupVersion.Group, + Resource: "nodepools", + Name: hypershiftNodePool.Name, + Namespace: hypershiftNodePool.Namespace, + } + + return c.buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName, maestroBundleResourceIdentifier, hypershiftNodePool) +} + +// buildInitialReadonlyMaestroBundle builds an initial readonly Maestro Bundle for a given resource specified in obj. +// objResourceIdentifier is the resource identifier of the resource specified in obj. +// maestroBundleNamespacedName is the namespaced name of the Maestro Bundle. +// Used to create the readonly Maestro bundle associated to the resource specified in obj. +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName types.NamespacedName, objResourceIdentifier workv1.ResourceIdentifier, obj runtime.Object) *workv1.ManifestWork { + maestroBundleObjMeta := metav1.ObjectMeta{ + Name: maestroBundleNamespacedName.Name, + Namespace: maestroBundleNamespacedName.Namespace, + ResourceVersion: "0", // TODO is this needed when creating a maestro bundle? + Labels: map[string]string{ + // We define it as a K8s label because Maestro supports server-side filtering based on K8s labels. + // We can define it as a K8s label because for this specific use case we can comply with + // K8s labels length and charset restrictions https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set. + readonlyBundleManagedByK8sLabelKey: readonlyBundleManagedByK8sLabelValueNodePoolScoped, + }, + } + + // We build the Maestro Bundle that will contain the resource specified in obj. + // Aside from putting the resource (manifest) previously built above, we + // also define a FeedbackRule that will allow us to retrieve the whole content + // from the management cluster + maestroBundle := &workv1.ManifestWork{ + ObjectMeta: maestroBundleObjMeta, + Spec: workv1.ManifestWorkSpec{ + Workload: workv1.ManifestsTemplate{ + Manifests: []workv1.Manifest{ + { + RawExtension: runtime.RawExtension{ + // We put the resource (manifest) specified in obj. + // In Maestro only the desired `spec` as defined in the bundle can be retrieved + // from here when querying the Maestro Bundle. + // To retrieve another section other than the desired spec Maestro + // requires defining FeedbackRule(s) in the Maestro bundle. + // For maestro readonly resources, not even the desired spec can be retrieved from here. For + // those type of resources it needs to be retrieved via status feedback rule(s) too. + // For owned resources, here the desired spec can be retrieved but that + // is not necessarily the actual spec in the management cluster side. If that is + // desired it is again necessary to get the spec via FeedbackRule(s). + Object: obj, + }, + }, + }, + }, + ManifestConfigs: []workv1.ManifestConfigOption{ + // We also need to define the ManifestConfig associated to the resource(manifest) + // that is being put within the Maestro Bundle. + { + // ResourceIdentifier needs to be specified and it is the information + // associated to the manifest that is being put within the Maestro Bundle. + ResourceIdentifier: objResourceIdentifier, + // We need to set the UpdateStrategy to read only. This + // creates a "readonly maestro bundle". + UpdateStrategy: &workv1.UpdateStrategy{ + Type: workv1.UpdateStrategyTypeReadOnly, + }, + // We define a feedbackrule based on JSONPath. We alias the name + // of this JSONPath as "resource" and its real JSONPath is "@" which + // signals the whole object is retrieved. This includes both spec + // and status. + FeedbackRules: []workv1.FeedbackRule{ + { + Type: workv1.JSONPathsType, + JsonPaths: []workv1.JsonPath{ + { + Name: "resource", + Path: "@", + }, + }, + }, + }, + }, + }, + }, + } + + return maestroBundle +} + +// buildInitialMaestroBundleReference builds an initial Maestro Bundle reference for a given maestro bundle internal name. +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildInitialMaestroBundleReference(internalName api.MaestroBundleInternalName) (*api.MaestroBundleReference, error) { + maestroAPIMaestroBundleName, err := c.generateNewMaestroAPIMaestroBundleName() + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to generate Maestro API Maestro Bundle name: %w", err)) + } + nodePoolMaestroBundleReference := &api.MaestroBundleReference{ + Name: internalName, + MaestroAPIMaestroBundleName: maestroAPIMaestroBundleName, + MaestroAPIMaestroBundleID: "", + } + + return nodePoolMaestroBundleReference, nil +} + +// generateNewMaestroAPIMaestroBundleName generates a new Maestro API Maestro Bundle name. +// Used to generate a new Maestro API Maestro Bundle name for a new Maestro Bundle reference. +// The generated name is a UUIDv4. +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) generateNewMaestroAPIMaestroBundleName() (string, error) { + newUUIDForMaestroAPIMaestroBundleName, err := c.uuidV4Generator() + if err != nil { + return "", utils.TrackError(fmt.Errorf("failed to generate UUIDv4 for Maestro API Maestro Bundle name: %w", err)) + } + return newUUIDForMaestroAPIMaestroBundleName.String(), nil +} + +// getNodePoolNamespace gets the namespace for the node pool based on the environment name, the cluster service OCM Cluster ID and the node pool service OCM Node Pool ID. +// For example, if the Node Pool URL is /api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111/nodepools/XXXX then the +// cluster service OCM Cluster ID is 11111111111111111111111111111111 and the node pool service OCM Node Pool ID is XXXX. +// The namespace is of the format ocm---. This is how CS calculates Hypershift's NodePool namespace. +// Internally in CS this is the "CDNamespace" attribute associated to the cluster. +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) getNodePoolNamespace(envName string, csClusterID string, csNodePoolID string) string { + return fmt.Sprintf("ocm-%s-%s-%s", envName, csClusterID, csNodePoolID) +} + +// getOrCreateMaestroBundle gets (or creates if it does not exist) a Maestro Bundle for a given Maestro Bundle namespaced name. +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) getOrCreateMaestroBundle(ctx context.Context, maestroClient maestro.Client, maestroBundle *workv1.ManifestWork) (*workv1.ManifestWork, error) { + logger := utils.LoggerFromContext(ctx) + existingMaestroBundle, err := maestroClient.Get(ctx, maestroBundle.Name, metav1.GetOptions{}) + if err == nil { + logger.Info(fmt.Sprintf("retrieved maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) + return existingMaestroBundle, nil + } + if !k8serrors.IsNotFound(err) { + logger.Error(err, "failed to get Maestro Bundle and it is not already exists error") + return nil, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle: %w", err)) + } + + logger.Info(fmt.Sprintf("attempting to create maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) + existingMaestroBundle, err = maestroClient.Create(ctx, maestroBundle, metav1.CreateOptions{}) + if err == nil { + logger.Info(fmt.Sprintf("created maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) + return existingMaestroBundle, nil + } + if !k8serrors.IsAlreadyExists(err) { + logger.Error(err, "failed to create Maestro Bundle and it is not already exists error") + return nil, utils.TrackError(fmt.Errorf("failed to create Maestro Bundle: %w", err)) + } + logger.Error(err, "failed to create Maestro Bundle because it returned already exists error. Attempting to get it again") + existingMaestroBundle, err = maestroClient.Get(ctx, maestroBundle.Name, metav1.GetOptions{}) + return existingMaestroBundle, err +} + +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) CooldownChecker() controllerutils.CooldownChecker { + return c.cooldownChecker +} + +// createMaestroClientFromProvisionShard creates a Maestro client for the given provision shard. +// The client is scoped to the Maestro Consumer associated to the provision shard, as well +// as to the the Maestro Source ID associated to the provision shard which is calculated from the provision shard ID and the +// environment specified in c.maestroSourceEnvironmentIdentifier. +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) createMaestroClientFromProvisionShard( + ctx context.Context, provisionShard *arohcpv1alpha1.ProvisionShard, +) (maestro.Client, error) { + provisionShardMaestroConsumerName := provisionShard.MaestroConfig().ConsumerName() + provisionShardMaestroRESTAPIEndpoint := provisionShard.MaestroConfig().RestApiConfig().Url() + provisionShardMaestroGRPCAPIEndpoint := provisionShard.MaestroConfig().GrpcApiConfig().Url() + // This allows us to be able to have visibility on the Maestro Bundles owned by the same source ID for a given + // provision shard and environment. This should have the same source ID as what CS has in each corresponding environment + // because otherwise we would not have visibility on the Maestro Bundles owned + // TODO do we want to use the same source ID that CS uses or do we want intentionally a different one? This has consequences + // on the visibility of the Maestro Bundles, including processing of events sent by Maestro. + maestroSourceID := maestro.GenerateMaestroSourceID(c.maestroSourceEnvironmentIdentifier, provisionShard.ID()) + + maestroClient, err := c.maestroClientBuilder.NewClient(ctx, provisionShardMaestroRESTAPIEndpoint, provisionShardMaestroGRPCAPIEndpoint, provisionShardMaestroConsumerName, maestroSourceID) + + return maestroClient, err +} diff --git a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go index e7e8e60b858..a2b5db8b513 100644 --- a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go @@ -73,21 +73,21 @@ func NewDeleteOrphanedMaestroReadonlyBundlesController(cosmosClient database.DBC return c } -// SyncOnce current algorithm is: -// 1. List all ServiceProviderClusters (initial snapshot). -// 2. Build a map from Cluster Service provision shard ID to Maestro client (one client per registered provision shard). -// 3. Build initialShardToSPCs: map provision shard ID to the ServiceProviderClusters on that shard (from the initial list). -// This assumes the Maestro server for a provision shard uses a single Maestro source ID for resources we list; if not, -// client construction would need to change (Maestro Consumer Name + Maestro Source ID scope). -// 4. For each shard, list Maestro bundles (paginated, same label selector as today). A bundle is a delete candidate if -// it passes the readonly managed-by label filter and its name is not referenced by any SPC on that shard in initialShardToSPCs. -// Each candidate records the provision shard id and a pointer to the listed ManifestWork. -// 5. List all ServiceProviderClusters again (fresh snapshot), rebuild freshShardToSPCs the same way as (3). -// 6. For each candidate, if the bundle name is still not referenced on that shard in the fresh snapshot, delete it via Maestro +// SyncOnce algorithm: +// 1. Build a map from Cluster Service provision shard ID to Maestro client (one client per registered provision shard). +// 2. For each resource type that contains Maestro readonly bundles that we decide to include here: +// 2.1 List all documents (initial snapshot). +// 2.2 Build a map from Cluster Service provision shard ID to the documents on that shard (from the initial list). +// 2.3 For each shard, list Maestro bundles (paginated, and with a label selector that filters for the readonly managed-by label associated to +// the specifc resource type we are processing). A bundle is a delete candidate if it passes the resource-scoped readonly managed-by label +// filter and its name is not referenced by any document on that shard in the initial map. Each candidate records the provision shard +// id and a pointer to the listed ManifestWork. +// 2.4 List all documents again (fresh snapshot), rebuild the map from it in the same way as 2.2. +// 2.5 For each candidate, if the bundle name is still not referenced on that shard in the fresh snapshot, delete it via Maestro +// bundle deletion API. // -// Cross-store: The fresh SPC list and per-shard reference set (steps 5-6) prevent deleting a bundle that is already referenced -// in committed Cosmos documents by the time that snapshot is built, so a stale initial list alone does not cause accidental -// delete. +// Cross-store: The fresh Cosmos snapshots and per-shard reference sets prevent deleting a bundle that is already referenced +// in committed documents by the time that snapshot is built, so a stale initial list alone does not cause accidental delete. // IMPORTANT NOTE: This assumes that the maestro server associated to the provision shard // has resources with always the same source ID. If it turns out we cannot have this assumption this logic would not @@ -98,11 +98,6 @@ func NewDeleteOrphanedMaestroReadonlyBundlesController(cosmosClient database.DBC func (c *deleteOrphanedMaestroReadonlyBundles) SyncOnce(ctx context.Context, _ any) error { logger := utils.LoggerFromContext(ctx) logger.Info("Syncing orphaned Maestro Readonly Bundles") - initialServiceProviderClusters, err := c.getAllServiceProviderClusters(ctx) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to get all ServiceProviderClusters: %w", err)) - } - logger.Info(fmt.Sprintf("Found %d ServiceProviderClusters (initial)", len(initialServiceProviderClusters))) logger.Info("Building Maestro clients per Cluster Service provision shard") maestroClientsByShard, err := c.buildMaestroClientsByProvisionShard(ctx) @@ -113,6 +108,12 @@ func (c *deleteOrphanedMaestroReadonlyBundles) SyncOnce(ctx context.Context, _ a } logger.Info(fmt.Sprintf("Built Maestro clients for %d provision shards", len(maestroClientsByShard))) + initialServiceProviderClusters, err := c.getAllServiceProviderClusters(ctx) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get all ServiceProviderClusters: %w", err)) + } + logger.Info(fmt.Sprintf("Found %d ServiceProviderClusters (initial)", len(initialServiceProviderClusters))) + logger.Info("Mapping initial ServiceProviderClusters to provision shards") initialShardToSPCs, err := c.mapServiceProviderClustersByProvisionShard(ctx, initialServiceProviderClusters, maestroClientsByShard) if err != nil { @@ -120,14 +121,32 @@ func (c *deleteOrphanedMaestroReadonlyBundles) SyncOnce(ctx context.Context, _ a } logger.Info(fmt.Sprintf("Initial ServiceProviderClusters mapped to %d provision shards", len(initialShardToSPCs))) - logger.Info("Ensuring orphaned Maestro Readonly Bundles are deleted") - err = c.ensureOrphanedMaestroReadonlyBundlesAreDeleted(ctx, maestroClientsByShard, initialShardToSPCs) + allServiceProviderNodePools, err := c.getAllServiceProviderNodePools(ctx) if err != nil { - return utils.TrackError(fmt.Errorf("failed to ensure orphaned Maestro Bundles are deleted: %w", err)) + return utils.TrackError(fmt.Errorf("failed to get all ServiceProviderNodePools: %w", err)) + } + logger.Info(fmt.Sprintf("Found %d ServiceProviderNodePools (initial)", len(allServiceProviderNodePools))) + + logger.Info("Mapping initial ServiceProviderNodePools to provision shards") + initialShardToSPNPs, err := c.mapServiceProviderNodePoolsByProvisionShard(ctx, allServiceProviderNodePools, maestroClientsByShard) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to map ServiceProviderNodePools to provision shards: %w", err)) + } + logger.Info(fmt.Sprintf("Initial ServiceProviderNodePools mapped to %d provision shards", len(initialShardToSPNPs))) + + var syncErrors []error + logger.Info("Ensuring orphaned cluster-scoped Maestro Readonly Bundles are deleted") + if err := c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, maestroClientsByShard, initialShardToSPCs); err != nil { + syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to ensure orphaned cluster-scoped Maestro Bundles are deleted: %w", err))) } - logger.Info("End of orphaned Maestro Readonly Bundles sync") - return nil + logger.Info("Ensuring orphaned nodepool-scoped Maestro Readonly Bundles are deleted") + if err := c.ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted(ctx, maestroClientsByShard, initialShardToSPNPs); err != nil { + syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to ensure orphaned nodepool-scoped Maestro Bundles are deleted: %w", err))) + } + + logger.Info("End of orphaned Maestro Readonly Bundles sync") + return errors.Join(syncErrors...) } // getAllServiceProviderClusters returns the list of all ServiceProviderClusters in the database. @@ -164,6 +183,35 @@ func (c *deleteOrphanedMaestroReadonlyBundles) getAllServiceProviderClusters(ctx return allServiceProviderClusters, nil } +// getAllServiceProviderNodePools returns the list of all ServiceProviderNodePools in the database. +func (c *deleteOrphanedMaestroReadonlyBundles) getAllServiceProviderNodePools(ctx context.Context) ([]*api.ServiceProviderNodePool, error) { + listOptions := &database.DBClientListResourceDocsOptions{ + PageSizeHint: ptr.To(int32(500)), + } + allServiceproviderNodePools := []*api.ServiceProviderNodePool{} + for { + iterator, err := c.cosmosClient.GlobalListers().ServiceProviderNodePools().List(ctx, listOptions) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to list ServiceProviderNodePools: %w", err)) + } + for _, spnp := range iterator.Items(ctx) { + allServiceproviderNodePools = append(allServiceproviderNodePools, spnp) + } + err = iterator.GetError() + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed iterating ServiceProviderNodePools: %w", err)) + } + + continuationToken := iterator.GetContinuationToken() + if continuationToken == "" { + break + } + listOptions.ContinuationToken = &continuationToken + } + + return allServiceproviderNodePools, nil +} + // shardMaestroClient holds a Maestro API client for one Cluster Service provision shard and its teardown cancel func. type shardMaestroClient struct { maestroClient maestro.Client @@ -223,21 +271,38 @@ func (c *deleteOrphanedMaestroReadonlyBundles) mapServiceProviderClustersByProvi return res, nil } +// mapServiceProviderNodePoolsByProvisionShard groups ServiceProviderNodePools by Cluster Service provision shard ID +// of their owning cluster. Every resolved shard must exist in maestroClientsByShard (registered provision shards). +func (c *deleteOrphanedMaestroReadonlyBundles) mapServiceProviderNodePoolsByProvisionShard(ctx context.Context, spnps []*api.ServiceProviderNodePool, maestroClientsByShard map[string]*shardMaestroClient) (map[string][]*api.ServiceProviderNodePool, error) { + res := make(map[string][]*api.ServiceProviderNodePool) + for _, spnp := range spnps { + shardID, err := c.clusterProvisionShardIDForServiceProviderNodePool(ctx, spnp) + if err != nil { + return nil, err + } + if _, ok := maestroClientsByShard[shardID]; !ok { + return nil, utils.TrackError(fmt.Errorf("provision shard %s for ServiceProviderNodePool %s is not present in provision shards map", shardID, spnp.ResourceID.String())) + } + res[shardID] = append(res[shardID], spnp) + } + return res, nil +} + // orphanReadonlyBundleDeleteCandidate is a Maestro bundle listed on a provision shard that was not referenced by the -// initial SPC snapshot for that shard; delete still requires a fresh snapshot check. +// initial snapshot for that shard; delete still requires a fresh snapshot check. type orphanReadonlyBundleDeleteCandidate struct { csShardID string bundle *workv1.ManifestWork } -// ensureOrphanedMaestroReadonlyBundlesAreDeleted ensures that Maestro readonly bundles managed by the cluster-scoped +// ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted ensures that Maestro readonly bundles managed by the cluster-scoped // controller are deleted when no ServiceProviderCluster on that provision shard references them. // // 1. From initialShardToSPCs, build per-shard sets of referenced Maestro bundle names. // 2. For each shard with a Maestro client, list bundles (paginated) and add candidates when the bundle is not referenced on that shard. // 3. List all ServiceProviderClusters again (fresh), map them by shard, rebuild referenced sets. // 4. Delete each candidate that is still unreferenced on its shard in the fresh snapshot. -func (c *deleteOrphanedMaestroReadonlyBundles) ensureOrphanedMaestroReadonlyBundlesAreDeleted(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient, initialShardToSPCs map[string][]*api.ServiceProviderCluster) error { +func (c *deleteOrphanedMaestroReadonlyBundles) ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient, initialShardToSPCs map[string][]*api.ServiceProviderCluster) error { logger := utils.LoggerFromContext(ctx) var syncErrors []error @@ -252,7 +317,7 @@ func (c *deleteOrphanedMaestroReadonlyBundles) ensureOrphanedMaestroReadonlyBund shardLogger := logger.WithValues("csProvisionShardID", csShardID) ctxShard := utils.ContextWithLogger(ctx, shardLogger) initialOnShard := initialShardToSPCs[csShardID] - shardLogger.Info(fmt.Sprintf("listing Maestro bundles on cluster service provision shard %s (%d ServiceProviderClusters in initial shard map)", csShardID, len(initialOnShard))) + shardLogger.Info(fmt.Sprintf("listing cluster-scoped Maestro bundles on cluster service provision shard %s (%d ServiceProviderClusters in initial shard map)", csShardID, len(initialOnShard))) maestroClient := shardEntry.maestroClient listOptions := metav1.ListOptions{Limit: 400, Continue: "", LabelSelector: fmt.Sprintf("%s=%s", readonlyBundleManagedByK8sLabelKey, readonlyBundleManagedByK8sLabelValueClusterScoped)} for { @@ -332,6 +397,104 @@ func (c *deleteOrphanedMaestroReadonlyBundles) ensureOrphanedMaestroReadonlyBund return errors.Join(syncErrors...) } +// ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted ensures that Maestro readonly bundles managed by the +// nodepool-scoped controller are deleted when no ServiceProviderNodePool on that provision shard references them. +// It uses the same two-snapshot pattern as ensureOrphanedMaestroReadonlyBundlesAreDeleted. +func (c *deleteOrphanedMaestroReadonlyBundles) ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient, initialShardToSPNPs map[string][]*api.ServiceProviderNodePool) error { + logger := utils.LoggerFromContext(ctx) + var syncErrors []error + + referencedByShardInitial, err := referencedMaestroAPIMaestroBundleNamesByShardFromNodePools(initialShardToSPNPs) + if err != nil { + return utils.TrackError(fmt.Errorf("error building referenced Maestro API Maestro bundle names by shard from node pools (initial snapshot): %w", err)) + } + + var deleteCandidates []orphanReadonlyBundleDeleteCandidate + + for csShardID, shardEntry := range maestroClientsByShard { + shardLogger := logger.WithValues("csProvisionShardID", csShardID) + ctxShard := utils.ContextWithLogger(ctx, shardLogger) + initialOnShard := initialShardToSPNPs[csShardID] + shardLogger.Info(fmt.Sprintf("listing nodepool-scoped Maestro bundles on cluster service provision shard %s (%d ServiceProviderNodePools in initial shard map)", csShardID, len(initialOnShard))) + maestroClient := shardEntry.maestroClient + listOptions := metav1.ListOptions{Limit: 400, Continue: "", LabelSelector: fmt.Sprintf("%s=%s", readonlyBundleManagedByK8sLabelKey, readonlyBundleManagedByK8sLabelValueNodePoolScoped)} + for { + maestroBundles, err := maestroClient.List(ctxShard, listOptions) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to list nodepool-scoped Maestro Bundles for shard %s: %w", csShardID, err)) + } + for i := range maestroBundles.Items { + maestroBundle := &maestroBundles.Items[i] + // Even though Maestro should filter by the K8s label we specified we double check it here to be sure + if maestroBundle.Labels[readonlyBundleManagedByK8sLabelKey] != readonlyBundleManagedByK8sLabelValueNodePoolScoped { + continue + } + // We check if the Maestro bundle is referenced by any of the ServiceProviderNodePools on the shard in the initial snapshot. + // If it is referenced we skip it as it is not an orphan. + // The Maestro API Maestro Bundle Name should be unique within a given Maestro Consumer Name and Maestro Source ID. + if shardRefSet := referencedByShardInitial[csShardID]; shardRefSet != nil { + if _, referenced := shardRefSet[maestroBundle.Name]; referenced { + continue + } + } + deleteCandidates = append(deleteCandidates, orphanReadonlyBundleDeleteCandidate{ + csShardID: csShardID, + bundle: maestroBundle, + }) + } + continuationToken := maestroBundles.GetContinue() + if continuationToken == "" { + break + } + listOptions.Continue = continuationToken + } + } + + freshServiceProviderNodePools, err := c.getAllServiceProviderNodePools(ctx) + if err != nil { + return utils.TrackError(fmt.Errorf("error getting all ServiceProviderNodePools (fresh snapshot): %w", err)) + } + freshShardToSPNPs, err := c.mapServiceProviderNodePoolsByProvisionShard(ctx, freshServiceProviderNodePools, maestroClientsByShard) + if err != nil { + return utils.TrackError(fmt.Errorf("error mapping fresh ServiceProviderNodePools to provision shards (fresh snapshot): %w", err)) + } + referencedByShardFresh, err := referencedMaestroAPIMaestroBundleNamesByShardFromNodePools(freshShardToSPNPs) + if err != nil { + return utils.TrackError(fmt.Errorf("error building referenced Maestro API Maestro bundle names by shard from node pools (fresh snapshot): %w", err)) + } + + for _, cand := range deleteCandidates { + csShardID := cand.csShardID + candidateMaestroBundle := cand.bundle + shardEntry, ok := maestroClientsByShard[csShardID] + if !ok { + syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("no Maestro client for shard %s when deleting nodepool bundle %q", csShardID, candidateMaestroBundle.Name))) + continue + } + maestroClient := shardEntry.maestroClient + + shardLogger := utils.LoggerFromContext(ctx).WithValues("csProvisionShardID", csShardID) + ctxShard := utils.ContextWithLogger(ctx, shardLogger) + if shardRefSet := referencedByShardFresh[csShardID]; shardRefSet != nil { + if _, referenced := shardRefSet[candidateMaestroBundle.Name]; referenced { + // If the Maestro bundle is referenced by any of the ServiceProviderClusters on the shard in the fresh snapshot we skip it as it is not an orphan. + continue + } + } + + shardLogger.Info("Deleting orphaned nodepool-scoped Maestro readonly Bundle", "maestroConsumerName", candidateMaestroBundle.Namespace, "maestroAPIMaestroBundleName", candidateMaestroBundle.Name, "maestroAPIMaestroBundleID", candidateMaestroBundle.UID) + err = maestroClient.Delete(ctxShard, candidateMaestroBundle.Name, metav1.DeleteOptions{}) + if err != nil { + // Failure to delete does not end the sync process. We log the error and we continue with the processing of other Maestro bundle deletion candidates. + syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to delete nodepool-scoped Maestro Bundle: %w", err))) + } else { + shardLogger.Info("Deleted orphaned nodepool-scoped Maestro readonly Bundle", "maestroConsumerName", candidateMaestroBundle.Namespace, "maestroAPIMaestroBundleName", candidateMaestroBundle.Name, "maestroAPIMaestroBundleID", candidateMaestroBundle.UID) + } + } + + return errors.Join(syncErrors...) +} + // clusterProvisionShardIDForServiceProviderCluster returns the Cluster Service provision shard ID for the cluster that owns the SPC. func (c *deleteOrphanedMaestroReadonlyBundles) clusterProvisionShardIDForServiceProviderCluster(ctx context.Context, spc *api.ServiceProviderCluster) (string, error) { clusterResourceID := spc.ResourceID.Parent @@ -354,6 +517,32 @@ func (c *deleteOrphanedMaestroReadonlyBundles) clusterProvisionShardIDForService return clusterCSShard.ID(), nil } +// clusterProvisionShardIDForServiceProviderNodePool returns the Cluster Service provision shard ID for the cluster that owns the node pool. +func (c *deleteOrphanedMaestroReadonlyBundles) clusterProvisionShardIDForServiceProviderNodePool(ctx context.Context, spnp *api.ServiceProviderNodePool) (string, error) { + nodePoolResourceID := spnp.ResourceID.Parent + if nodePoolResourceID == nil { + return "", utils.TrackError(fmt.Errorf("ServiceProviderNodePool %s has no parent resource ID", spnp.ResourceID.String())) + } + clusterResourceID := nodePoolResourceID.Parent + if clusterResourceID == nil { + return "", utils.TrackError(fmt.Errorf("ServiceProviderNodePool %s has no grandparent cluster resource ID", spnp.ResourceID.String())) + } + cluster, err := c.cosmosClient.HCPClusters(clusterResourceID.SubscriptionID, clusterResourceID.ResourceGroupName).Get(ctx, clusterResourceID.Name) + if err != nil { + return "", utils.TrackError(fmt.Errorf("failed to get Cluster: %w", err)) + } + // TODO We get the provision shard ID from CS but at some point we should have + // the information in Cosmos and this should be changed to use that instead. + // TODO should we take into account that at some point in the future we will implement migration between management + // clusters, where a cluster could have bundles allocated to different provision shards at the same time? For now + // we assume that the cluster is associated to a single provision shard at a time. + clusterCSShard, err := c.clusterServiceClient.GetClusterProvisionShard(ctx, cluster.ServiceProviderProperties.ClusterServiceID) + if err != nil { + return "", utils.TrackError(fmt.Errorf("failed to get Cluster Provision Shard: %w", err)) + } + return clusterCSShard.ID(), nil +} + // referencedMaestroAPIMaestroBundleNamesByShard maps provision shard ID to the set of Maestro API bundle names referenced by // SPCs grouped under that shard (shard assignment is already resolved in spcsByShard). Nil list entries or empty // maestroAPIMaestroBundleName return an error so the reference set cannot silently omit in-use bundles. @@ -385,6 +574,36 @@ func referencedMaestroAPIMaestroBundleNamesByShard(spcsByShard map[string][]*api return out, nil } +// referencedMaestroAPIMaestroBundleNamesByShardFromNodePools maps provision shard ID to the set of Maestro API bundle names +// referenced by ServiceProviderNodePools on that shard. +func referencedMaestroAPIMaestroBundleNamesByShardFromNodePools(spnpsByShard map[string][]*api.ServiceProviderNodePool) (map[string]map[string]struct{}, error) { + out := make(map[string]map[string]struct{}) + + for shardID, spnps := range spnpsByShard { + // If it is the first time we are processing this shard we initialize the map entry for it + if out[shardID] == nil { + out[shardID] = make(map[string]struct{}) + } + // We iterate over the ServiceProviderNodePools on the shard and we add the Maestro API Maestro bundle names to the map. + for _, spnp := range spnps { + if spnp == nil { + return nil, utils.TrackError(fmt.Errorf("nil ServiceProviderNodePool under provision shard %s", shardID)) + } + for i, ref := range spnp.Status.MaestroReadonlyBundles { + if ref == nil { + return nil, utils.TrackError(fmt.Errorf("serviceProviderNodePool %s: MaestroReadonlyBundles[%d] is nil", spnp.ResourceID.String(), i)) + } + if ref.MaestroAPIMaestroBundleName == "" { + return nil, utils.TrackError(fmt.Errorf("serviceProviderNodePool %s: MaestroReadonlyBundles[%d] (internal name %q) has empty maestroAPIMaestroBundleName", spnp.ResourceID.String(), i, ref.Name)) + } + out[shardID][ref.MaestroAPIMaestroBundleName] = struct{}{} + } + } + } + + return out, nil +} + func (c *deleteOrphanedMaestroReadonlyBundles) Run(ctx context.Context, threadiness int) { // don't let panics crash the process defer utilruntime.HandleCrash() diff --git a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller_test.go b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller_test.go index b3923d7d3a0..9e790e18a8c 100644 --- a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller_test.go +++ b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller_test.go @@ -175,6 +175,92 @@ func TestReferencedMaestroAPIMaestroBundleNamesByShard(t *testing.T) { } } +func TestReferencedMaestroAPIMaestroBundleNamesByShardFromNodePools(t *testing.T) { + spnpRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/nodePools/np/serviceProviderNodePools/default")) + + tests := []struct { + name string + spnpsByShard map[string][]*api.ServiceProviderNodePool + wantErr bool + errSubstr string + wantShard string + wantBundleName string + }{ + { + name: "empty maestroAPIMaestroBundleName", + spnpsByShard: map[string][]*api.ServiceProviderNodePool{ + "shard1": { + { + ResourceID: *spnpRID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + {Name: "logical-name", MaestroAPIMaestroBundleName: ""}, + }, + }, + }, + }, + }, + wantErr: true, + errSubstr: "has empty maestroAPIMaestroBundleName", + }, + { + name: "nil ref entry", + spnpsByShard: map[string][]*api.ServiceProviderNodePool{ + "shard1": { + { + ResourceID: *spnpRID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{nil}, + }, + }, + }, + }, + wantErr: true, + errSubstr: "is nil", + }, + { + name: "nil ServiceProviderNodePool", + spnpsByShard: map[string][]*api.ServiceProviderNodePool{ + "shard1": {nil}, + }, + wantErr: true, + errSubstr: "nil ServiceProviderNodePool", + }, + { + name: "success", + spnpsByShard: map[string][]*api.ServiceProviderNodePool{ + "s": { + { + ResourceID: *spnpRID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + {MaestroAPIMaestroBundleName: "bundle-np"}, + }, + }, + }, + }, + }, + wantShard: "s", + wantBundleName: "bundle-np", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out, err := referencedMaestroAPIMaestroBundleNamesByShardFromNodePools(tt.spnpsByShard) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errSubstr) + return + } + require.NoError(t, err) + shardSet := out[tt.wantShard] + require.NotNil(t, shardSet, "expected shard %q in result", tt.wantShard) + _, ok := shardSet[tt.wantBundleName] + assert.True(t, ok, "expected bundle name %q under shard %q", tt.wantBundleName, tt.wantShard) + }) + } +} + // paginationSPCGlobalLister returns different iterators based on ContinuationToken to simulate pagination. type paginationSPCGlobalLister struct { iter1 database.DBClientIterator[api.ServiceProviderCluster] @@ -281,6 +367,8 @@ func (s *simpleIterator[T]) GetError() error { var _ database.DBClientIterator[api.ServiceProviderCluster] = &simpleIterator[api.ServiceProviderCluster]{} +var _ database.DBClientIterator[api.ServiceProviderNodePool] = &simpleIterator[api.ServiceProviderNodePool]{} + func TestDeleteOrphanedMaestroReadonlyBundles_getAllServiceProviderClusters_Pagination(t *testing.T) { ctx := context.Background() spc1ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/serviceProviderClusters/default")) @@ -345,6 +433,54 @@ func TestDeleteOrphanedMaestroReadonlyBundles_getAllServiceProviderClusters_Iter assert.Contains(t, err.Error(), "iteration error") } +func TestDeleteOrphanedMaestroReadonlyBundles_getAllServiceProviderNodePools(t *testing.T) { + ctx := context.Background() + clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/nodePools/worker/serviceProviderNodePools/default")) + + tests := []struct { + name string + setupDB func(t *testing.T, ctx context.Context, mockDB *databasetesting.MockDBClient) + wantLen int + wantFirstResourceID string + }{ + { + name: "empty DB returns no SPNPs", + setupDB: nil, + wantLen: 0, + }, + { + name: "returns SPNPs created via CRUD", + setupDB: func(t *testing.T, ctx context.Context, mockDB *databasetesting.MockDBClient) { + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + } + spnpCRUD := mockDB.ServiceProviderNodePools(clusterResourceID.SubscriptionID, clusterResourceID.ResourceGroupName, clusterResourceID.Name, "worker") + _, err := spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + }, + wantLen: 1, + wantFirstResourceID: spnpResourceID.String(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockDB := databasetesting.NewMockDBClient() + if tt.setupDB != nil { + tt.setupDB(t, ctx, mockDB) + } + c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB} + all, err := c.getAllServiceProviderNodePools(ctx) + require.NoError(t, err) + require.Len(t, all, tt.wantLen) + if tt.wantFirstResourceID != "" { + assert.Equal(t, tt.wantFirstResourceID, all[0].ResourceID.String()) + } + }) + } +} + // alwaysErrorGlobalListers is a test double that makes the returned global listers // always return an error type alwaysErrorGlobalListers struct { @@ -398,20 +534,104 @@ func (f *alwaysErrorGlobalLister[T]) List(ctx context.Context, options *database var _ database.GlobalLister[any] = (*alwaysErrorGlobalLister[any])(nil) +// emptyGlobalLister returns an empty page for global list tests. +type emptyGlobalLister[T any] struct{} + +func (e *emptyGlobalLister[T]) List(ctx context.Context, opts *database.DBClientListResourceDocsOptions) (database.DBClientIterator[T], error) { + return &simpleIterator[T]{}, nil +} + +// failOnSecondSPNPGlobalLister fails ServiceProviderNodePools.List starting on the second call (initial SyncOnce list succeeds, fresh list inside nodepool orphan pass fails). +type failOnSecondSPNPGlobalLister struct { + call int + err error +} + +func (f *failOnSecondSPNPGlobalLister) List(ctx context.Context, opts *database.DBClientListResourceDocsOptions) (database.DBClientIterator[api.ServiceProviderNodePool], error) { + f.call++ + if f.call >= 2 { + return nil, f.err + } + return &simpleIterator[api.ServiceProviderNodePool]{}, nil +} + +// syncOnceSPCOKFailSecondSPNPGlobalListers lists empty SPCs always, and fails the second global SPNP list (for SyncOnce integration). +type syncOnceSPCOKFailSecondSPNPGlobalListers struct { + spnp *failOnSecondSPNPGlobalLister +} + +func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) Subscriptions() database.GlobalLister[arm.Subscription] { + return &panicGlobalLister[arm.Subscription]{} +} +func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) Clusters() database.GlobalLister[api.HCPOpenShiftCluster] { + return &panicGlobalLister[api.HCPOpenShiftCluster]{} +} +func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) NodePools() database.GlobalLister[api.HCPOpenShiftClusterNodePool] { + return &panicGlobalLister[api.HCPOpenShiftClusterNodePool]{} +} +func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) ExternalAuths() database.GlobalLister[api.HCPOpenShiftClusterExternalAuth] { + return &panicGlobalLister[api.HCPOpenShiftClusterExternalAuth]{} +} +func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) ServiceProviderClusters() database.GlobalLister[api.ServiceProviderCluster] { + return &emptyGlobalLister[api.ServiceProviderCluster]{} +} +func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) ServiceProviderNodePools() database.GlobalLister[api.ServiceProviderNodePool] { + return g.spnp +} +func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) Controllers() database.GlobalLister[api.Controller] { + return &panicGlobalLister[api.Controller]{} +} +func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) ManagementClusterContents() database.GlobalLister[api.ManagementClusterContent] { + return &panicGlobalLister[api.ManagementClusterContent]{} +} +func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) Operations() database.GlobalLister[api.Operation] { + return &panicGlobalLister[api.Operation]{} +} +func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) ActiveOperations() database.GlobalLister[api.Operation] { + return &panicGlobalLister[api.Operation]{} +} +func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) BillingDocs() database.GlobalLister[database.BillingDocument] { + return &panicGlobalLister[database.BillingDocument]{} +} + +var _ database.GlobalListers = (*syncOnceSPCOKFailSecondSPNPGlobalListers)(nil) + func TestDeleteOrphanedMaestroReadonlyBundles_SyncOnce_ListServiceProviderClustersError(t *testing.T) { + ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) + ctrl := gomock.NewController(t) mockDB := databasetesting.NewMockDBClient() listErr := fmt.Errorf("list SPCs error") mockDB.SetGlobalListers(&alwaysErrorGlobalListers{err: listErr}) - c := &deleteOrphanedMaestroReadonlyBundles{ - cosmosClient: mockDB, - } - err := c.SyncOnce(context.Background(), nil) + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + mockCS.EXPECT().ListProvisionShards().Return(ocm.NewSimpleProvisionShardListIterator(nil, nil)) + + c := NewDeleteOrphanedMaestroReadonlyBundlesController(mockDB, mockCS, nil, "test-env") + err := c.SyncOnce(ctx, nil) require.Error(t, err) assert.Contains(t, err.Error(), "failed to get all ServiceProviderClusters") assert.Contains(t, err.Error(), "list SPCs error") } +func TestDeleteOrphanedMaestroReadonlyBundles_SyncOnce_ListServiceProviderNodePoolsError(t *testing.T) { + ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) + ctrl := gomock.NewController(t) + mockDB := databasetesting.NewMockDBClient() + listErr := fmt.Errorf("list SPNPs error") + mockDB.SetGlobalListers(&syncOnceSPCOKFailSecondSPNPGlobalListers{ + spnp: &failOnSecondSPNPGlobalLister{err: listErr}, + }) + + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + mockCS.EXPECT().ListProvisionShards().Return(ocm.NewSimpleProvisionShardListIterator(nil, nil)) + + c := NewDeleteOrphanedMaestroReadonlyBundlesController(mockDB, mockCS, nil, "test-env") + err := c.SyncOnce(ctx, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "error getting all ServiceProviderNodePools (fresh snapshot)") + assert.Contains(t, err.Error(), "list SPNPs error") +} + func TestDeleteOrphanedMaestroReadonlyBundles_SyncOnce_NoServiceProviderClusters_Success(t *testing.T) { ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) ctrl := gomock.NewController(t) @@ -762,7 +982,46 @@ func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderClustersByProvis } } -func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundlesAreDeleted(t *testing.T) { +func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderNodePoolsByProvisionShard(t *testing.T) { + ctx := context.Background() + noopMaestroShardClient := &shardMaestroClient{ + maestroClient: nil, + maestroClientCancelFunc: func() {}, + } + clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/nodePools/worker/serviceProviderNodePools/default")) + + t.Run("success single shard", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + cluster := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterResourceID}}, + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid")), + }, + } + _, err := mockDB.HCPClusters("sub", "rg").Create(ctx, cluster, nil) + require.NoError(t, err) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + } + provisionShard := buildTestProvisionShard("test-consumer") + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster.ServiceProviderProperties.ClusterServiceID).Return(provisionShard, nil) + clients := map[string]*shardMaestroClient{provisionShard.ID(): noopMaestroShardClient} + c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} + defer cancelMaestroClientsByProvisionShard(clients) + shardToSPNPs, err := c.mapServiceProviderNodePoolsByProvisionShard(ctx, []*api.ServiceProviderNodePool{spnp}, clients) + require.NoError(t, err) + require.Len(t, shardToSPNPs, 1) + spnps := shardToSPNPs[provisionShard.ID()] + require.Len(t, spnps, 1) + assert.Equal(t, spnpResourceID.String(), spnps[0].ResourceID.String()) + }) +} + +func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(t *testing.T) { ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) spcResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/serviceProviderClusters/default")) @@ -975,7 +1234,168 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl shard := buildTestProvisionShard("test-consumer") clients, initialShardToSPCs := tt.setupMock(t, mockMaestro, mockDB, mockCS, shard) c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} - err := c.ensureOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPCs) + err := c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPCs) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errSubstr) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted(t *testing.T) { + ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/nodePools/worker/serviceProviderNodePools/default")) + + tests := []struct { + name string + setupMock func(*testing.T, *maestro.MockClient, *databasetesting.MockDBClient, *ocm.MockClusterServiceClientSpec, *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderNodePool) + wantErr bool + errSubstr string + }{ + { + name: "empty maestro clients map does not perform anything", + setupMock: func(t *testing.T, _ *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, _ *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderNodePool) { + return nil, nil + }, + wantErr: false, + }, + { + name: "second global ServiceProviderNodePools list error", + setupMock: func(_ *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderNodePool) { + listErr := fmt.Errorf("fresh list SPNPs error") + mockDB.SetGlobalListers(&alwaysErrorGlobalListers{err: listErr}) + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{}, nil) + return map[string]*shardMaestroClient{ + shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, + }, map[string][]*api.ServiceProviderNodePool{} + }, + wantErr: true, + errSubstr: "error getting all ServiceProviderNodePools (fresh snapshot)", + }, + { + name: "an error is returned when listing Maestro bundles fails", + setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderNodePool) { + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("maestro list error")) + return map[string]*shardMaestroClient{ + shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, + }, map[string][]*api.ServiceProviderNodePool{} + }, + wantErr: true, + errSubstr: "failed to list nodepool-scoped Maestro Bundles", + }, + { + name: "skips bundle without nodepool readonly managed-by label", + setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderNodePool) { + bundleList := &workv1.ManifestWorkList{ + Items: []workv1.ManifestWork{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "other-bundle", + Namespace: "consumer", + Labels: map[string]string{readonlyBundleManagedByK8sLabelKey: readonlyBundleManagedByK8sLabelValueClusterScoped}, + }, + }, + }, + } + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) + return map[string]*shardMaestroClient{ + shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, + }, map[string][]*api.ServiceProviderNodePool{} + }, + }, + { + name: "skips Maestro bundle referenced by a ServiceProviderNodePool on the shard", + setupMock: func(t *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, mockCS *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderNodePool) { + clusterRID := spnpResourceID.Parent.Parent + cluster := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterRID}}, + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid")), + }, + } + _, err := mockDB.HCPClusters(clusterRID.SubscriptionID, clusterRID.ResourceGroupName).Create(ctx, cluster, nil) + require.NoError(t, err) + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster.ServiceProviderProperties.ClusterServiceID).Return(shard, nil).AnyTimes() + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + {MaestroAPIMaestroBundleName: "referenced-np-bundle"}, + }, + }, + } + _, err = mockDB.ServiceProviderNodePools(clusterRID.SubscriptionID, clusterRID.ResourceGroupName, clusterRID.Name, spnpResourceID.Parent.Name).Create(context.Background(), spnp, nil) + require.NoError(t, err) + bundleList := &workv1.ManifestWorkList{ + Items: []workv1.ManifestWork{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "referenced-np-bundle", + Namespace: "consumer", + Labels: map[string]string{readonlyBundleManagedByK8sLabelKey: readonlyBundleManagedByK8sLabelValueNodePoolScoped}, + }, + }, + }, + } + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) + return map[string]*shardMaestroClient{ + shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, + }, map[string][]*api.ServiceProviderNodePool{shard.ID(): {spnp}} + }, + }, + { + name: "deletes Maestro bundle not referenced by any ServiceProviderNodePool on the shard", + setupMock: func(t *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, mockCS *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderNodePool) { + clusterRID := spnpResourceID.Parent.Parent + cluster := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterRID}}, + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid")), + }, + } + _, err := mockDB.HCPClusters(clusterRID.SubscriptionID, clusterRID.ResourceGroupName).Create(ctx, cluster, nil) + require.NoError(t, err) + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster.ServiceProviderProperties.ClusterServiceID).Return(shard, nil).AnyTimes() + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + {MaestroAPIMaestroBundleName: "kept-np-bundle"}, + }, + }, + } + _, err = mockDB.ServiceProviderNodePools(clusterRID.SubscriptionID, clusterRID.ResourceGroupName, clusterRID.Name, spnpResourceID.Parent.Name).Create(context.Background(), spnp, nil) + require.NoError(t, err) + orphanMeta := metav1.ObjectMeta{ + Name: "orphaned-np-bundle", + Namespace: "consumer", + UID: types.UID("orphan-np-uid"), + Labels: map[string]string{readonlyBundleManagedByK8sLabelKey: readonlyBundleManagedByK8sLabelValueNodePoolScoped}, + } + bundleList := &workv1.ManifestWorkList{Items: []workv1.ManifestWork{{ObjectMeta: orphanMeta}}} + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) + m.EXPECT().Delete(gomock.Any(), "orphaned-np-bundle", metav1.DeleteOptions{}).Return(nil) + return map[string]*shardMaestroClient{ + shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, + }, map[string][]*api.ServiceProviderNodePool{shard.ID(): {spnp}} + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + shard := buildTestProvisionShard("test-consumer") + clients, initialShardToSPNPs := tt.setupMock(t, mockMaestro, mockDB, mockCS, shard) + c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} + err := c.ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPNPs) if tt.wantErr { require.Error(t, err) assert.Contains(t, err.Error(), tt.errSubstr) @@ -986,10 +1406,10 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl } } -// TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundlesAreDeleted_shardScopedDeletes +// TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted_shardScopedDeletes // verifies that when processing a Maestro bundle in shard A, if it doesn't have a corresponding SPC associated to shard A // it is deleted, even if there's a SPC associated to other shards that contains the same maestro bundle name (per-shard reference scope). -func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundlesAreDeleted_shardScopedDeletes(t *testing.T) { +func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted_clusterScopedDeletes(t *testing.T) { ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) ctrl := gomock.NewController(t) mockShard1 := maestro.NewMockClient(ctrl) @@ -1078,14 +1498,14 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl mockShard2.EXPECT().Delete(gomock.Any(), "bundle-X", metav1.DeleteOptions{}).Return(nil) c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} - err = c.ensureOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPCs) + err = c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPCs) require.NoError(t, err) } -// TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundlesAreDeleted_bundleOnlyOnShardANoDeleteOnShardB +// TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted_bundleOnlyOnShardANoDeleteOnShardB // verifies that bundle N exists only on shard A's Maestro and is referenced by an SPC on shard A, while shard B's // Maestro lists no such bundle: processing shard B never issues a Delete for N (and shard A skips N as referenced). -func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundlesAreDeleted_bundleOnlyOnShardANoDeleteOnShardB(t *testing.T) { +func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted_bundleOnlyOnShardANoDeleteOnShardB(t *testing.T) { ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) ctrl := gomock.NewController(t) mockShardA := maestro.NewMockClient(ctrl) @@ -1170,14 +1590,14 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl mockShardB.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{Items: []workv1.ManifestWork{}}, nil) c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} - err = c.ensureOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPCs) + err = c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPCs) require.NoError(t, err) } -// TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundlesAreDeleted_ReferenceOnlyOnFreshGlobalList +// TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted_ReferenceOnlyOnFreshGlobalList // verifies that an SPC document present only on the second global Cosmos list (not in the cached SPC snapshot) // still prevents deletion of the referenced bundle. -func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundlesAreDeleted_ReferenceOnlyOnFreshGlobalList(t *testing.T) { +func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted_ReferenceOnlyOnFreshGlobalList(t *testing.T) { ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) ctrl := gomock.NewController(t) mockMaestro := maestro.NewMockClient(ctrl) @@ -1233,7 +1653,7 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl mockMaestro.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} - err = c.ensureOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPCs) + err = c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPCs) require.NoError(t, err) } @@ -1299,6 +1719,8 @@ func TestDeleteOrphanedMaestroReadonlyBundles_SyncOnce_FullFlow_DeletesOrphanedB } mockMaestro.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) mockMaestro.EXPECT().Delete(gomock.Any(), "orphaned-bundle", metav1.DeleteOptions{}).Return(nil) + // Second Maestro list: nodepool-scoped bundles pass after cluster-scoped orphan cleanup. + mockMaestro.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{}, nil) controller := NewDeleteOrphanedMaestroReadonlyBundlesController(mockDB, mockCS, mockMaestroBuilder, "test-env") err = controller.SyncOnce(ctx, nil) diff --git a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go new file mode 100644 index 00000000000..5fc08010820 --- /dev/null +++ b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go @@ -0,0 +1,370 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package controllers + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "time" + + workv1 "open-cluster-management.io/api/work/v1" + + "k8s.io/apimachinery/pkg/api/equality" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + + arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" + + "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" + "github.com/Azure/ARO-HCP/backend/pkg/informers" + "github.com/Azure/ARO-HCP/backend/pkg/listers" + "github.com/Azure/ARO-HCP/backend/pkg/maestro" + "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/database" + "github.com/Azure/ARO-HCP/internal/ocm" + "github.com/Azure/ARO-HCP/internal/utils" +) + +// readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer is a controller that reads the Maestro readonly bundles +// references stored in the ServiceProviderCluster resource, retrieves the Maestro readonly bundles using those +// references, extracts the content of the Maestro readonly bundles and persists them in Cosmos. +// It is not responsible for creating the Maestro readonly bundles themselves. That is the responsibility of +// the createMaestroReadonlyBundlesSyncer controller. +// As of now we support reading the content of the Maestro readonly bundle of the Hypershift's NodePools associated +// to the Cluster. +// This controller assumes that it has full ownership of the ManagementClusterContent resource. +type readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer struct { + cooldownChecker controllerutils.CooldownChecker + + activeOperationLister listers.ActiveOperationLister + + cosmosClient database.DBClient + + clusterServiceClient ocm.ClusterServiceClientSpec + + maestroSourceEnvironmentIdentifier string + maestroClientBuilder maestro.MaestroClientBuilder +} + +var _ controllerutils.NodePoolSyncer = (*readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer)(nil) + +func NewReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentController( + activeOperationLister listers.ActiveOperationLister, + cosmosClient database.DBClient, + clusterServiceClient ocm.ClusterServiceClientSpec, + informers informers.BackendInformers, + maestroSourceEnvironmentIdentifier string, + maestroClientBuilder maestro.MaestroClientBuilder, +) controllerutils.Controller { + + syncer := &readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer{ + cooldownChecker: controllerutils.DefaultActiveOperationPrioritizingCooldown(activeOperationLister), + cosmosClient: cosmosClient, + clusterServiceClient: clusterServiceClient, + activeOperationLister: activeOperationLister, + maestroSourceEnvironmentIdentifier: maestroSourceEnvironmentIdentifier, + maestroClientBuilder: maestroClientBuilder, + } + + controller := controllerutils.NewNodePoolWatchingController( + "ReadAndPersistNodePoolScopedMaestroReadonlyBundlesContent", + cosmosClient, + informers, + 1*time.Minute, + syncer, + ) + + return controller +} + +func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) SyncOnce(ctx context.Context, key controllerutils.HCPNodePoolKey) error { + existingNodePool, err := c.cosmosClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName).Get(ctx, key.HCPNodePoolName) + if database.IsResponseError(err, http.StatusNotFound) { + return nil // nodepool doesn't exist, no work to do + } + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get NodePool: %w", err)) + } + + existingServiceProviderNodePool, err := database.GetOrCreateServiceProviderNodePool(ctx, c.cosmosClient, key.GetResourceID()) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get or create ServiceProviderNodePool: %w", err)) + } + + // We return early if there are no Maestro Bundle references to process. + if len(existingServiceProviderNodePool.Status.MaestroReadonlyBundles) == 0 { + return nil + } + + // We get the provision shard (management cluster) the CS cluster is allocated to. + // As of now in CS the shard allocation occurs synchronously during aro-hcp cluster creation call in CS API so + // we are guaranteed to have a shard allocated for the cluster. If this changes in the future + // we would need to change the logic in controllers to check that the retrieved cluster has a + // shard allocated. + clusterProvisionShard, err := c.clusterServiceClient.GetClusterProvisionShard(ctx, existingNodePool.ServiceProviderProperties.ClusterServiceID) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get Cluster Provision Shard from Cluster Service: %w", err)) + } + maestroClient, err := c.createMaestroClientFromProvisionShard(ctx, clusterProvisionShard) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) + } + // We create a new context with a cancel function so we can cancel the Maestro client when the sync is done. + // This is important to avoid leaking resources when the sync is done. + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var syncErrors []error + for _, maestroBundleReference := range existingServiceProviderNodePool.Status.MaestroReadonlyBundles { + err = c.readAndPersistMaestroBundleContent(ctx, existingNodePool, maestroBundleReference, maestroClient) + if err != nil { + syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to read and persist NodePool: %w", err))) + } + + } + + return utils.TrackError(errors.Join(syncErrors...)) +} + +// calculateManagementClusterContentFromMaestroBundle calculates the desired ManagementClusterContent from the given Maestro Bundle reference. +// It returns the desired ManagementClusterContent or an error if the calculation fails. +func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) calculateManagementClusterContentFromMaestroBundle( + ctx context.Context, nodePool *api.HCPOpenShiftClusterNodePool, nodePoolMaestroBundleReference *api.MaestroBundleReference, + maestroClient maestro.Client, +) (*api.ManagementClusterContent, error) { + managementClusterContentResourceID := c.managementClusterContentResourceIDFromClusterResourceID(nodePool.ID.Parent, nodePoolMaestroBundleReference.Name) + desired := c.newInitialManagementClusterContent(managementClusterContentResourceID) + + existingMaestroBundle, err := maestroClient.Get(ctx, nodePoolMaestroBundleReference.MaestroAPIMaestroBundleName, metav1.GetOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return nil, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle: %w", err)) + } + if k8serrors.IsNotFound(err) { + degradedCondition := c.buildDegradedCondition(api.ConditionTrue, "MaestroBundleNotFound", err.Error()) + controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) + return desired, nil + } + + rawBytes, err := c.getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(existingMaestroBundle) + if err != nil { + degradedCondition := c.buildDegradedCondition(api.ConditionTrue, "MaestroBundleStatusFeedbackNotAvailable", err.Error()) + controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) + return desired, nil + } + + kubeContentMaxSizeExceeded := len(rawBytes) > c.kubeContentMaxSizeBytes() + var kubeContextMaxSizeExceededConditionMessage string + // We only set the retrieved content if it is within the size limit. If it + // is outside the limit we set the Degraded condition communicating the issue. + // We use unstructuredObj.Unstructured to deserialize the content so we can + // implement logic agnostic to the type of the content being retrieved. + unstructuredObj := &unstructured.Unstructured{} + err = json.Unmarshal(rawBytes, unstructuredObj) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to unmarshal object from status feedback value: %w", err)) + } + kind := unstructuredObj.GetKind() + if kind == "" { + return nil, utils.TrackError(fmt.Errorf("expected kind to be not empty")) + } + + objs, err := c.buildObjectsFromUnstructuredObj(unstructuredObj) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to build objects from unstructured object: %w", err)) + } + var degradedCondition api.Condition + if !kubeContentMaxSizeExceeded { + // TODO is ListMeta or TypeMeta required at the metav1.List level? + desired.Status.KubeContent = &metav1.List{Items: objs} + degradedCondition = c.buildDegradedCondition(api.ConditionFalse, "NoErrors", "As expected.") + } else { + kubeContextMaxSizeExceededConditionMessage = fmt.Sprintf("%s serialized size %.2f MiB exceeds Kube content max size %.2f MiB;", kind, float64(len(rawBytes))/(1024*1024), float64(c.kubeContentMaxSizeBytes())/(1024*1024)) + degradedCondition = c.buildDegradedCondition(api.ConditionTrue, "KubeContentMaxSizeExceeded", kubeContextMaxSizeExceededConditionMessage) + } + controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) + + return desired, nil +} + +// buildObjectsFromUnstructuredObj builds the list of objects from the given unstructured object. +// If the unstructured object is a list, it flattens the list of objects from the list of items. Nested lists are not flattened. +// If the unstructured object is not a list, it returns a list with a single item being the single object. +func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) buildObjectsFromUnstructuredObj(unstructuredObj *unstructured.Unstructured) ([]runtime.RawExtension, error) { + if !unstructuredObj.IsList() { + return []runtime.RawExtension{{Object: unstructuredObj}}, nil + } + + objs := []runtime.RawExtension{} + err := unstructuredObj.EachListItem(func(o runtime.Object) error { + objs = append(objs, runtime.RawExtension{Object: o}) + return nil + }) + if err != nil { + return nil, utils.TrackError(err) + } + + return objs, nil +} + +func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) buildDegradedCondition(conditionStatus api.ConditionStatus, conditionReason string, conditionMessage string) api.Condition { + return api.Condition{ + Type: "Degraded", + Status: conditionStatus, + Reason: conditionReason, + Message: conditionMessage, + } +} + +// readAndPersistMaestroBundleContent reads the Maestro Bundle content from the given Maestro Bundle reference +// and persists it in Cosmos. +// To achieve that, it gets the Maestro readonly bundle pointing to the Cluster's NodePool, it extracts the +// returned content by Maestro by taking it from the Maestro bundles's status feedback rule that contains the whole object and then it persists it +// in Cosmos. +func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) readAndPersistMaestroBundleContent( + ctx context.Context, nodePool *api.HCPOpenShiftClusterNodePool, nodePoolMaestroBundleReference *api.MaestroBundleReference, + maestroClient maestro.Client, +) error { + + desired, err := c.calculateManagementClusterContentFromMaestroBundle(ctx, nodePool, nodePoolMaestroBundleReference, maestroClient) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to calculate ManagementClusterContent from Maestro Bundle: %w", err)) + } + + managementClusterContentsDBClient := c.cosmosClient.ManagementClusterContents( + nodePool.ID.SubscriptionID, + nodePool.ID.ResourceGroupName, + nodePool.ID.Parent.Name, + ) + + existing, err := managementClusterContentsDBClient.Get(ctx, desired.CosmosMetadata.ResourceID.Name) + if err != nil && !database.IsResponseError(err, http.StatusNotFound) { + return utils.TrackError(fmt.Errorf("failed to get ManagementClusterContent: %w", err)) + } + if database.IsResponseError(err, http.StatusNotFound) { + _, err := managementClusterContentsDBClient.Create(ctx, desired, nil) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to create ManagementClusterContent: %w", err)) + } + return nil + } + + // We set the Cosmos ETag to the existing one to avoid conflicts when replacing the document + // unless someone else has modified the document since we last read it. + desired.CosmosETag = existing.CosmosETag + + // If we haven't been able to retrieve the content but there was already content + // stored we keep the previously existing stored content. + if desired.Status.KubeContent == nil && existing.Status.KubeContent != nil { + desired.Status.KubeContent = existing.Status.KubeContent + } + + if equality.Semantic.DeepEqual(existing, desired) { + return nil + } + + _, err = managementClusterContentsDBClient.Replace(ctx, desired, nil) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to replace ManagementClusterContent: %w", err)) + } + + return nil +} + +// kubeContentMaxSizeBytes returns the maximum size of a Cosmos DB item in bytes. +// 2MB is the maximum size of a Cosmos DB item (https://learn.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-item-limits). +func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) kubeContentMaxSizeBytes() int { + return 1887436 // 2MB * 0.9 +} + +// getSingleResourceStatusFeedbackRawJSONFromMaestroBundle gets the single resource status feedback raw JSON from a Maestro Bundle. +// Used to extract the content of the resource from the Maestro Bundle. +// An error is returned if the Maestro Bundle does not contain a single resource or if the resource does not contain a single status feedback value +// with its name being "resource" and its type being JsonRaw. +func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(maestroBundle *workv1.ManifestWork) (json.RawMessage, error) { + resourceStatusManifests := maestroBundle.Status.ResourceStatus.Manifests + if len(resourceStatusManifests) != 1 { + return nil, utils.TrackError(fmt.Errorf("expected exactly one resource within the Maestro Bundle, got %d", len(resourceStatusManifests))) + } + + statusFeedbackValues := resourceStatusManifests[0].StatusFeedbacks.Values + if len(statusFeedbackValues) == 0 { + return nil, utils.TrackError(fmt.Errorf("expected exactly one status feedback value within the Maestro Bundle resource, got %d", len(statusFeedbackValues))) + } + if len(statusFeedbackValues) > 1 { + return nil, utils.TrackError(fmt.Errorf("expected exactly one status feedback value within the Maestro Bundle resource, got %d", len(statusFeedbackValues))) + } + statusFeedbackValue := statusFeedbackValues[0] + if statusFeedbackValue.Name != "resource" { + return nil, utils.TrackError(fmt.Errorf("expected status feedback value name to be 'resource', got %s", statusFeedbackValue.Name)) + } + if statusFeedbackValue.Value.Type != workv1.JsonRaw { + return nil, utils.TrackError(fmt.Errorf("expected status feedback value type to be JsonRaw, got %s", statusFeedbackValue.Value.Type)) + } + if statusFeedbackValue.Value.JsonRaw == nil { + return nil, utils.TrackError(fmt.Errorf("expected status feedback value JsonRaw to be not nil")) + } + + return []byte(*statusFeedbackValue.Value.JsonRaw), nil +} + +func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) CooldownChecker() controllerutils.CooldownChecker { + return c.cooldownChecker +} + +// createMaestroClientFromProvisionShard creates a Maestro client for the given cluster provision shard. +// the client is scoped to the Consumer Name associated to the provision shard, and to +// the source ID associated to the provision shard and the environment specified +// in c.maestroSourceEnvironmentIdentifier, which is a configuration parameter at +// deployment time. +func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) createMaestroClientFromProvisionShard( + ctx context.Context, clusterProvisionShard *arohcpv1alpha1.ProvisionShard, +) (maestro.Client, error) { + provisionShardMaestroConsumerName := clusterProvisionShard.MaestroConfig().ConsumerName() + provisionShardMaestroRESTAPIEndpoint := clusterProvisionShard.MaestroConfig().RestApiConfig().Url() + provisionShardMaestroGRPCAPIEndpoint := clusterProvisionShard.MaestroConfig().GrpcApiConfig().Url() + // This allows us to be able to have visibility on the Maestro Bundles owned by the same source ID for a given + // provision shard and environment. This should have the same source ID as what CS has in each corresponding environment + // because otherwise we would not have visibility on the Maestro Bundles owned + maestroSourceID := maestro.GenerateMaestroSourceID(c.maestroSourceEnvironmentIdentifier, clusterProvisionShard.ID()) + + maestroClient, err := c.maestroClientBuilder.NewClient(ctx, provisionShardMaestroRESTAPIEndpoint, provisionShardMaestroGRPCAPIEndpoint, provisionShardMaestroConsumerName, maestroSourceID) + + return maestroClient, err +} + +// newInitialManagementClusterContent returns a new ManagementClusterContent with +// the given resource ID as its parent. The resource ID is assumed to be a +// cluster resource ID. +// The returned value can be used to consistently initialize a new ManagementClusterContent +func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) newInitialManagementClusterContent(managementClusterContentResourceID *azcorearm.ResourceID) *api.ManagementClusterContent { + return &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ + ResourceID: managementClusterContentResourceID, + }, + ResourceID: *managementClusterContentResourceID, + } +} + +// managementClusterContentResourceIDFromClusterResourceID returns the resource ID for the +// ManagementClusterContent associated to the given cluster resource ID and maestro bundle internal name. +func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) managementClusterContentResourceIDFromClusterResourceID(clusterResourceID *azcorearm.ResourceID, maestroBundleInternalName api.MaestroBundleInternalName) *azcorearm.ResourceID { + return api.Must(azcorearm.ParseResourceID(fmt.Sprintf("%s/%s/%s", clusterResourceID.String(), api.ManagementClusterContentResourceTypeName, maestroBundleInternalName))) +} diff --git a/internal/api/types_serviceprovider_nodepool.go b/internal/api/types_serviceprovider_nodepool.go index bd8756b7212..1db008e65f9 100644 --- a/internal/api/types_serviceprovider_nodepool.go +++ b/internal/api/types_serviceprovider_nodepool.go @@ -102,6 +102,12 @@ type ServiceProviderNodePoolStatus struct { // } // } NodePoolVersion ServiceProviderNodePoolStatusVersion `json:"nodePoolVersion,omitempty"` + + // MaestroReadonlyBundles contains a list of Maestro readonly bundles references. + // These bundles are used to retrieve particular K8s resources from the Management Cluster. + // The reference contains a mapping between the logical name we give to the Maestro bundle internally + // and the Maestro Bundle Name and ID at the Maestro API level. + MaestroReadonlyBundles MaestroBundleReferenceList `json:"maestroReadonlyBundles,omitempty"` } // ServiceProviderNodePoolStatusVersion contains the actual version information. @@ -111,6 +117,14 @@ type ServiceProviderNodePoolStatusVersion struct { ActiveVersions []HCPNodePoolActiveVersion `json:"activeVersions,omitempty"` } +const ( + // MaestroBundleInternalNameReadonlyHypershiftNodePool is the prefix of the internal name of the Maestro Bundle + // that represents a NodePool's Hypershift's NodePool K8s resource. The full name of the maestro internal bundle + // is constructed by concatenating the prefix with the lowercased name of the NodePool. + // Example: "readonlyHypershiftNodePool-my-nodepool" + MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix = "readonlyHypershiftNodePool-" +) + // HCPNodePoolActiveVersion represents a single version active in the nodepool. type HCPNodePoolActiveVersion struct { // Version is the full version in x.y.z format (e.g., "4.19.2") diff --git a/internal/api/zz_generated.deepcopy.go b/internal/api/zz_generated.deepcopy.go index 833b45d7a60..bf8f9266d1d 100644 --- a/internal/api/zz_generated.deepcopy.go +++ b/internal/api/zz_generated.deepcopy.go @@ -1581,6 +1581,17 @@ func (in *ServiceProviderNodePoolStatus) DeepCopyInto(out *ServiceProviderNodePo } } in.NodePoolVersion.DeepCopyInto(&out.NodePoolVersion) + if in.MaestroReadonlyBundles != nil { + in, out := &in.MaestroReadonlyBundles, &out.MaestroReadonlyBundles + *out = make(MaestroBundleReferenceList, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(MaestroBundleReference) + **out = **in + } + } + } return } From 25ce6b89e95fe9185dfa4d5ddea07f4375856a79 Mon Sep 17 00:00:00 2001 From: Miguel Soriano Date: Mon, 23 Mar 2026 10:56:51 +0100 Subject: [PATCH 02/16] refactor: move GetOrCreateMaestroBundle into maestro package --- ...ped_maestro_readonly_bundles_controller.go | 31 +---- ...aestro_readonly_bundles_controller_test.go | 102 -------------- ...ped_maestro_readonly_bundles_controller.go | 31 +---- backend/pkg/maestro/maestro_bundle.go | 55 ++++++++ backend/pkg/maestro/maestro_bundle_test.go | 130 ++++++++++++++++++ 5 files changed, 187 insertions(+), 162 deletions(-) create mode 100644 backend/pkg/maestro/maestro_bundle.go create mode 100644 backend/pkg/maestro/maestro_bundle_test.go diff --git a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go index 3a424c1bd8d..70c77eaeeca 100644 --- a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go @@ -23,7 +23,6 @@ import ( "github.com/google/uuid" workv1 "open-cluster-management.io/api/work/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -266,7 +265,7 @@ func (c *createClusterScopedMaestroReadonlyBundlesSyncer) syncMaestroBundle( return lastPersistedSPC, utils.TrackError(fmt.Errorf("unrecognized Maestro Bundle internal name: %s", maestroBundleInternalName)) } - resultMaestroBundle, err := c.getOrCreateMaestroBundle(ctx, maestroClient, desiredMaestroBundle) + resultMaestroBundle, err := maestro.GetOrCreateMaestroBundle(ctx, maestroClient, desiredMaestroBundle) if err != nil { return lastPersistedSPC, utils.TrackError(fmt.Errorf("failed to get or create Maestro Bundle: %w", err)) } @@ -452,34 +451,6 @@ func (c *createClusterScopedMaestroReadonlyBundlesSyncer) getHostedClusterNamesp return fmt.Sprintf("ocm-%s-%s", envName, csClusterID) } -// getOrCreateMaestroBundle gets (or creates if it does not exist) a Maestro Bundle for a given Maestro Bundle namespaced name. -func (c *createClusterScopedMaestroReadonlyBundlesSyncer) getOrCreateMaestroBundle(ctx context.Context, maestroClient maestro.Client, maestroBundle *workv1.ManifestWork) (*workv1.ManifestWork, error) { - logger := utils.LoggerFromContext(ctx) - existingMaestroBundle, err := maestroClient.Get(ctx, maestroBundle.Name, metav1.GetOptions{}) - if err == nil { - logger.Info(fmt.Sprintf("retrieved maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) - return existingMaestroBundle, nil - } - if !k8serrors.IsNotFound(err) { - logger.Error(err, "failed to get Maestro Bundle and it is not already exists error") - return nil, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle: %w", err)) - } - - logger.Info(fmt.Sprintf("attempting to create maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) - existingMaestroBundle, err = maestroClient.Create(ctx, maestroBundle, metav1.CreateOptions{}) - if err == nil { - logger.Info(fmt.Sprintf("created maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) - return existingMaestroBundle, nil - } - if !k8serrors.IsAlreadyExists(err) { - logger.Error(err, "failed to create Maestro Bundle and it is not already exists error") - return nil, utils.TrackError(fmt.Errorf("failed to create Maestro Bundle: %w", err)) - } - logger.Error(err, "failed to create Maestro Bundle because it returned already exists error. Attempting to get it again") - existingMaestroBundle, err = maestroClient.Get(ctx, maestroBundle.Name, metav1.GetOptions{}) - return existingMaestroBundle, err -} - func (c *createClusterScopedMaestroReadonlyBundlesSyncer) CooldownChecker() controllerutils.CooldownChecker { return c.cooldownChecker } diff --git a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller_test.go b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller_test.go index 932279ae082..3601f266d1d 100644 --- a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller_test.go +++ b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller_test.go @@ -166,108 +166,6 @@ func TestBuildInitialReadonlyMaestroBundleForHostedCluster(t *testing.T) { assert.Equal(t, expectedHostedCluster, bundle.Spec.Workload.Manifests[0].Object) } -func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_getOrCreateMaestroBundle(t *testing.T) { - desiredBundle := &workv1.ManifestWork{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-maestro-api-maestro-bundle-name", - Namespace: "test-maestro-consumer", - }, - Spec: workv1.ManifestWorkSpec{ - ManifestConfigs: []workv1.ManifestConfigOption{ - { - ResourceIdentifier: workv1.ResourceIdentifier{ - Name: "hostedcluster-name", - Namespace: "ocm-testenv-11111111111111111111111111111111", - }, - }, - }, - }, - } - - tests := []struct { - name string - setupMock func(*maestro.MockClient, *workv1.ManifestWork) - wantBundle *workv1.ManifestWork - wantErr bool - errSubstr string - }{ - { - name: "returns existing bundle if it already exists", - setupMock: func(m *maestro.MockClient, want *workv1.ManifestWork) { - m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(want, nil) - }, - wantBundle: &workv1.ManifestWork{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-maestro-api-maestro-bundle-name", Namespace: "test-maestro-consumer", UID: "existing-uid", - }, - }, - }, - { - name: "creates new bundle if it does not exist", - setupMock: func(m *maestro.MockClient, want *workv1.ManifestWork) { - m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "not-found")) - m.EXPECT().Create(gomock.Any(), desiredBundle, gomock.Any()).Return(want, nil) - }, - wantBundle: &workv1.ManifestWork{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-maestro-api-maestro-bundle-name", Namespace: "test-maestro-consumer", UID: "new-uid", - }, - }, - }, - { - name: "returns existing bundle when internal call to create returns AlreadyExists and then the following get succeeds", - setupMock: func(m *maestro.MockClient, want *workv1.ManifestWork) { - m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "not-found")) - m.EXPECT().Create(gomock.Any(), desiredBundle, gomock.Any()).Return(nil, k8serrors.NewAlreadyExists(schema.GroupResource{}, "test-maestro-api-maestro-bundle-name")) - m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(want, nil) - }, - wantBundle: &workv1.ManifestWork{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-maestro-api-maestro-bundle-name", Namespace: "test-maestro-consumer", UID: "existing-uid", - }, - }, - }, - { - name: "returns error if it fails to get the bundle", - setupMock: func(m *maestro.MockClient, _ *workv1.ManifestWork) { - m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(nil, fmt.Errorf("connection error")) - }, - wantErr: true, - errSubstr: "failed to get Maestro Bundle", - }, - { - name: "returns error if it fails to create the bundle", - setupMock: func(m *maestro.MockClient, _ *workv1.ManifestWork) { - m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "test-maestro-api-maestro-bundle-name")) - m.EXPECT().Create(gomock.Any(), desiredBundle, gomock.Any()).Return(nil, fmt.Errorf("maestro API error")) - }, - wantErr: true, - errSubstr: "failed to create Maestro Bundle: maestro API error", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - tt.setupMock(mockMaestro, tt.wantBundle) - syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - uuidV4Generator: uuid.NewRandom, - } - - result, err := syncer.getOrCreateMaestroBundle(context.Background(), mockMaestro, desiredBundle) - - if tt.wantErr { - require.Error(t, err) - assert.Nil(t, result) - assert.Contains(t, err.Error(), tt.errSubstr) - } else { - require.NoError(t, err) - assert.Equal(t, tt.wantBundle, result) - } - }) - } -} func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_syncMaestroBundle(t *testing.T) { // syncMaestroBundleTestDeterministicUUID is the fixed UUID used when testing "no bundle reference initially" so returned values are deterministic. syncMaestroBundleTestDeterministicUUID := uuid.MustParse("aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee") diff --git a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go index 67b43e83f4e..d67c685e47e 100644 --- a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go @@ -24,7 +24,6 @@ import ( "github.com/google/uuid" workv1 "open-cluster-management.io/api/work/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -274,7 +273,7 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) syncMaestroBundle( return lastPersistedSPNP, utils.TrackError(fmt.Errorf("unrecognized Maestro Bundle internal name: %s", maestroBundleInternalName)) } - resultMaestroBundle, err := c.getOrCreateMaestroBundle(ctx, maestroClient, desiredMaestroBundle) + resultMaestroBundle, err := maestro.GetOrCreateMaestroBundle(ctx, maestroClient, desiredMaestroBundle) if err != nil { return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to get or create Maestro Bundle: %w", err)) } @@ -460,34 +459,6 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) getNodePoolNamespace( return fmt.Sprintf("ocm-%s-%s-%s", envName, csClusterID, csNodePoolID) } -// getOrCreateMaestroBundle gets (or creates if it does not exist) a Maestro Bundle for a given Maestro Bundle namespaced name. -func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) getOrCreateMaestroBundle(ctx context.Context, maestroClient maestro.Client, maestroBundle *workv1.ManifestWork) (*workv1.ManifestWork, error) { - logger := utils.LoggerFromContext(ctx) - existingMaestroBundle, err := maestroClient.Get(ctx, maestroBundle.Name, metav1.GetOptions{}) - if err == nil { - logger.Info(fmt.Sprintf("retrieved maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) - return existingMaestroBundle, nil - } - if !k8serrors.IsNotFound(err) { - logger.Error(err, "failed to get Maestro Bundle and it is not already exists error") - return nil, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle: %w", err)) - } - - logger.Info(fmt.Sprintf("attempting to create maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) - existingMaestroBundle, err = maestroClient.Create(ctx, maestroBundle, metav1.CreateOptions{}) - if err == nil { - logger.Info(fmt.Sprintf("created maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) - return existingMaestroBundle, nil - } - if !k8serrors.IsAlreadyExists(err) { - logger.Error(err, "failed to create Maestro Bundle and it is not already exists error") - return nil, utils.TrackError(fmt.Errorf("failed to create Maestro Bundle: %w", err)) - } - logger.Error(err, "failed to create Maestro Bundle because it returned already exists error. Attempting to get it again") - existingMaestroBundle, err = maestroClient.Get(ctx, maestroBundle.Name, metav1.GetOptions{}) - return existingMaestroBundle, err -} - func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) CooldownChecker() controllerutils.CooldownChecker { return c.cooldownChecker } diff --git a/backend/pkg/maestro/maestro_bundle.go b/backend/pkg/maestro/maestro_bundle.go new file mode 100644 index 00000000000..4536e34c5fa --- /dev/null +++ b/backend/pkg/maestro/maestro_bundle.go @@ -0,0 +1,55 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maestro + +import ( + "context" + "fmt" + + workv1 "open-cluster-management.io/api/work/v1" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/Azure/ARO-HCP/internal/utils" +) + +// getOrCreateMaestroBundle gets (or creates if it does not exist) a Maestro Bundle for a given Maestro Bundle namespaced name. +func GetOrCreateMaestroBundle(ctx context.Context, maestroClient Client, maestroBundle *workv1.ManifestWork) (*workv1.ManifestWork, error) { + logger := utils.LoggerFromContext(ctx) + existingMaestroBundle, err := maestroClient.Get(ctx, maestroBundle.Name, metav1.GetOptions{}) + if err == nil { + logger.Info(fmt.Sprintf("retrieved maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) + return existingMaestroBundle, nil + } + if !k8serrors.IsNotFound(err) { + logger.Error(err, "failed to get Maestro Bundle and it is not already exists error") + return nil, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle: %w", err)) + } + + logger.Info(fmt.Sprintf("attempting to create maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) + existingMaestroBundle, err = maestroClient.Create(ctx, maestroBundle, metav1.CreateOptions{}) + if err == nil { + logger.Info(fmt.Sprintf("created maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) + return existingMaestroBundle, nil + } + if !k8serrors.IsAlreadyExists(err) { + logger.Error(err, "failed to create Maestro Bundle and it is not already exists error") + return nil, utils.TrackError(fmt.Errorf("failed to create Maestro Bundle: %w", err)) + } + logger.Error(err, "failed to create Maestro Bundle because it returned already exists error. Attempting to get it again") + existingMaestroBundle, err = maestroClient.Get(ctx, maestroBundle.Name, metav1.GetOptions{}) + return existingMaestroBundle, err +} diff --git a/backend/pkg/maestro/maestro_bundle_test.go b/backend/pkg/maestro/maestro_bundle_test.go new file mode 100644 index 00000000000..7397e9d4da2 --- /dev/null +++ b/backend/pkg/maestro/maestro_bundle_test.go @@ -0,0 +1,130 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maestro + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + gomock "go.uber.org/mock/gomock" + workv1 "open-cluster-management.io/api/work/v1" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func TestGetOrCreateMaestroBundle(t *testing.T) { + desiredBundle := &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-maestro-api-maestro-bundle-name", + Namespace: "test-maestro-consumer", + }, + Spec: workv1.ManifestWorkSpec{ + ManifestConfigs: []workv1.ManifestConfigOption{ + { + ResourceIdentifier: workv1.ResourceIdentifier{ + Name: "hostedcluster-name", + Namespace: "ocm-testenv-11111111111111111111111111111111", + }, + }, + }, + }, + } + + tests := []struct { + name string + setupMock func(*MockClient, *workv1.ManifestWork) + wantBundle *workv1.ManifestWork + wantErr bool + errSubstr string + }{ + { + name: "returns existing bundle if it already exists", + setupMock: func(m *MockClient, want *workv1.ManifestWork) { + m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(want, nil) + }, + wantBundle: &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-maestro-api-maestro-bundle-name", Namespace: "test-maestro-consumer", UID: "existing-uid", + }, + }, + }, + { + name: "creates new bundle if it does not exist", + setupMock: func(m *MockClient, want *workv1.ManifestWork) { + m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "not-found")) + m.EXPECT().Create(gomock.Any(), desiredBundle, gomock.Any()).Return(want, nil) + }, + wantBundle: &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-maestro-api-maestro-bundle-name", Namespace: "test-maestro-consumer", UID: "new-uid", + }, + }, + }, + { + name: "returns existing bundle when internal call to create returns AlreadyExists and then the following get succeeds", + setupMock: func(m *MockClient, want *workv1.ManifestWork) { + m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "not-found")) + m.EXPECT().Create(gomock.Any(), desiredBundle, gomock.Any()).Return(nil, k8serrors.NewAlreadyExists(schema.GroupResource{}, "test-maestro-api-maestro-bundle-name")) + m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(want, nil) + }, + wantBundle: &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-maestro-api-maestro-bundle-name", Namespace: "test-maestro-consumer", UID: "existing-uid", + }, + }, + }, + { + name: "returns error if it fails to get the bundle", + setupMock: func(m *MockClient, _ *workv1.ManifestWork) { + m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(nil, fmt.Errorf("connection error")) + }, + wantErr: true, + errSubstr: "failed to get Maestro Bundle", + }, + { + name: "returns error if it fails to create the bundle", + setupMock: func(m *MockClient, _ *workv1.ManifestWork) { + m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "test-maestro-api-maestro-bundle-name")) + m.EXPECT().Create(gomock.Any(), desiredBundle, gomock.Any()).Return(nil, fmt.Errorf("maestro API error")) + }, + wantErr: true, + errSubstr: "failed to create Maestro Bundle: maestro API error", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := NewMockClient(ctrl) + tt.setupMock(mockMaestro, tt.wantBundle) + + result, err := GetOrCreateMaestroBundle(context.Background(), mockMaestro, desiredBundle) + + if tt.wantErr { + require.Error(t, err) + assert.Nil(t, result) + assert.Contains(t, err.Error(), tt.errSubstr) + } else { + require.NoError(t, err) + assert.Equal(t, tt.wantBundle, result) + } + }) + } +} From ac05a2a496aa89595f9426c6267bed7a9d910092 Mon Sep 17 00:00:00 2001 From: Miguel Soriano Date: Mon, 23 Mar 2026 11:20:47 +0100 Subject: [PATCH 03/16] refactor: move maestro api maestro bundle name generator into maestro package --- backend/pkg/app/backend.go | 5 + ...ped_maestro_readonly_bundles_controller.go | 34 ++----- ...aestro_readonly_bundles_controller_test.go | 93 +++++++------------ ...ped_maestro_readonly_bundles_controller.go | 34 ++----- ...estro_api_maestro_bundle_name_generator.go | 71 ++++++++++++++ ..._api_maestro_bundle_name_generator_test.go | 45 +++++++++ 6 files changed, 175 insertions(+), 107 deletions(-) create mode 100644 backend/pkg/maestro/maestro_api_maestro_bundle_name_generator.go create mode 100644 backend/pkg/maestro/maestro_api_maestro_bundle_name_generator_test.go diff --git a/backend/pkg/app/backend.go b/backend/pkg/app/backend.go index e719b3e04e8..9bfe68a748e 100644 --- a/backend/pkg/app/backend.go +++ b/backend/pkg/app/backend.go @@ -495,9 +495,13 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, activeOperationLister, backendInformers, ) + + maestroAPIMaestroBundleNameGenerator := maestro.NewMaestroAPIMaestroBundleNameGenerator() + maestroCreateClusterScopedReadonlyBundlesController := controllers.NewCreateClusterScopedMaestroReadonlyBundlesController( activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient, backendInformers, b.options.MaestroSourceEnvironmentIdentifier, maestroClientBuilder, + maestroAPIMaestroBundleNameGenerator, ) maestroReadAndPersistClusterScopedReadonlyBundlesContentController := controllers.NewReadAndPersistClusterScopedMaestroReadonlyBundlesContentController( activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient, @@ -507,6 +511,7 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, maestroCreateNodePoolScopedReadonlyBundlesController := controllers.NewCreateNodePoolScopedMaestroReadonlyBundlesController( activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient, backendInformers, b.options.MaestroSourceEnvironmentIdentifier, maestroClientBuilder, + maestroAPIMaestroBundleNameGenerator, ) maestroReadAndPersistNodePoolScopedReadonlyBundlesContentController := controllers.NewReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentController( activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient, diff --git a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go index 70c77eaeeca..a2aece91089 100644 --- a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go @@ -20,7 +20,6 @@ import ( "net/http" "time" - "github.com/google/uuid" workv1 "open-cluster-management.io/api/work/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -68,10 +67,7 @@ type createClusterScopedMaestroReadonlyBundlesSyncer struct { maestroClientBuilder maestro.MaestroClientBuilder - // uuidV4Generator is used to generate a new UUIDv4. It must be provided. - // We define it as a dependency to enable deterministic testing in some - // scenarios. - uuidV4Generator func() (uuid.UUID, error) + maestroAPIMaestroBundleNameGenerator maestro.MaestroAPIMaestroBundleNameGenerator } var _ controllerutils.ClusterSyncer = (*createClusterScopedMaestroReadonlyBundlesSyncer)(nil) @@ -83,16 +79,17 @@ func NewCreateClusterScopedMaestroReadonlyBundlesController( informers informers.BackendInformers, maestroSourceEnvironmentIdentifier string, maestroClientBuilder maestro.MaestroClientBuilder, + maestroAPIMaestroBundleNameGenerator maestro.MaestroAPIMaestroBundleNameGenerator, ) controllerutils.Controller { syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - cooldownChecker: controllerutils.DefaultActiveOperationPrioritizingCooldown(activeOperationLister), - cosmosClient: cosmosClient, - clusterServiceClient: clusterServiceClient, - activeOperationLister: activeOperationLister, - maestroSourceEnvironmentIdentifier: maestroSourceEnvironmentIdentifier, - maestroClientBuilder: maestroClientBuilder, - uuidV4Generator: uuid.NewRandom, + cooldownChecker: controllerutils.DefaultActiveOperationPrioritizingCooldown(activeOperationLister), + cosmosClient: cosmosClient, + clusterServiceClient: clusterServiceClient, + activeOperationLister: activeOperationLister, + maestroSourceEnvironmentIdentifier: maestroSourceEnvironmentIdentifier, + maestroClientBuilder: maestroClientBuilder, + maestroAPIMaestroBundleNameGenerator: maestroAPIMaestroBundleNameGenerator, } controller := controllerutils.NewClusterWatchingController( @@ -418,7 +415,7 @@ func (c *createClusterScopedMaestroReadonlyBundlesSyncer) buildInitialReadonlyMa // buildInitialMaestroBundleReference builds an initial Maestro Bundle reference for a given maestro bundle internal name. func (c *createClusterScopedMaestroReadonlyBundlesSyncer) buildInitialMaestroBundleReference(internalName api.MaestroBundleInternalName) (*api.MaestroBundleReference, error) { - maestroAPIMaestroBundleName, err := c.generateNewMaestroAPIMaestroBundleName() + maestroAPIMaestroBundleName, err := c.maestroAPIMaestroBundleNameGenerator.NewMaestroAPIMaestroBundleName() if err != nil { return nil, utils.TrackError(fmt.Errorf("failed to generate Maestro API Maestro Bundle name: %w", err)) } @@ -431,17 +428,6 @@ func (c *createClusterScopedMaestroReadonlyBundlesSyncer) buildInitialMaestroBun return hostedClusterMWMaestroBundleReference, nil } -// generateNewMaestroAPIMaestroBundleName generates a new Maestro API Maestro Bundle name. -// Used to generate a new Maestro API Maestro Bundle name for a new Maestro Bundle reference. -// The generated name is a UUIDv4. -func (c *createClusterScopedMaestroReadonlyBundlesSyncer) generateNewMaestroAPIMaestroBundleName() (string, error) { - newUUIDForMaestroAPIMaestroBundleName, err := c.uuidV4Generator() - if err != nil { - return "", utils.TrackError(fmt.Errorf("failed to generate UUIDv4 for Maestro API Maestro Bundle name: %w", err)) - } - return newUUIDForMaestroAPIMaestroBundleName.String(), nil -} - // getHostedClusterNamespace gets the namespace for the hosted cluster based on the environment name and the cluster service OCM Cluster ID. // For example, if the CS cluster URL is /api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111 then the cluster service OCM Cluster // ID is 11111111111111111111111111111111. diff --git a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller_test.go b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller_test.go index 3601f266d1d..bb35d6f1f37 100644 --- a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller_test.go +++ b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller_test.go @@ -76,8 +76,8 @@ var _ database.ServiceProviderClusterCRUD = &errorInjectingSPCCRUDForCreate{} func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_buildClusterEmptyHostedCluster(t *testing.T) { syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - maestroSourceEnvironmentIdentifier: "testenv", - uuidV4Generator: uuid.NewRandom, + maestroSourceEnvironmentIdentifier: "testenv", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } csClusterID := "11111111111111111111111111111111" @@ -97,8 +97,8 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_getHostedClusterNamespa expected := fmt.Sprintf("ocm-%s-%s", envName, csClusterID) syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - maestroSourceEnvironmentIdentifier: envName, - uuidV4Generator: uuid.NewRandom, + maestroSourceEnvironmentIdentifier: envName, + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } result := syncer.getHostedClusterNamespace(envName, csClusterID) @@ -107,7 +107,7 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_getHostedClusterNamespa func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_buildInitialMaestroBundleReference(t *testing.T) { syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - uuidV4Generator: uuid.NewRandom, + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } ref, err := syncer.buildInitialMaestroBundleReference(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster) @@ -125,8 +125,8 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_buildInitialMaestroBund func TestBuildInitialReadonlyMaestroBundleForHostedCluster(t *testing.T) { syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - maestroSourceEnvironmentIdentifier: "testenv", - uuidV4Generator: uuid.NewRandom, + maestroSourceEnvironmentIdentifier: "testenv", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster")) @@ -401,10 +401,9 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_syncMaestroBundle(t *te mockMaestro := maestro.NewMockClient(ctrl) tt.maestroClientSetupMock(mockMaestro) - deterministicUUIDGenerator := func() (uuid.UUID, error) { return syncMaestroBundleTestDeterministicUUID, nil } syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - maestroSourceEnvironmentIdentifier: "test-env", - uuidV4Generator: deterministicUUIDGenerator, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewAlwaysSameNameMaestroAPIMaestroBundleNameGenerator(syncMaestroBundleTestDeterministicUUID.String()), } ctx := context.Background() cluster := &api.HCPOpenShiftCluster{ @@ -450,33 +449,9 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_syncMaestroBundle(t *te } } -func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_generateNewMaestroAPIMaestroBundleName(t *testing.T) { - syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - uuidV4Generator: uuid.NewRandom, - } - - // Test successful generation - name1, err := syncer.generateNewMaestroAPIMaestroBundleName() - require.NoError(t, err) - assert.NotEmpty(t, name1) - - // Verify it's a valid UUID - _, err = uuid.Parse(name1) - assert.NoError(t, err, "Generated name should be a valid UUID") - - // Test that multiple calls generate different UUIDs - name2, err := syncer.generateNewMaestroAPIMaestroBundleName() - require.NoError(t, err) - assert.NotEqual(t, name1, name2, "Multiple calls should generate different UUIDs") - - // Verify second name is also a valid UUID - _, err = uuid.Parse(name2) - assert.NoError(t, err, "Second generated name should also be a valid UUID") -} - func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_buildInitialReadonlyMaestroBundle(t *testing.T) { syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - uuidV4Generator: uuid.NewRandom, + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } maestroBundleNamespacedName := types.NamespacedName{ @@ -542,10 +517,10 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_buildInitialReadonlyMae func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_SyncOnce_ClusterNotFound(t *testing.T) { mockDBClient := databasetesting.NewMockDBClient() syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - cooldownChecker: &alwaysSyncCooldownChecker{}, - cosmosClient: mockDBClient, - maestroSourceEnvironmentIdentifier: "test-env", - uuidV4Generator: uuid.NewRandom, + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } key := controllerutils.HCPClusterKey{ @@ -595,10 +570,10 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_SyncOnce_GetServiceProv } syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - cooldownChecker: &alwaysSyncCooldownChecker{}, - cosmosClient: mockDBClient, - maestroSourceEnvironmentIdentifier: "test-env", - uuidV4Generator: uuid.NewRandom, + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } err = syncer.SyncOnce(ctx, key) @@ -610,10 +585,10 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_SyncOnce_AllBundlesAlre ctx := context.Background() mockDBClient := databasetesting.NewMockDBClient() syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - cooldownChecker: &alwaysSyncCooldownChecker{}, - cosmosClient: mockDBClient, - maestroSourceEnvironmentIdentifier: "test-env", - uuidV4Generator: uuid.NewRandom, + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } key := controllerutils.HCPClusterKey{ @@ -677,12 +652,12 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_SyncOnce_SyncLoopExecut mockMaestroClient := maestro.NewMockClient(ctrl) syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - cooldownChecker: &alwaysSyncCooldownChecker{}, - cosmosClient: mockDBClient, - clusterServiceClient: mockClusterService, - maestroClientBuilder: mockMaestroBuilder, - maestroSourceEnvironmentIdentifier: "test-env", - uuidV4Generator: uuid.NewRandom, + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + clusterServiceClient: mockClusterService, + maestroClientBuilder: mockMaestroBuilder, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } key := controllerutils.HCPClusterKey{ @@ -792,12 +767,12 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_SyncOnce_ProcessesParti mockMaestroClient := maestro.NewMockClient(ctrl) syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - cooldownChecker: &alwaysSyncCooldownChecker{}, - cosmosClient: mockDBClient, - clusterServiceClient: mockClusterService, - maestroClientBuilder: mockMaestroBuilder, - maestroSourceEnvironmentIdentifier: "test-env", - uuidV4Generator: uuid.NewRandom, + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + clusterServiceClient: mockClusterService, + maestroClientBuilder: mockMaestroBuilder, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } key := controllerutils.HCPClusterKey{ diff --git a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go index d67c685e47e..dbfd061df98 100644 --- a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go @@ -21,7 +21,6 @@ import ( "strings" "time" - "github.com/google/uuid" workv1 "open-cluster-management.io/api/work/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -67,10 +66,7 @@ type createNodePoolScopedMaestroReadonlyBundlesSyncer struct { maestroClientBuilder maestro.MaestroClientBuilder - // uuidV4Generator is used to generate a new UUIDv4. It must be provided. - // We define it as a dependency to enable deterministic testing in some - // scenarios. - uuidV4Generator func() (uuid.UUID, error) + maestroAPIMaestroBundleNameGenerator maestro.MaestroAPIMaestroBundleNameGenerator } var _ controllerutils.NodePoolSyncer = (*createNodePoolScopedMaestroReadonlyBundlesSyncer)(nil) @@ -82,16 +78,17 @@ func NewCreateNodePoolScopedMaestroReadonlyBundlesController( informers informers.BackendInformers, maestroSourceEnvironmentIdentifier string, maestroClientBuilder maestro.MaestroClientBuilder, + maestroAPIMaestroBundleNameGenerator maestro.MaestroAPIMaestroBundleNameGenerator, ) controllerutils.Controller { syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ - cooldownChecker: controllerutils.DefaultActiveOperationPrioritizingCooldown(activeOperationLister), - cosmosClient: cosmosClient, - clusterServiceClient: clusterServiceClient, - activeOperationLister: activeOperationLister, - maestroSourceEnvironmentIdentifier: maestroSourceEnvironmentIdentifier, - maestroClientBuilder: maestroClientBuilder, - uuidV4Generator: uuid.NewRandom, + cooldownChecker: controllerutils.DefaultActiveOperationPrioritizingCooldown(activeOperationLister), + cosmosClient: cosmosClient, + clusterServiceClient: clusterServiceClient, + activeOperationLister: activeOperationLister, + maestroSourceEnvironmentIdentifier: maestroSourceEnvironmentIdentifier, + maestroClientBuilder: maestroClientBuilder, + maestroAPIMaestroBundleNameGenerator: maestroAPIMaestroBundleNameGenerator, } controller := controllerutils.NewNodePoolWatchingController( @@ -426,7 +423,7 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildInitialReadonlyM // buildInitialMaestroBundleReference builds an initial Maestro Bundle reference for a given maestro bundle internal name. func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildInitialMaestroBundleReference(internalName api.MaestroBundleInternalName) (*api.MaestroBundleReference, error) { - maestroAPIMaestroBundleName, err := c.generateNewMaestroAPIMaestroBundleName() + maestroAPIMaestroBundleName, err := c.maestroAPIMaestroBundleNameGenerator.NewMaestroAPIMaestroBundleName() if err != nil { return nil, utils.TrackError(fmt.Errorf("failed to generate Maestro API Maestro Bundle name: %w", err)) } @@ -439,17 +436,6 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildInitialMaestroBu return nodePoolMaestroBundleReference, nil } -// generateNewMaestroAPIMaestroBundleName generates a new Maestro API Maestro Bundle name. -// Used to generate a new Maestro API Maestro Bundle name for a new Maestro Bundle reference. -// The generated name is a UUIDv4. -func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) generateNewMaestroAPIMaestroBundleName() (string, error) { - newUUIDForMaestroAPIMaestroBundleName, err := c.uuidV4Generator() - if err != nil { - return "", utils.TrackError(fmt.Errorf("failed to generate UUIDv4 for Maestro API Maestro Bundle name: %w", err)) - } - return newUUIDForMaestroAPIMaestroBundleName.String(), nil -} - // getNodePoolNamespace gets the namespace for the node pool based on the environment name, the cluster service OCM Cluster ID and the node pool service OCM Node Pool ID. // For example, if the Node Pool URL is /api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111/nodepools/XXXX then the // cluster service OCM Cluster ID is 11111111111111111111111111111111 and the node pool service OCM Node Pool ID is XXXX. diff --git a/backend/pkg/maestro/maestro_api_maestro_bundle_name_generator.go b/backend/pkg/maestro/maestro_api_maestro_bundle_name_generator.go new file mode 100644 index 00000000000..2d3dcc2890e --- /dev/null +++ b/backend/pkg/maestro/maestro_api_maestro_bundle_name_generator.go @@ -0,0 +1,71 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maestro + +import ( + "fmt" + + "github.com/google/uuid" + + "github.com/Azure/ARO-HCP/internal/utils" +) + +// MaestroAPIMaestroBundleNameGenerator is an interface that defines a method to generate a new Maestro API Maestro Bundle name. +// The generated name must be globally unique within a given Maestro Consumer Name and Maestro Source ID. +// It can be used to generate a new Maestro API Maestro Bundle name for a new Maestro Bundle reference. +type MaestroAPIMaestroBundleNameGenerator interface { + // NewMaestroAPIMaestroBundleName generates a new Maestro API Maestro Bundle name. + // The generated name must be globally unique within a given Maestro Consumer Name and Maestro Source ID. + NewMaestroAPIMaestroBundleName() (string, error) +} + +// NewMaestroAPIMaestroBundleNameGenerator creates a new Maestro API Maestro Bundle name generator. +// The generator generates a new Maestro API Maestro Bundle name whose value is a UUIDv4. +func NewMaestroAPIMaestroBundleNameGenerator() MaestroAPIMaestroBundleNameGenerator { + return &maestroAPIMaestroBundleNameGenerator{ + uuidV4Generator: uuid.NewRandom, + } +} + +type maestroAPIMaestroBundleNameGenerator struct { + uuidV4Generator func() (uuid.UUID, error) +} + +// generateNewMaestroAPIMaestroBundleName generates a new Maestro API Maestro Bundle name. +// Used to generate a new Maestro API Maestro Bundle name for a new Maestro Bundle reference. +// The generated name is a UUIDv4. +func (c *maestroAPIMaestroBundleNameGenerator) NewMaestroAPIMaestroBundleName() (string, error) { + newUUIDForMaestroAPIMaestroBundleName, err := c.uuidV4Generator() + if err != nil { + return "", utils.TrackError(fmt.Errorf("failed to generate UUIDv4 for Maestro API Maestro Bundle name: %w", err)) + } + return newUUIDForMaestroAPIMaestroBundleName.String(), nil +} + +type alwaysSameNameMaestroAPIMaestroBundleNameGenerator struct { + maestroAPIMaestroBundleName string +} + +// NewAlwaysSameNameMaestroAPIMaestroBundleNameGenerator creates a new Maestro API Maestro Bundle name generator that always returns the same name. +// This is useful for testing purposes. +func NewAlwaysSameNameMaestroAPIMaestroBundleNameGenerator(maestroAPIMaestroBundleName string) MaestroAPIMaestroBundleNameGenerator { + return &alwaysSameNameMaestroAPIMaestroBundleNameGenerator{ + maestroAPIMaestroBundleName: maestroAPIMaestroBundleName, + } +} + +func (c *alwaysSameNameMaestroAPIMaestroBundleNameGenerator) NewMaestroAPIMaestroBundleName() (string, error) { + return c.maestroAPIMaestroBundleName, nil +} diff --git a/backend/pkg/maestro/maestro_api_maestro_bundle_name_generator_test.go b/backend/pkg/maestro/maestro_api_maestro_bundle_name_generator_test.go new file mode 100644 index 00000000000..67ac94f2cf0 --- /dev/null +++ b/backend/pkg/maestro/maestro_api_maestro_bundle_name_generator_test.go @@ -0,0 +1,45 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maestro + +import ( + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMaestroAPIMaestroBundleNameGenerator_NewMaestroAPIMaestroBundleName(t *testing.T) { + generator := NewMaestroAPIMaestroBundleNameGenerator() + + // Test successful generation + name1, err := generator.NewMaestroAPIMaestroBundleName() + require.NoError(t, err) + assert.NotEmpty(t, name1) + + // Verify it's a valid UUID + _, err = uuid.Parse(name1) + assert.NoError(t, err, "Generated name should be a valid UUID") + + // Test that multiple calls generate different UUIDs + name2, err := generator.NewMaestroAPIMaestroBundleName() + require.NoError(t, err) + assert.NotEqual(t, name1, name2, "Multiple calls should generate different UUIDs") + + // Verify second name is also a valid UUID + _, err = uuid.Parse(name2) + assert.NoError(t, err, "Second generated name should also be a valid UUID") +} From 3848c8511022d8585a2f497709da9ff17446d1cb Mon Sep 17 00:00:00 2001 From: Miguel Soriano Date: Mon, 23 Mar 2026 11:36:52 +0100 Subject: [PATCH 04/16] refactor: move management cluster content utils into controllerutils package --- .../management_cluster_content_controller.go | 42 +++++++++++++++++++ ...agement_cluster_content_controller_test.go | 36 ++++++++++++++++ ...tro_readonly_bundles_content_controller.go | 25 +---------- ...eadonly_bundles_content_controller_test.go | 11 ----- ...tro_readonly_bundles_content_controller.go | 25 +---------- 5 files changed, 82 insertions(+), 57 deletions(-) create mode 100644 backend/pkg/controllers/controllerutils/management_cluster_content_controller.go create mode 100644 backend/pkg/controllers/controllerutils/management_cluster_content_controller_test.go diff --git a/backend/pkg/controllers/controllerutils/management_cluster_content_controller.go b/backend/pkg/controllers/controllerutils/management_cluster_content_controller.go new file mode 100644 index 00000000000..8f80e81de28 --- /dev/null +++ b/backend/pkg/controllers/controllerutils/management_cluster_content_controller.go @@ -0,0 +1,42 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllerutils + +import ( + "fmt" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + + "github.com/Azure/ARO-HCP/internal/api" +) + +// NewInitialManagementClusterContent returns a new ManagementClusterContent with +// the given resource ID as its parent. The resource ID is assumed to be a +// cluster resource ID. +// The returned value can be used to consistently initialize a new ManagementClusterContent +func NewInitialManagementClusterContent(managementClusterContentResourceID *azcorearm.ResourceID) *api.ManagementClusterContent { + return &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ + ResourceID: managementClusterContentResourceID, + }, + ResourceID: *managementClusterContentResourceID, + } +} + +// ManagementClusterContentResourceIDFromClusterResourceID returns the resource ID for the +// ManagementClusterContent associated to the given cluster resource ID and maestro bundle internal name. +func ManagementClusterContentResourceIDFromClusterResourceID(clusterResourceID *azcorearm.ResourceID, maestroBundleInternalName api.MaestroBundleInternalName) *azcorearm.ResourceID { + return api.Must(azcorearm.ParseResourceID(fmt.Sprintf("%s/%s/%s", clusterResourceID.String(), api.ManagementClusterContentResourceTypeName, maestroBundleInternalName))) +} diff --git a/backend/pkg/controllers/controllerutils/management_cluster_content_controller_test.go b/backend/pkg/controllers/controllerutils/management_cluster_content_controller_test.go new file mode 100644 index 00000000000..f5532cc597f --- /dev/null +++ b/backend/pkg/controllers/controllerutils/management_cluster_content_controller_test.go @@ -0,0 +1,36 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllerutils + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + + "github.com/Azure/ARO-HCP/internal/api" +) + +func TestManagementClusterContentResourceIDFromClusterResourceID(t *testing.T) { + clusterRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/mycluster")) + + got := ManagementClusterContentResourceIDFromClusterResourceID(clusterRID, api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster) + require.NotNil(t, got) + assert.Equal(t, got.ResourceType.Type, api.ManagementClusterContentResourceType.Type) + // Name is the last segment of the resource ID (the management cluster content name) + assert.Equal(t, got.Name, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) +} diff --git a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go index e050c014b14..7a6dea11197 100644 --- a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go +++ b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go @@ -29,8 +29,6 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" - arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" @@ -151,8 +149,8 @@ func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) calcula ctx context.Context, cluster *api.HCPOpenShiftCluster, hostedClusterMaestroBundleReference *api.MaestroBundleReference, maestroClient maestro.Client, ) (*api.ManagementClusterContent, error) { - managementClusterContentResourceID := c.managementClusterContentResourceIDFromClusterResourceID(cluster.ID, hostedClusterMaestroBundleReference.Name) - desired := c.newInitialManagementClusterContent(managementClusterContentResourceID) + managementClusterContentResourceID := controllerutils.ManagementClusterContentResourceIDFromClusterResourceID(cluster.ID, hostedClusterMaestroBundleReference.Name) + desired := controllerutils.NewInitialManagementClusterContent(managementClusterContentResourceID) existingMaestroBundle, err := maestroClient.Get(ctx, hostedClusterMaestroBundleReference.MaestroAPIMaestroBundleName, metav1.GetOptions{}) if err != nil && !k8serrors.IsNotFound(err) { @@ -350,22 +348,3 @@ func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) createM return maestroClient, err } - -// newInitialManagementClusterContent returns a new ManagementClusterContent with -// the given resource ID as its parent. The resource ID is assumed to be a -// cluster resource ID. -// The returned value can be used to consistently initialize a new ManagementClusterContent -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) newInitialManagementClusterContent(managementClusterContentResourceID *azcorearm.ResourceID) *api.ManagementClusterContent { - return &api.ManagementClusterContent{ - CosmosMetadata: api.CosmosMetadata{ - ResourceID: managementClusterContentResourceID, - }, - ResourceID: *managementClusterContentResourceID, - } -} - -// managementClusterContentResourceIDFromClusterResourceID returns the resource ID for the -// ManagementClusterContent associated to the given cluster resource ID and maestro bundle internal name. -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) managementClusterContentResourceIDFromClusterResourceID(clusterResourceID *azcorearm.ResourceID, maestroBundleInternalName api.MaestroBundleInternalName) *azcorearm.ResourceID { - return api.Must(azcorearm.ParseResourceID(fmt.Sprintf("%s/%s/%s", clusterResourceID.String(), api.ManagementClusterContentResourceTypeName, maestroBundleInternalName))) -} diff --git a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go index 408470fbd87..dfc0197e968 100644 --- a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go +++ b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go @@ -340,17 +340,6 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_getSingl } } -func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_managementClusterContentResourceIDFromClusterResourceID(t *testing.T) { - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{} - clusterRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/mycluster")) - - got := syncer.managementClusterContentResourceIDFromClusterResourceID(clusterRID, api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster) - require.NotNil(t, got) - assert.Equal(t, got.ResourceType.Type, api.ManagementClusterContentResourceType.Type) - // Name is the last segment of the resource ID (the management cluster content name) - assert.Equal(t, got.Name, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) -} - func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_calculateManagementClusterContentFromMaestroBundle(t *testing.T) { clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) cluster := &api.HCPOpenShiftCluster{ diff --git a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go index 5fc08010820..baa9f98849a 100644 --- a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go +++ b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go @@ -29,8 +29,6 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" - arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" @@ -150,8 +148,8 @@ func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) calcul ctx context.Context, nodePool *api.HCPOpenShiftClusterNodePool, nodePoolMaestroBundleReference *api.MaestroBundleReference, maestroClient maestro.Client, ) (*api.ManagementClusterContent, error) { - managementClusterContentResourceID := c.managementClusterContentResourceIDFromClusterResourceID(nodePool.ID.Parent, nodePoolMaestroBundleReference.Name) - desired := c.newInitialManagementClusterContent(managementClusterContentResourceID) + managementClusterContentResourceID := controllerutils.ManagementClusterContentResourceIDFromClusterResourceID(nodePool.ID.Parent, nodePoolMaestroBundleReference.Name) + desired := controllerutils.NewInitialManagementClusterContent(managementClusterContentResourceID) existingMaestroBundle, err := maestroClient.Get(ctx, nodePoolMaestroBundleReference.MaestroAPIMaestroBundleName, metav1.GetOptions{}) if err != nil && !k8serrors.IsNotFound(err) { @@ -349,22 +347,3 @@ func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) create return maestroClient, err } - -// newInitialManagementClusterContent returns a new ManagementClusterContent with -// the given resource ID as its parent. The resource ID is assumed to be a -// cluster resource ID. -// The returned value can be used to consistently initialize a new ManagementClusterContent -func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) newInitialManagementClusterContent(managementClusterContentResourceID *azcorearm.ResourceID) *api.ManagementClusterContent { - return &api.ManagementClusterContent{ - CosmosMetadata: api.CosmosMetadata{ - ResourceID: managementClusterContentResourceID, - }, - ResourceID: *managementClusterContentResourceID, - } -} - -// managementClusterContentResourceIDFromClusterResourceID returns the resource ID for the -// ManagementClusterContent associated to the given cluster resource ID and maestro bundle internal name. -func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) managementClusterContentResourceIDFromClusterResourceID(clusterResourceID *azcorearm.ResourceID, maestroBundleInternalName api.MaestroBundleInternalName) *azcorearm.ResourceID { - return api.Must(azcorearm.ParseResourceID(fmt.Sprintf("%s/%s/%s", clusterResourceID.String(), api.ManagementClusterContentResourceTypeName, maestroBundleInternalName))) -} From 93a4ebd1463e7e71ed76dd09b401946355057659 Mon Sep 17 00:00:00 2001 From: Miguel Soriano Date: Mon, 23 Mar 2026 11:42:40 +0100 Subject: [PATCH 05/16] refactor: move createMaestroClientFromCSProvisionShard into package level function --- ...ped_maestro_readonly_bundles_controller.go | 24 +--------- ...ped_maestro_readonly_bundles_controller.go | 24 +--------- backend/pkg/controllers/cs_maestro_utils.go | 44 +++++++++++++++++++ ...ned_maestro_readonly_bundles_controller.go | 26 +---------- ...tro_readonly_bundles_content_controller.go | 25 +---------- ...tro_readonly_bundles_content_controller.go | 25 +---------- 6 files changed, 49 insertions(+), 119 deletions(-) create mode 100644 backend/pkg/controllers/cs_maestro_utils.go diff --git a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go index a2aece91089..a7f153d17a8 100644 --- a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go @@ -169,7 +169,7 @@ func (c *createClusterScopedMaestroReadonlyBundlesSyncer) SyncOnce(ctx context.C // This is important to avoid leaking resources when the sync is done. ctx, cancel := context.WithCancel(ctx) defer cancel() - maestroClient, err := c.createMaestroClientFromProvisionShard(ctx, clusterProvisionShard) + maestroClient, err := createMaestroClientFromCSProvisionShard(ctx, c.maestroSourceEnvironmentIdentifier, c.maestroClientBuilder, clusterProvisionShard) if err != nil { return utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) } @@ -440,25 +440,3 @@ func (c *createClusterScopedMaestroReadonlyBundlesSyncer) getHostedClusterNamesp func (c *createClusterScopedMaestroReadonlyBundlesSyncer) CooldownChecker() controllerutils.CooldownChecker { return c.cooldownChecker } - -// createMaestroClientFromProvisionShard creates a Maestro client for the given provision shard. -// The client is scoped to the Maestro Consumer associated to the provision shard, as well -// as to the the Maestro Source ID associated to the provision shard which is calculated from the provision shard ID and the -// environment specified in c.maestroSourceEnvironmentIdentifier. -func (c *createClusterScopedMaestroReadonlyBundlesSyncer) createMaestroClientFromProvisionShard( - ctx context.Context, provisionShard *arohcpv1alpha1.ProvisionShard, -) (maestro.Client, error) { - provisionShardMaestroConsumerName := provisionShard.MaestroConfig().ConsumerName() - provisionShardMaestroRESTAPIEndpoint := provisionShard.MaestroConfig().RestApiConfig().Url() - provisionShardMaestroGRPCAPIEndpoint := provisionShard.MaestroConfig().GrpcApiConfig().Url() - // This allows us to be able to have visibility on the Maestro Bundles owned by the same source ID for a given - // provision shard and environment. This should have the same source ID as what CS has in each corresponding environment - // because otherwise we would not have visibility on the Maestro Bundles owned - // TODO do we want to use the same source ID that CS uses or do we want intentionally a different one? This has consequences - // on the visibility of the Maestro Bundles, including processing of events sent by Maestro. - maestroSourceID := maestro.GenerateMaestroSourceID(c.maestroSourceEnvironmentIdentifier, provisionShard.ID()) - - maestroClient, err := c.maestroClientBuilder.NewClient(ctx, provisionShardMaestroRESTAPIEndpoint, provisionShardMaestroGRPCAPIEndpoint, provisionShardMaestroConsumerName, maestroSourceID) - - return maestroClient, err -} diff --git a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go index dbfd061df98..6ac47074045 100644 --- a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go @@ -177,7 +177,7 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) SyncOnce(ctx context. // This is important to avoid leaking resources when the sync is done. ctx, cancel := context.WithCancel(ctx) defer cancel() - maestroClient, err := c.createMaestroClientFromProvisionShard(ctx, clusterProvisionShard) + maestroClient, err := createMaestroClientFromCSProvisionShard(ctx, c.maestroSourceEnvironmentIdentifier, c.maestroClientBuilder, clusterProvisionShard) if err != nil { return utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) } @@ -448,25 +448,3 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) getNodePoolNamespace( func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) CooldownChecker() controllerutils.CooldownChecker { return c.cooldownChecker } - -// createMaestroClientFromProvisionShard creates a Maestro client for the given provision shard. -// The client is scoped to the Maestro Consumer associated to the provision shard, as well -// as to the the Maestro Source ID associated to the provision shard which is calculated from the provision shard ID and the -// environment specified in c.maestroSourceEnvironmentIdentifier. -func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) createMaestroClientFromProvisionShard( - ctx context.Context, provisionShard *arohcpv1alpha1.ProvisionShard, -) (maestro.Client, error) { - provisionShardMaestroConsumerName := provisionShard.MaestroConfig().ConsumerName() - provisionShardMaestroRESTAPIEndpoint := provisionShard.MaestroConfig().RestApiConfig().Url() - provisionShardMaestroGRPCAPIEndpoint := provisionShard.MaestroConfig().GrpcApiConfig().Url() - // This allows us to be able to have visibility on the Maestro Bundles owned by the same source ID for a given - // provision shard and environment. This should have the same source ID as what CS has in each corresponding environment - // because otherwise we would not have visibility on the Maestro Bundles owned - // TODO do we want to use the same source ID that CS uses or do we want intentionally a different one? This has consequences - // on the visibility of the Maestro Bundles, including processing of events sent by Maestro. - maestroSourceID := maestro.GenerateMaestroSourceID(c.maestroSourceEnvironmentIdentifier, provisionShard.ID()) - - maestroClient, err := c.maestroClientBuilder.NewClient(ctx, provisionShardMaestroRESTAPIEndpoint, provisionShardMaestroGRPCAPIEndpoint, provisionShardMaestroConsumerName, maestroSourceID) - - return maestroClient, err -} diff --git a/backend/pkg/controllers/cs_maestro_utils.go b/backend/pkg/controllers/cs_maestro_utils.go new file mode 100644 index 00000000000..4c71aa3b0ef --- /dev/null +++ b/backend/pkg/controllers/cs_maestro_utils.go @@ -0,0 +1,44 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "context" + + arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" + + "github.com/Azure/ARO-HCP/backend/pkg/maestro" +) + +// createMaestroClientFromCSProvisionShard creates a Maestro client for the given cluster provision shard. +// the client is scoped to the Consumer Name associated to the provision shard, and to +// the source ID associated to the provision shard and the environment specified +// in c.maestroSourceEnvironmentIdentifier, which is a configuration parameter at +// deployment time. +func createMaestroClientFromCSProvisionShard( + ctx context.Context, maestroSourceEnvironmentIdentifier string, maestroClientBuilder maestro.MaestroClientBuilder, clusterProvisionShard *arohcpv1alpha1.ProvisionShard, +) (maestro.Client, error) { + provisionShardMaestroConsumerName := clusterProvisionShard.MaestroConfig().ConsumerName() + provisionShardMaestroRESTAPIEndpoint := clusterProvisionShard.MaestroConfig().RestApiConfig().Url() + provisionShardMaestroGRPCAPIEndpoint := clusterProvisionShard.MaestroConfig().GrpcApiConfig().Url() + // This allows us to be able to have visibility on the Maestro Bundles owned by the same source ID for a given + // provision shard and environment. This should have the same source ID as what CS has in each corresponding environment + // because otherwise we would not have visibility on the Maestro Bundles owned + maestroSourceID := maestro.GenerateMaestroSourceID(maestroSourceEnvironmentIdentifier, clusterProvisionShard.ID()) + + maestroClient, err := maestroClientBuilder.NewClient(ctx, provisionShardMaestroRESTAPIEndpoint, provisionShardMaestroGRPCAPIEndpoint, provisionShardMaestroConsumerName, maestroSourceID) + + return maestroClient, err +} diff --git a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go index a2b5db8b513..6c563947560 100644 --- a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go @@ -28,8 +28,6 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/utils/ptr" - arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" - "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" "github.com/Azure/ARO-HCP/backend/pkg/maestro" "github.com/Azure/ARO-HCP/internal/api" @@ -240,7 +238,7 @@ func (c *deleteOrphanedMaestroReadonlyBundles) buildMaestroClientsByProvisionSha // We create a new context with a cancel function so we can cancel the Maestro client when the sync is done. // This is important to avoid leaking resources when the sync is done. maestroClientCtx, cancel := context.WithCancel(ctx) - maestroClient, err := c.createMaestroClientFromProvisionShard(maestroClientCtx, provisionShard) + maestroClient, err := createMaestroClientFromCSProvisionShard(maestroClientCtx, c.maestroSourceEnvironmentIdentifier, c.maestroClientBuilder, provisionShard) if err != nil { cancel() // on error creating the Maestro client we ensure we cancel the context that we just created too return maestroClientsByProvisionShard, utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) @@ -660,25 +658,3 @@ func (c *deleteOrphanedMaestroReadonlyBundles) processNextWorkItem(ctx context.C return true } - -// createMaestroClientFromProvisionShard creates a Maestro client for the given provision shard. -// The client is scoped to the Maestro Consumer associated to the provision shard, as well -// as to the the Maestro Source ID associated to the provision shard which is calculated from the provision shard ID and the -// environment specified in c.maestroSourceEnvironmentIdentifier. -func (c *deleteOrphanedMaestroReadonlyBundles) createMaestroClientFromProvisionShard( - ctx context.Context, provisionShard *arohcpv1alpha1.ProvisionShard, -) (maestro.Client, error) { - provisionShardMaestroConsumerName := provisionShard.MaestroConfig().ConsumerName() - provisionShardMaestroRESTAPIEndpoint := provisionShard.MaestroConfig().RestApiConfig().Url() - provisionShardMaestroGRPCAPIEndpoint := provisionShard.MaestroConfig().GrpcApiConfig().Url() - // This allows us to be able to have visibility on the Maestro Bundles owned by the same source ID for a given - // provision shard and environment. This should have the same source ID as what CS has in each corresponding environment - // because otherwise we would not have visibility on the Maestro Bundles owned - // TODO do we want to use the same source ID that CS uses or do we want intentionally a different one? This has consequences - // on the visibility of the Maestro Bundles, including processing of events sent by Maestro. - maestroSourceID := maestro.GenerateMaestroSourceID(c.maestroSourceEnvironmentIdentifier, provisionShard.ID()) - - maestroClient, err := c.maestroClientBuilder.NewClient(ctx, provisionShardMaestroRESTAPIEndpoint, provisionShardMaestroGRPCAPIEndpoint, provisionShardMaestroConsumerName, maestroSourceID) - - return maestroClient, err -} diff --git a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go index 7a6dea11197..61593e7b65e 100644 --- a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go +++ b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go @@ -29,8 +29,6 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" - "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" "github.com/Azure/ARO-HCP/backend/pkg/informers" "github.com/Azure/ARO-HCP/backend/pkg/listers" @@ -126,7 +124,7 @@ func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) SyncOnc // This is important to avoid leaking resources when the sync is done. ctx, cancel := context.WithCancel(ctx) defer cancel() - maestroClient, err := c.createMaestroClientFromProvisionShard(ctx, clusterProvisionShard) + maestroClient, err := createMaestroClientFromCSProvisionShard(ctx, c.maestroSourceEnvironmentIdentifier, c.maestroClientBuilder, clusterProvisionShard) if err != nil { return utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) } @@ -327,24 +325,3 @@ func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) getSing func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) CooldownChecker() controllerutils.CooldownChecker { return c.cooldownChecker } - -// createMaestroClientFromProvisionShard creates a Maestro client for the given cluster provision shard. -// the client is scoped to the Consumer Name associated to the provision shard, and to -// the source ID associated to the provision shard and the environment specified -// in c.maestroSourceEnvironmentIdentifier, which is a configuration parameter at -// deployment time. -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) createMaestroClientFromProvisionShard( - ctx context.Context, clusterProvisionShard *arohcpv1alpha1.ProvisionShard, -) (maestro.Client, error) { - provisionShardMaestroConsumerName := clusterProvisionShard.MaestroConfig().ConsumerName() - provisionShardMaestroRESTAPIEndpoint := clusterProvisionShard.MaestroConfig().RestApiConfig().Url() - provisionShardMaestroGRPCAPIEndpoint := clusterProvisionShard.MaestroConfig().GrpcApiConfig().Url() - // This allows us to be able to have visibility on the Maestro Bundles owned by the same source ID for a given - // provision shard and environment. This should have the same source ID as what CS has in each corresponding environment - // because otherwise we would not have visibility on the Maestro Bundles owned - maestroSourceID := maestro.GenerateMaestroSourceID(c.maestroSourceEnvironmentIdentifier, clusterProvisionShard.ID()) - - maestroClient, err := c.maestroClientBuilder.NewClient(ctx, provisionShardMaestroRESTAPIEndpoint, provisionShardMaestroGRPCAPIEndpoint, provisionShardMaestroConsumerName, maestroSourceID) - - return maestroClient, err -} diff --git a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go index baa9f98849a..56bd42bc735 100644 --- a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go +++ b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go @@ -29,8 +29,6 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" - "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" "github.com/Azure/ARO-HCP/backend/pkg/informers" "github.com/Azure/ARO-HCP/backend/pkg/listers" @@ -121,7 +119,7 @@ func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) SyncOn if err != nil { return utils.TrackError(fmt.Errorf("failed to get Cluster Provision Shard from Cluster Service: %w", err)) } - maestroClient, err := c.createMaestroClientFromProvisionShard(ctx, clusterProvisionShard) + maestroClient, err := createMaestroClientFromCSProvisionShard(ctx, c.maestroSourceEnvironmentIdentifier, c.maestroClientBuilder, clusterProvisionShard) if err != nil { return utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) } @@ -326,24 +324,3 @@ func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) getSin func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) CooldownChecker() controllerutils.CooldownChecker { return c.cooldownChecker } - -// createMaestroClientFromProvisionShard creates a Maestro client for the given cluster provision shard. -// the client is scoped to the Consumer Name associated to the provision shard, and to -// the source ID associated to the provision shard and the environment specified -// in c.maestroSourceEnvironmentIdentifier, which is a configuration parameter at -// deployment time. -func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) createMaestroClientFromProvisionShard( - ctx context.Context, clusterProvisionShard *arohcpv1alpha1.ProvisionShard, -) (maestro.Client, error) { - provisionShardMaestroConsumerName := clusterProvisionShard.MaestroConfig().ConsumerName() - provisionShardMaestroRESTAPIEndpoint := clusterProvisionShard.MaestroConfig().RestApiConfig().Url() - provisionShardMaestroGRPCAPIEndpoint := clusterProvisionShard.MaestroConfig().GrpcApiConfig().Url() - // This allows us to be able to have visibility on the Maestro Bundles owned by the same source ID for a given - // provision shard and environment. This should have the same source ID as what CS has in each corresponding environment - // because otherwise we would not have visibility on the Maestro Bundles owned - maestroSourceID := maestro.GenerateMaestroSourceID(c.maestroSourceEnvironmentIdentifier, clusterProvisionShard.ID()) - - maestroClient, err := c.maestroClientBuilder.NewClient(ctx, provisionShardMaestroRESTAPIEndpoint, provisionShardMaestroGRPCAPIEndpoint, provisionShardMaestroConsumerName, maestroSourceID) - - return maestroClient, err -} From 4ab9adee45cf2736bd171ac482e0052532678f1f Mon Sep 17 00:00:00 2001 From: Miguel Soriano Date: Mon, 23 Mar 2026 14:28:59 +0100 Subject: [PATCH 06/16] feat: add ListAll method that iterates over all internal api types using a cosmos lister --- internal/database/iterators.go | 40 +++++ internal/database/iterators_test.go | 238 ++++++++++++++++++++++++++++ 2 files changed, 278 insertions(+) create mode 100644 internal/database/iterators_test.go diff --git a/internal/database/iterators.go b/internal/database/iterators.go index f11f6851b22..143186a9e97 100644 --- a/internal/database/iterators.go +++ b/internal/database/iterators.go @@ -157,3 +157,43 @@ func (iter *queryBillingIterator) GetContinuationToken() string { func (iter *queryBillingIterator) GetError() error { return iter.err } + +// ListAll accumulates all pages from a DB GlobalLister into a slice. +// It calls listFn with a page size hint of pageSize and follows continuation tokens +// until all items have been collected. +// Any failure to list or iterate causes an early return because the caller needs the complete +// set of data to make correct decisions. +// listFn is expected to accept a opts *DBClientListResourceDocsOptions that supports +// paging via the PageSizeHint attribute. +// ListAll assists client code in breaking large list queries into multiple smaller chunks of pageSize or smaller. This +// helps reduce the load on the database at the cost of more round trips. +// pageSize determines the maximum number of items to be retrieved at once. A negative value will set a dynamic +// page size controlled by Cosmos. +func ListAll[InternalAPIType any]( + ctx context.Context, + pageSize int32, + listFn func(ctx context.Context, opts *DBClientListResourceDocsOptions) (DBClientIterator[InternalAPIType], error), +) ([]*InternalAPIType, error) { + opts := &DBClientListResourceDocsOptions{ + PageSizeHint: &pageSize, + } + all := make([]*InternalAPIType, 0) + for { + iterator, err := listFn(ctx, opts) + if err != nil { + return nil, fmt.Errorf("failed to list: %w", err) + } + for _, item := range iterator.Items(ctx) { + all = append(all, item) + } + if err := iterator.GetError(); err != nil { + return nil, fmt.Errorf("failed iterating: %w", err) + } + token := iterator.GetContinuationToken() + if token == "" { + break + } + opts.ContinuationToken = &token + } + return all, nil +} diff --git a/internal/database/iterators_test.go b/internal/database/iterators_test.go new file mode 100644 index 00000000000..c0973a51a39 --- /dev/null +++ b/internal/database/iterators_test.go @@ -0,0 +1,238 @@ +// Copyright 2025 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeIteratorEntry holds one item yielded by a fakeIterator. +type fakeIteratorEntry[T any] struct { + id string + item *T +} + +// fakeIterator is a test implementation of DBClientIterator[T] that yields a +// fixed set of items, optionally returning a continuation token and/or an error. +type fakeIterator[T any] struct { + entries []fakeIteratorEntry[T] + token string + err error +} + +func (f *fakeIterator[T]) Items(_ context.Context) DBClientIteratorItem[T] { + return func(yield func(string, *T) bool) { + for _, e := range f.entries { + if !yield(e.id, e.item) { + return + } + } + } +} + +func (f *fakeIterator[T]) GetContinuationToken() string { return f.token } +func (f *fakeIterator[T]) GetError() error { return f.err } + +func TestListAll(t *testing.T) { + // testListItem is a minimal type used as the ListAll type parameter in tests. + type testListItem struct{ value string } + + type listFn = func(context.Context, *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) + + type testCase struct { + name string + pageSize int32 + listFn listFn + wantItems []string // nil means expect empty/nil result + wantErr string // non-empty means expect an error containing this substring + verify func(*testing.T) // optional: extra assertions beyond result/error + } + + // --- Stateful setup: state variables defined here are captured by the closures below. --- + + // multi-page: verify the continuation token is forwarded on the second call + var mpCallCount int + var mpReceivedTokens []*string + mpListFn := func(_ context.Context, opts *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) { + mpCallCount++ + mpReceivedTokens = append(mpReceivedTokens, opts.ContinuationToken) + if mpCallCount == 1 { + return &fakeIterator[testListItem]{ + entries: []fakeIteratorEntry[testListItem]{ + {id: "p1-0", item: &testListItem{value: "p1-a"}}, + {id: "p1-1", item: &testListItem{value: "p1-b"}}, + }, + token: "tok-1", + }, nil + } + return &fakeIterator[testListItem]{ + entries: []fakeIteratorEntry[testListItem]{ + {id: "p2-0", item: &testListItem{value: "p2-a"}}, + }, + }, nil + } + + // three-page: verify listFn is called exactly three times + var threeCallCount int + threeListFn := func(_ context.Context, _ *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) { + threeCallCount++ + switch threeCallCount { + case 1: + return &fakeIterator[testListItem]{ + entries: []fakeIteratorEntry[testListItem]{{id: "0", item: &testListItem{value: "first"}}}, + token: "tok-a", + }, nil + case 2: + return &fakeIterator[testListItem]{ + entries: []fakeIteratorEntry[testListItem]{{id: "1", item: &testListItem{value: "second"}}}, + token: "tok-b", + }, nil + default: + return &fakeIterator[testListItem]{ + entries: []fakeIteratorEntry[testListItem]{{id: "2", item: &testListItem{value: "third"}}}, + }, nil + } + } + + // error on second page: verify listFn is called exactly twice + var secondPageCallCount int + secondPageListFn := func(_ context.Context, _ *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) { + secondPageCallCount++ + if secondPageCallCount == 1 { + return &fakeIterator[testListItem]{ + entries: []fakeIteratorEntry[testListItem]{{id: "id-0", item: &testListItem{value: "first"}}}, + token: "tok", + }, nil + } + return nil, fmt.Errorf("second page error") + } + + // page-size hint: capture the PageSizeHint seen by listFn + var capturedPageSize *int32 + pageSizeListFn := func(_ context.Context, opts *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) { + capturedPageSize = opts.PageSizeHint + return &fakeIterator[testListItem]{}, nil + } + + // --- Test table --- + + tests := []testCase{ + { + name: "empty result", + pageSize: 10, + listFn: func(_ context.Context, _ *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) { + return &fakeIterator[testListItem]{}, nil + }, + }, + { + name: "single page returns all items", + pageSize: 10, + wantItems: []string{"a", "b", "c"}, + listFn: func(_ context.Context, _ *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) { + return &fakeIterator[testListItem]{ + entries: []fakeIteratorEntry[testListItem]{ + {id: "id-0", item: &testListItem{value: "a"}}, + {id: "id-1", item: &testListItem{value: "b"}}, + {id: "id-2", item: &testListItem{value: "c"}}, + }, + }, nil + }, + }, + { + name: "multi-page follows continuation token", + pageSize: 5, + listFn: mpListFn, + wantItems: []string{"p1-a", "p1-b", "p2-a"}, + verify: func(t *testing.T) { + assert.Equal(t, 2, mpCallCount, "expected exactly two listFn calls") + assert.Nil(t, mpReceivedTokens[0], "first call should have no continuation token") + require.NotNil(t, mpReceivedTokens[1], "second call should carry the continuation token from page 1") + assert.Equal(t, "tok-1", *mpReceivedTokens[1]) + }, + }, + { + name: "three pages accumulates all items", + pageSize: 1, + listFn: threeListFn, + wantItems: []string{"first", "second", "third"}, + verify: func(t *testing.T) { + assert.Equal(t, 3, threeCallCount) + }, + }, + { + name: "pageSize is forwarded as PageSizeHint", + pageSize: 42, + listFn: pageSizeListFn, + verify: func(t *testing.T) { + require.NotNil(t, capturedPageSize) + assert.Equal(t, int32(42), *capturedPageSize) + }, + }, + { + name: "listFn error on first call", + pageSize: 10, + listFn: func(_ context.Context, _ *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) { + return nil, fmt.Errorf("connection refused") + }, + wantErr: "failed to list: connection refused", + }, + { + name: "listFn error on second page", + pageSize: 10, + listFn: secondPageListFn, + wantErr: "failed to list: second page error", + verify: func(t *testing.T) { + assert.Equal(t, 2, secondPageCallCount) + }, + }, + { + name: "iterator error", + pageSize: 10, + listFn: func(_ context.Context, _ *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) { + return &fakeIterator[testListItem]{err: fmt.Errorf("unmarshal error")}, nil + }, + wantErr: "failed iterating: unmarshal error", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := ListAll(context.Background(), tt.pageSize, tt.listFn) + + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + assert.Nil(t, result) + } else { + require.NoError(t, err) + assert.NotNil(t, result) + var gotValues []string + for _, item := range result { + gotValues = append(gotValues, item.value) + } + assert.Equal(t, tt.wantItems, gotValues) + } + + if tt.verify != nil { + tt.verify(t) + } + }) + } +} From 67ae777280378857558fa7589c512f9829aa5349 Mon Sep 17 00:00:00 2001 From: Miguel Soriano Date: Mon, 23 Mar 2026 14:47:31 +0100 Subject: [PATCH 07/16] feat: add orphan cleanup for nodepools and reuse parts of maestro readonly bundles logic Extract shared readonly-bundle helpers (build bundle, references, kube content) into maestro_readonly_bundle_helpers.go and use them from cluster- and nodepool-scoped create/persist controllers. Refactor DeleteOrphanedMaestroReadonlyBundles: - List SPC/SPNP with database.ListAll - Add maestro.ForEachMaestroBundle for paginated List and unit tests. - Reuse code in some places - Update/add controller tests for the new paths. --- ...ped_maestro_readonly_bundles_controller.go | 100 +-- ...aestro_readonly_bundles_controller_test.go | 12 +- ...ped_maestro_readonly_bundles_controller.go | 120 +-- ...aestro_readonly_bundles_controller_test.go | 785 ++++++++++++++++++ ...ned_maestro_readonly_bundles_controller.go | 424 ++++------ ...aestro_readonly_bundles_controller_test.go | 730 +++++++++------- .../maestro_readonly_bundle_helpers.go | 318 +++++++ .../maestro_readonly_bundle_helpers_test.go | 229 +++++ ...tro_readonly_bundles_content_controller.go | 193 +---- ...eadonly_bundles_content_controller_test.go | 227 +---- ...tro_readonly_bundles_content_controller.go | 193 +---- ...eadonly_bundles_content_controller_test.go | 648 +++++++++++++++ backend/pkg/maestro/maestro_bundle.go | 24 + backend/pkg/maestro/maestro_bundle_test.go | 107 +++ 14 files changed, 2758 insertions(+), 1352 deletions(-) create mode 100644 backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go create mode 100644 backend/pkg/controllers/maestro_readonly_bundle_helpers.go create mode 100644 backend/pkg/controllers/maestro_readonly_bundle_helpers_test.go create mode 100644 backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go diff --git a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go index a7f153d17a8..2f84d4884bf 100644 --- a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go @@ -23,7 +23,6 @@ import ( workv1 "open-cluster-management.io/api/work/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" @@ -40,8 +39,6 @@ import ( ) const ( - // readonlyBundleManagedByK8sLabelKey is the key of the K8s label that is used to identify the controller that manages the readonly Maestro bundle. - readonlyBundleManagedByK8sLabelKey = "aro-hcp.azure.com/readonly-bundle-managed-by" // readonlyBundleManagedByK8sLabelValueClusterScoped is the K8s label associated to the readonlyBundleManagedByK8sLabelKey // key that indicates that the readonly Maestro bundle is managed by the create // cluster scoped maestro readonly bundles controller. @@ -226,7 +223,7 @@ func (c *createClusterScopedMaestroReadonlyBundlesSyncer) syncMaestroBundle( // and it makes it resistant to crashes/reboots. if existingMaestroBundleRef == nil { var err error - existingMaestroBundleRef, err = c.buildInitialMaestroBundleReference(maestroBundleInternalName) + existingMaestroBundleRef, err = buildInitialMaestroBundleReference(maestroBundleInternalName, c.maestroAPIMaestroBundleNameGenerator) if err != nil { return lastPersistedSPC, utils.TrackError(fmt.Errorf("failed to build initial Maestro Bundle reference: %w", err)) } @@ -332,100 +329,7 @@ func (c *createClusterScopedMaestroReadonlyBundlesSyncer) buildInitialReadonlyMa Namespace: hostedCluster.Namespace, } - return c.buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName, maestroBundleResourceIdentifier, hostedCluster) -} - -// buildInitialReadonlyMaestroBundle builds an initial readonly Maestro Bundle for a given resource specified in obj. -// objResourceIdentifier is the resource identifier of the resource specified in obj. -// maestroBundleNamespacedName is the namespaced name of the Maestro Bundle. -// Used to create the readonly Maestro bundle associated to the resource specified in obj. -func (c *createClusterScopedMaestroReadonlyBundlesSyncer) buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName types.NamespacedName, objResourceIdentifier workv1.ResourceIdentifier, obj runtime.Object) *workv1.ManifestWork { - maestroBundleObjMeta := metav1.ObjectMeta{ - Name: maestroBundleNamespacedName.Name, - Namespace: maestroBundleNamespacedName.Namespace, - ResourceVersion: "0", // TODO is this needed when creating a maestro bundle? - Labels: map[string]string{ - // We define it as a K8s label because Maestro supports server-side filtering based on K8s labels. - // We can define it as a K8s label because for this specific use case we can comply with - // K8s labels length and charset restrictions https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set. - readonlyBundleManagedByK8sLabelKey: readonlyBundleManagedByK8sLabelValueClusterScoped, - }, - } - - // We build the Maestro Bundle that will contain the resource specified in obj. - // Aside from putting the resource (manifest) previously built above, we - // also define a FeedbackRule that will allow us to retrieve the whole content - // from the management cluster - maestroBundle := &workv1.ManifestWork{ - ObjectMeta: maestroBundleObjMeta, - Spec: workv1.ManifestWorkSpec{ - Workload: workv1.ManifestsTemplate{ - Manifests: []workv1.Manifest{ - { - RawExtension: runtime.RawExtension{ - // We put the resource (manifest) specified in obj. - // In Maestro only the desired `spec` as defined in the bundle can be retrieved - // from here when querying the Maestro Bundle. - // To retrieve another section other than the desired spec Maestro - // requires defining FeedbackRule(s) in the Maestro bundle. - // For maestro readonly resources, not even the desired spec can be retrieved from here. For - // those type of resources it needs to be retrieved via status feedback rule(s) too. - // For owned resources, here the desired spec can be retrieved but that - // is not necessarily the actual spec in the management cluster side. If that is - // desired it is again necessary to get the spec via FeedbackRule(s). - Object: obj, - }, - }, - }, - }, - ManifestConfigs: []workv1.ManifestConfigOption{ - // We also need to define the ManifestConfig associated to the resource(manifest) - // that is being put within the Maestro Bundle. - { - // ResourceIdentifier needs to be specified and it is the information - // associated to the manifest that is being put within the Maestro Bundle. - ResourceIdentifier: objResourceIdentifier, - // We need to set the UpdateStrategy to read only. This - // creates a "readonly maestro bundle". - UpdateStrategy: &workv1.UpdateStrategy{ - Type: workv1.UpdateStrategyTypeReadOnly, - }, - // We define a feedbackrule based on JSONPath. We alias the name - // of this JSONPath as "resource" and its real JSONPath is "@" which - // signals the whole object is retrieved. This includes both spec - // and status. - FeedbackRules: []workv1.FeedbackRule{ - { - Type: workv1.JSONPathsType, - JsonPaths: []workv1.JsonPath{ - { - Name: "resource", - Path: "@", - }, - }, - }, - }, - }, - }, - }, - } - - return maestroBundle -} - -// buildInitialMaestroBundleReference builds an initial Maestro Bundle reference for a given maestro bundle internal name. -func (c *createClusterScopedMaestroReadonlyBundlesSyncer) buildInitialMaestroBundleReference(internalName api.MaestroBundleInternalName) (*api.MaestroBundleReference, error) { - maestroAPIMaestroBundleName, err := c.maestroAPIMaestroBundleNameGenerator.NewMaestroAPIMaestroBundleName() - if err != nil { - return nil, utils.TrackError(fmt.Errorf("failed to generate Maestro API Maestro Bundle name: %w", err)) - } - hostedClusterMWMaestroBundleReference := &api.MaestroBundleReference{ - Name: internalName, - MaestroAPIMaestroBundleName: maestroAPIMaestroBundleName, - MaestroAPIMaestroBundleID: "", - } - - return hostedClusterMWMaestroBundleReference, nil + return buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName, maestroBundleResourceIdentifier, hostedCluster, readonlyBundleManagedByK8sLabelValueClusterScoped) } // getHostedClusterNamespace gets the namespace for the hosted cluster based on the environment name and the cluster service OCM Cluster ID. diff --git a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller_test.go b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller_test.go index bb35d6f1f37..8f2a70fb855 100644 --- a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller_test.go +++ b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller_test.go @@ -106,11 +106,9 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_getHostedClusterNamespa } func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_buildInitialMaestroBundleReference(t *testing.T) { - syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), - } + generator := maestro.NewMaestroAPIMaestroBundleNameGenerator() - ref, err := syncer.buildInitialMaestroBundleReference(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster) + ref, err := buildInitialMaestroBundleReference(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster, generator) require.NoError(t, err) assert.NotNil(t, ref) @@ -450,10 +448,6 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_syncMaestroBundle(t *te } func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_buildInitialReadonlyMaestroBundle(t *testing.T) { - syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), - } - maestroBundleNamespacedName := types.NamespacedName{ Name: "custom-bundle", Namespace: "custom-namespace", @@ -478,7 +472,7 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_buildInitialReadonlyMae Namespace: configMap.Namespace, } - bundle := syncer.buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName, resourceIdentifier, configMap) + bundle := buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName, resourceIdentifier, configMap, readonlyBundleManagedByK8sLabelValueClusterScoped) // Verify bundle metadata assert.NotNil(t, bundle) diff --git a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go index 6ac47074045..0a4610d5ebf 100644 --- a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go @@ -24,7 +24,6 @@ import ( workv1 "open-cluster-management.io/api/work/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" @@ -111,7 +110,7 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) generateMaestroBundle } func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) SyncOnce(ctx context.Context, key controllerutils.HCPNodePoolKey) error { - existingNodePool, err := c.cosmosClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName).Get(ctx, key.HCPClusterName) + existingNodePool, err := c.cosmosClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName).Get(ctx, key.HCPNodePoolName) if database.IsResponseError(err, http.StatusNotFound) { return nil // nodepool doesn't exist, no work to do } @@ -189,9 +188,9 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) SyncOnce(ctx context. csClusterDomainPrefix := csCluster.DomainPrefix() // We sync the Maestro Bundles that need to be synced. - // We pass the latest existingServiceProviderCluster into each iteration and use the returned - // updated SPC for the next, so that multiple bundles see persisted updates from previous iterations. - // We always apply updatedSPC (even on error) so in-memory state stays in sync with Cosmos + // We pass the latest existingServiceProviderNodePool into each iteration and use the returned + // updated SPNP for the next, so that multiple bundles see persisted updates from previous iterations. + // We always apply updatedSPNP (even on error) so in-memory state stays in sync with Cosmos // when syncMaestroBundle persisted a partial change before failing. var syncErrors []error for _, maestroBundleInternalName := range maestroBundlesToSync { @@ -208,9 +207,9 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) SyncOnce(ctx context. return utils.TrackError(errors.Join(syncErrors...)) } -// syncMaestroBundle ensures the given Maestro bundle exists in Maestro, as well as a reference to it in ServiceProviderCluster. -// It returns the updated ServiceProviderCluster (after any Replace calls) so the caller can pass it into the next sync. -// On error, the first return value is always the lastest persisted ServiceProviderClass SPC, so the +// syncMaestroBundle ensures the given Maestro bundle exists in Maestro, as well as a reference to it in ServiceProviderNodePool. +// It returns the updated ServiceProviderNodePool (after any Replace calls) so the caller can pass it into the next sync. +// On error, the first return value is always the lastest persisted ServiceProviderNodePool, so the // caller can keep in-memory state in sync and subsequent bundle syncs in the same run never see stale data. func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) syncMaestroBundle( ctx context.Context, @@ -234,7 +233,7 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) syncMaestroBundle( // and it makes it resistant to crashes/reboots. if existingMaestroBundleRef == nil { var err error - existingMaestroBundleRef, err = c.buildInitialMaestroBundleReference(maestroBundleInternalName) + existingMaestroBundleRef, err = buildInitialMaestroBundleReference(maestroBundleInternalName, c.maestroAPIMaestroBundleNameGenerator) if err != nil { return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to build initial Maestro Bundle reference: %w", err)) } @@ -244,7 +243,7 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) syncMaestroBundle( } existingServiceProviderNodePool, err = serviceProviderNodePoolsDBClient.Replace(ctx, existingServiceProviderNodePool, nil) if err != nil { - return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to replace ServiceProviderCluster in database: %w", err)) + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to replace ServiceProviderNodePool in database: %w", err)) } lastPersistedSPNP = existingServiceProviderNodePool existingMaestroBundleRef, err = existingServiceProviderNodePool.Status.MaestroReadonlyBundles.Get(maestroBundleInternalName) @@ -252,7 +251,7 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) syncMaestroBundle( return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle reference: %w", err)) } if existingMaestroBundleRef == nil { - return lastPersistedSPNP, utils.TrackError(fmt.Errorf("maestro Bundle reference %q not found in ServiceProviderCluster", maestroBundleInternalName)) + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("maestro Bundle reference %q not found in ServiceProviderNodePool", maestroBundleInternalName)) } } @@ -275,7 +274,7 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) syncMaestroBundle( return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to get or create Maestro Bundle: %w", err)) } - // If the Maestro API MaestroBundle ID is not set we store the returned Maestro Bundle ID in the corresponding Maestro Bundle reference of the ServiceProviderCluster in Cosmos. + // If the Maestro API MaestroBundle ID is not set we store the returned Maestro Bundle ID in the corresponding Maestro Bundle reference of the ServiceProviderNodePool in Cosmos. if len(existingMaestroBundleRef.MaestroAPIMaestroBundleID) == 0 { bundleID := string(resultMaestroBundle.UID) existingMaestroBundleRef.MaestroAPIMaestroBundleID = bundleID @@ -285,7 +284,7 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) syncMaestroBundle( } existingServiceProviderNodePool, err = serviceProviderNodePoolsDBClient.Replace(ctx, existingServiceProviderNodePool, nil) if err != nil { - return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to replace ServiceProviderCluster in database: %w", err)) + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to replace ServiceProviderNodePool in database: %w", err)) } lastPersistedSPNP = existingServiceProviderNodePool } @@ -340,100 +339,7 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildInitialReadonlyM Namespace: hypershiftNodePool.Namespace, } - return c.buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName, maestroBundleResourceIdentifier, hypershiftNodePool) -} - -// buildInitialReadonlyMaestroBundle builds an initial readonly Maestro Bundle for a given resource specified in obj. -// objResourceIdentifier is the resource identifier of the resource specified in obj. -// maestroBundleNamespacedName is the namespaced name of the Maestro Bundle. -// Used to create the readonly Maestro bundle associated to the resource specified in obj. -func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName types.NamespacedName, objResourceIdentifier workv1.ResourceIdentifier, obj runtime.Object) *workv1.ManifestWork { - maestroBundleObjMeta := metav1.ObjectMeta{ - Name: maestroBundleNamespacedName.Name, - Namespace: maestroBundleNamespacedName.Namespace, - ResourceVersion: "0", // TODO is this needed when creating a maestro bundle? - Labels: map[string]string{ - // We define it as a K8s label because Maestro supports server-side filtering based on K8s labels. - // We can define it as a K8s label because for this specific use case we can comply with - // K8s labels length and charset restrictions https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set. - readonlyBundleManagedByK8sLabelKey: readonlyBundleManagedByK8sLabelValueNodePoolScoped, - }, - } - - // We build the Maestro Bundle that will contain the resource specified in obj. - // Aside from putting the resource (manifest) previously built above, we - // also define a FeedbackRule that will allow us to retrieve the whole content - // from the management cluster - maestroBundle := &workv1.ManifestWork{ - ObjectMeta: maestroBundleObjMeta, - Spec: workv1.ManifestWorkSpec{ - Workload: workv1.ManifestsTemplate{ - Manifests: []workv1.Manifest{ - { - RawExtension: runtime.RawExtension{ - // We put the resource (manifest) specified in obj. - // In Maestro only the desired `spec` as defined in the bundle can be retrieved - // from here when querying the Maestro Bundle. - // To retrieve another section other than the desired spec Maestro - // requires defining FeedbackRule(s) in the Maestro bundle. - // For maestro readonly resources, not even the desired spec can be retrieved from here. For - // those type of resources it needs to be retrieved via status feedback rule(s) too. - // For owned resources, here the desired spec can be retrieved but that - // is not necessarily the actual spec in the management cluster side. If that is - // desired it is again necessary to get the spec via FeedbackRule(s). - Object: obj, - }, - }, - }, - }, - ManifestConfigs: []workv1.ManifestConfigOption{ - // We also need to define the ManifestConfig associated to the resource(manifest) - // that is being put within the Maestro Bundle. - { - // ResourceIdentifier needs to be specified and it is the information - // associated to the manifest that is being put within the Maestro Bundle. - ResourceIdentifier: objResourceIdentifier, - // We need to set the UpdateStrategy to read only. This - // creates a "readonly maestro bundle". - UpdateStrategy: &workv1.UpdateStrategy{ - Type: workv1.UpdateStrategyTypeReadOnly, - }, - // We define a feedbackrule based on JSONPath. We alias the name - // of this JSONPath as "resource" and its real JSONPath is "@" which - // signals the whole object is retrieved. This includes both spec - // and status. - FeedbackRules: []workv1.FeedbackRule{ - { - Type: workv1.JSONPathsType, - JsonPaths: []workv1.JsonPath{ - { - Name: "resource", - Path: "@", - }, - }, - }, - }, - }, - }, - }, - } - - return maestroBundle -} - -// buildInitialMaestroBundleReference builds an initial Maestro Bundle reference for a given maestro bundle internal name. -func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildInitialMaestroBundleReference(internalName api.MaestroBundleInternalName) (*api.MaestroBundleReference, error) { - maestroAPIMaestroBundleName, err := c.maestroAPIMaestroBundleNameGenerator.NewMaestroAPIMaestroBundleName() - if err != nil { - return nil, utils.TrackError(fmt.Errorf("failed to generate Maestro API Maestro Bundle name: %w", err)) - } - nodePoolMaestroBundleReference := &api.MaestroBundleReference{ - Name: internalName, - MaestroAPIMaestroBundleName: maestroAPIMaestroBundleName, - MaestroAPIMaestroBundleID: "", - } - - return nodePoolMaestroBundleReference, nil + return buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName, maestroBundleResourceIdentifier, hypershiftNodePool, readonlyBundleManagedByK8sLabelValueNodePoolScoped) } // getNodePoolNamespace gets the namespace for the node pool based on the environment name, the cluster service OCM Cluster ID and the node pool service OCM Node Pool ID. diff --git a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go new file mode 100644 index 00000000000..b02b0107cae --- /dev/null +++ b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go @@ -0,0 +1,785 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + workv1 "open-cluster-management.io/api/work/v1" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + + arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" + hsv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + + "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" + "github.com/Azure/ARO-HCP/backend/pkg/maestro" + "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/api/arm" + "github.com/Azure/ARO-HCP/internal/database" + "github.com/Azure/ARO-HCP/internal/databasetesting" + "github.com/Azure/ARO-HCP/internal/ocm" +) + +// errorInjectingDBClientForNodePoolCreate wraps MockDBClient to return error-injecting CRUDs. +type errorInjectingDBClientForNodePoolCreate struct { + *databasetesting.MockDBClient + spnpCRUD database.ServiceProviderNodePoolCRUD +} + +func (e *errorInjectingDBClientForNodePoolCreate) ServiceProviderNodePools(subscriptionID, resourceGroupName, clusterName, nodePoolName string) database.ServiceProviderNodePoolCRUD { + if e.spnpCRUD != nil { + return e.spnpCRUD + } + return e.MockDBClient.ServiceProviderNodePools(subscriptionID, resourceGroupName, clusterName, nodePoolName) +} + +var _ database.DBClient = &errorInjectingDBClientForNodePoolCreate{} + +// errorInjectingSPNPCRUDForCreate wraps ServiceProviderNodePoolCRUD to allow error injection. +type errorInjectingSPNPCRUDForCreate struct { + database.ServiceProviderNodePoolCRUD + getErr error +} + +func (e *errorInjectingSPNPCRUDForCreate) Get(ctx context.Context, resourceID string) (*api.ServiceProviderNodePool, error) { + if e.getErr != nil { + return nil, e.getErr + } + return e.ServiceProviderNodePoolCRUD.Get(ctx, resourceID) +} + +var _ database.ServiceProviderNodePoolCRUD = &errorInjectingSPNPCRUDForCreate{} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_buildClusterEmptyNodePool(t *testing.T) { + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + maestroSourceEnvironmentIdentifier: "testenv", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + csClusterID := "11111111111111111111111111111111" + csClusterDomainPrefix := "test-domprefix" + csNodePoolID := "nodepool-id-1234" + np := syncer.buildClusterEmptyNodePool(csClusterID, csClusterDomainPrefix, csNodePoolID) + + assert.NotNil(t, np) + assert.Equal(t, "NodePool", np.Kind) + assert.Equal(t, hsv1beta1.SchemeGroupVersion.String(), np.APIVersion) + assert.Equal(t, csClusterDomainPrefix, np.Name) + assert.Equal(t, fmt.Sprintf("ocm-%s-%s-%s", syncer.maestroSourceEnvironmentIdentifier, csClusterID, csNodePoolID), np.Namespace) +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_getNodePoolNamespace(t *testing.T) { + envName := "testenv" + csClusterID := "11111111111111111111111111111111" + csNodePoolID := "nodepool-abc" + expected := fmt.Sprintf("ocm-%s-%s-%s", envName, csClusterID, csNodePoolID) + + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + maestroSourceEnvironmentIdentifier: envName, + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + result := syncer.getNodePoolNamespace(envName, csClusterID, csNodePoolID) + assert.Equal(t, expected, result) +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_buildInitialMaestroBundleReferenceForNodePool(t *testing.T) { + generator := maestro.NewMaestroAPIMaestroBundleNameGenerator() + bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "my-nodepool") + + ref, err := buildInitialMaestroBundleReference(bundleInternalName, generator) + require.NoError(t, err) + + assert.NotNil(t, ref) + assert.Equal(t, bundleInternalName, ref.Name) + assert.NotEmpty(t, ref.MaestroAPIMaestroBundleName) + assert.Empty(t, ref.MaestroAPIMaestroBundleID) + + // Verify the name is a valid UUID + _, err = uuid.Parse(ref.MaestroAPIMaestroBundleName) + assert.NoError(t, err, "MaestroAPIMaestroBundleName should be a valid UUID") +} + +func TestBuildInitialReadonlyMaestroBundleForNodePool(t *testing.T) { + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + maestroSourceEnvironmentIdentifier: "testenv", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + + csClusterDomainPrefix := "test-domprefix" + maestroBundleNamespacedName := types.NamespacedName{ + Name: "test-maestro-api-maestro-bundle-name", + Namespace: "test-maestro-consumer", + } + + bundle := syncer.buildInitialReadonlyMaestroBundleForNodePool(nodepool, csClusterDomainPrefix, maestroBundleNamespacedName) + require.NotNil(t, bundle) + + assert.Equal(t, "test-maestro-api-maestro-bundle-name", bundle.Name) + assert.Equal(t, "test-maestro-consumer", bundle.Namespace) + require.Len(t, bundle.Spec.Workload.Manifests, 1) + require.Len(t, bundle.Spec.ManifestConfigs, 1) + + // Verify manifest config + manifestConfig := bundle.Spec.ManifestConfigs[0] + assert.Equal(t, "nodepools", manifestConfig.ResourceIdentifier.Resource) + assert.Equal(t, hsv1beta1.SchemeGroupVersion.Group, manifestConfig.ResourceIdentifier.Group) + assert.Equal(t, csClusterDomainPrefix, manifestConfig.ResourceIdentifier.Name) + expectedNamespace := fmt.Sprintf("ocm-testenv-11111111111111111111111111111111-%s", strings.ToLower(nodepoolResourceID.Name)) + assert.Equal(t, expectedNamespace, manifestConfig.ResourceIdentifier.Namespace) + + expectedNodePool := syncer.buildClusterEmptyNodePool( + nodepool.ServiceProviderProperties.ClusterServiceID.ID(), + csClusterDomainPrefix, + nodepool.ID.Name, + ) + assert.Equal(t, expectedNodePool, bundle.Spec.Workload.Manifests[0].Object) +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_syncMaestroBundle(t *testing.T) { + syncMaestroBundleTestDeterministicUUID := uuid.MustParse("aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee") + var syncMaestroBundleTestOtherBundleName api.MaestroBundleInternalName = "otherReadonlyBundle" + + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "test-nodepool") + + tests := []struct { + name string + initialSPNP *api.ServiceProviderNodePool + maestroClientSetupMock func(*maestro.MockClient) + wantServiceProviderNodePool *api.ServiceProviderNodePool + wantErr bool + wantErrSubstr string + }{ + { + name: "existing reference but no ID - sets new ID and preserves name", + initialSPNP: &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "existing-bundle-name", + MaestroAPIMaestroBundleID: "", + }, + }, + }, + }, + maestroClientSetupMock: func(m *maestro.MockClient) { + createdBundle := &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{Name: "existing-bundle-name", Namespace: "test-consumer", UID: "new-bundle-uid"}, + } + m.EXPECT().Get(gomock.Any(), "existing-bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "not-found")) + m.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any()).Return(createdBundle, nil) + }, + wantServiceProviderNodePool: &api.ServiceProviderNodePool{ + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "existing-bundle-name", + MaestroAPIMaestroBundleID: "new-bundle-uid", + }, + }, + }, + }, + }, + { + name: "complete bundle reference - ID unchanged", + initialSPNP: &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "complete-bundle-name", + MaestroAPIMaestroBundleID: "complete-bundle-id", + }, + }, + }, + }, + maestroClientSetupMock: func(m *maestro.MockClient) { + existingBundle := &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{Name: "complete-bundle-name", Namespace: "test-consumer", UID: "complete-bundle-id"}, + } + m.EXPECT().Get(gomock.Any(), "complete-bundle-name", gomock.Any()).Return(existingBundle, nil) + }, + wantServiceProviderNodePool: &api.ServiceProviderNodePool{ + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "complete-bundle-name", + MaestroAPIMaestroBundleID: "complete-bundle-id", + }, + }, + }, + }, + }, + { + name: "multiple refs - only synced ref is updated, other refs unchanged", + initialSPNP: &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: syncMaestroBundleTestOtherBundleName, + MaestroAPIMaestroBundleName: "other-bundle-name", + MaestroAPIMaestroBundleID: "other-bundle-id", + }, + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "nodepool-bundle-name", + MaestroAPIMaestroBundleID: "", + }, + }, + }, + }, + maestroClientSetupMock: func(m *maestro.MockClient) { + createdBundle := &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{Name: "nodepool-bundle-name", Namespace: "test-consumer", UID: "nodepool-bundle-uid"}, + } + m.EXPECT().Get(gomock.Any(), "nodepool-bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "not-found")) + m.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any()).Return(createdBundle, nil) + }, + wantServiceProviderNodePool: &api.ServiceProviderNodePool{ + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: syncMaestroBundleTestOtherBundleName, + MaestroAPIMaestroBundleName: "other-bundle-name", + MaestroAPIMaestroBundleID: "other-bundle-id", + }, + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "nodepool-bundle-name", + MaestroAPIMaestroBundleID: "nodepool-bundle-uid", + }, + }, + }, + }, + }, + { + name: "maestro get or create error - returns last persisted SPNP", + initialSPNP: &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "bundle-name", + MaestroAPIMaestroBundleID: "", + }, + }, + }, + }, + maestroClientSetupMock: func(m *maestro.MockClient) { + m.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("maestro connection error")) + }, + wantServiceProviderNodePool: &api.ServiceProviderNodePool{ + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "bundle-name", + MaestroAPIMaestroBundleID: "", + }, + }, + }, + }, + wantErr: true, + wantErrSubstr: "failed to get or create Maestro Bundle", + }, + { + name: "no bundle reference initially - creates ref with deterministic UUID name and new ID", + initialSPNP: &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{}, + }, + }, + maestroClientSetupMock: func(m *maestro.MockClient) { + deterministicName := syncMaestroBundleTestDeterministicUUID.String() + createdBundle := &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{Name: deterministicName, Namespace: "test-consumer", UID: "new-bundle-uid"}, + } + m.EXPECT().Get(gomock.Any(), deterministicName, gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "not-found")) + m.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, mw *workv1.ManifestWork, opts metav1.CreateOptions) (*workv1.ManifestWork, error) { + assert.Equal(t, deterministicName, mw.Name) + assert.Equal(t, "test-consumer", mw.Namespace) + assert.Len(t, mw.Spec.Workload.Manifests, 1) + assert.Len(t, mw.Spec.ManifestConfigs, 1) + assert.Equal(t, workv1.UpdateStrategyTypeReadOnly, mw.Spec.ManifestConfigs[0].UpdateStrategy.Type) + return createdBundle, nil + }, + ) + }, + wantServiceProviderNodePool: &api.ServiceProviderNodePool{ + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: syncMaestroBundleTestDeterministicUUID.String(), + MaestroAPIMaestroBundleID: "new-bundle-uid", + }, + }, + }, + }, + }, + { + name: "no bundle ref initially - bundle name persisted then getOrCreate fails - returns SPNP with name set, no ID", + initialSPNP: &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{}, + }, + }, + maestroClientSetupMock: func(m *maestro.MockClient) { + deterministicName := syncMaestroBundleTestDeterministicUUID.String() + m.EXPECT().Get(gomock.Any(), deterministicName, gomock.Any()).Return(nil, fmt.Errorf("maestro connection error")) + }, + wantServiceProviderNodePool: &api.ServiceProviderNodePool{ + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: syncMaestroBundleTestDeterministicUUID.String(), + MaestroAPIMaestroBundleID: "", + }, + }, + }, + }, + wantErr: true, + wantErrSubstr: "failed to get or create Maestro Bundle", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + tt.maestroClientSetupMock(mockMaestro) + + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewAlwaysSameNameMaestroAPIMaestroBundleNameGenerator(syncMaestroBundleTestDeterministicUUID.String()), + } + ctx := context.Background() + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + + mockDB := databasetesting.NewMockDBClient() + spnpCRUD := mockDB.ServiceProviderNodePools("test-sub", "test-rg", "test-cluster", "test-nodepool") + createdSPNP, err := spnpCRUD.Create(ctx, tt.initialSPNP, nil) + require.NoError(t, err) + provisionShard := buildTestProvisionShard("test-consumer") + + result, err := syncer.syncMaestroBundle( + ctx, + bundleInternalName, + createdSPNP, + nodepool, + mockMaestro, + spnpCRUD, + provisionShard, + "test-domain", + ) + + assert.Equal(t, tt.wantErr, err != nil) + if tt.wantErr { + assert.Contains(t, err.Error(), tt.wantErrSubstr) + } + require.NotNil(t, result) + + wantList := tt.wantServiceProviderNodePool.Status.MaestroReadonlyBundles + gotList := result.Status.MaestroReadonlyBundles + require.Len(t, gotList, len(wantList), "result should have the same number of bundle refs as want") + for _, wantRef := range wantList { + gotRef, err := gotList.Get(wantRef.Name) + require.NoError(t, err) + require.NotNil(t, gotRef, "result missing bundle ref for name %q", wantRef.Name) + assert.Equal(t, wantRef.Name, gotRef.Name) + assert.Equal(t, wantRef.MaestroAPIMaestroBundleName, gotRef.MaestroAPIMaestroBundleName) + assert.Equal(t, wantRef.MaestroAPIMaestroBundleID, gotRef.MaestroAPIMaestroBundleID) + } + }) + } +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_NodePoolNotFound(t *testing.T) { + mockDBClient := databasetesting.NewMockDBClient() + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + // No nodepool in DB -> Get returns NotFound -> SyncOnce returns nil (no work to do) + err := syncer.SyncOnce(context.Background(), key) + assert.NoError(t, err) +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_GetServiceProviderNodePoolError(t *testing.T) { + ctx := context.Background() + + baseMockDB := databasetesting.NewMockDBClient() + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + + nodepoolsCRUD := baseMockDB.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + expectedError := fmt.Errorf("database error") + mockDBClient := &errorInjectingDBClientForNodePoolCreate{ + MockDBClient: baseMockDB, + spnpCRUD: &errorInjectingSPNPCRUDForCreate{ + getErr: expectedError, + }, + } + + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + err = syncer.SyncOnce(ctx, key) + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to get or create ServiceProviderNodePool") +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_AllBundlesAlreadySynced(t *testing.T) { + ctx := context.Background() + mockDBClient := databasetesting.NewMockDBClient() + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + nodepoolsCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "test-nodepool") + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "bundle-name", + MaestroAPIMaestroBundleID: "bundle-id", + }, + }, + }, + } + spnpCRUD := mockDBClient.ServiceProviderNodePools(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName, key.HCPNodePoolName) + _, err = spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + + // Since all bundles are synced, no cluster service or maestro calls should be made + err = syncer.SyncOnce(ctx, key) + assert.NoError(t, err) +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_SyncLoopExecutesWithBundleCreation(t *testing.T) { + ctrl := gomock.NewController(t) + ctx := context.Background() + + mockDBClient := databasetesting.NewMockDBClient() + mockClusterService := ocm.NewMockClusterServiceClientSpec(ctrl) + mockMaestroBuilder := maestro.NewMockMaestroClientBuilder(ctrl) + mockMaestroClient := maestro.NewMockClient(ctrl) + + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + clusterServiceClient: mockClusterService, + maestroClientBuilder: mockMaestroBuilder, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + nodepoolsCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + // SPNP with no bundle reference (needs syncing) + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{}, + }, + } + spnpCRUD := mockDBClient.ServiceProviderNodePools(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName, key.HCPNodePoolName) + _, err = spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + + provisionShard := buildTestProvisionShard("test-consumer") + mockClusterService.EXPECT(). + GetClusterProvisionShard(gomock.Any(), nodepool.ServiceProviderProperties.ClusterServiceID). + Return(provisionShard, nil) + + csCluster, err := arohcpv1alpha1.NewCluster(). + DomainPrefix("test-domain"). + Build() + require.NoError(t, err) + mockClusterService.EXPECT(). + GetCluster(gomock.Any(), nodepool.ServiceProviderProperties.ClusterServiceID). + Return(csCluster, nil) + + restEndpoint := provisionShard.MaestroConfig().RestApiConfig().Url() + grpcEndpoint := provisionShard.MaestroConfig().GrpcApiConfig().Url() + consumerName := provisionShard.MaestroConfig().ConsumerName() + sourceID := maestro.GenerateMaestroSourceID("test-env", provisionShard.ID()) + mockMaestroBuilder.EXPECT(). + NewClient(gomock.Any(), restEndpoint, grpcEndpoint, consumerName, sourceID). + Return(mockMaestroClient, nil) + + mockMaestroClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, k8serrors.NewNotFound(workv1.Resource("manifestwork"), "test-bundle")) + + createdBundle := &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "new-bundle-name", + Namespace: "test-consumer", + UID: "new-bundle-id", + }, + } + mockMaestroClient.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any()).Return(createdBundle, nil) + + err = syncer.SyncOnce(ctx, key) + require.NoError(t, err) + + updatedSPNP, err := spnpCRUD.Get(ctx, api.ServiceProviderNodePoolResourceName) + require.NoError(t, err) + require.NotNil(t, updatedSPNP) + + bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "test-nodepool") + bundleRef, err := updatedSPNP.Status.MaestroReadonlyBundles.Get(bundleInternalName) + require.NoError(t, err) + require.NotNil(t, bundleRef) + assert.NotEmpty(t, bundleRef.MaestroAPIMaestroBundleName) + assert.Equal(t, string(createdBundle.UID), bundleRef.MaestroAPIMaestroBundleID) +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_ProcessesPartiallySyncedBundles(t *testing.T) { + ctrl := gomock.NewController(t) + ctx := context.Background() + + mockDBClient := databasetesting.NewMockDBClient() + mockClusterService := ocm.NewMockClusterServiceClientSpec(ctrl) + mockMaestroBuilder := maestro.NewMockMaestroClientBuilder(ctrl) + mockMaestroClient := maestro.NewMockClient(ctrl) + + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + clusterServiceClient: mockClusterService, + maestroClientBuilder: mockMaestroBuilder, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + nodepoolsCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "test-nodepool") + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: api.MaestroBundleInternalName("otherReadonlyBundle"), + MaestroAPIMaestroBundleName: "other-bundle-name", + MaestroAPIMaestroBundleID: "other-bundle-id", // fully synced - never touched + }, + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "nodepool-bundle-name", + MaestroAPIMaestroBundleID: "", // partially synced - syncMaestroBundle will be called + }, + }, + }, + } + spnpCRUD := mockDBClient.ServiceProviderNodePools(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName, key.HCPNodePoolName) + _, err = spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + + provisionShard := buildTestProvisionShard("test-consumer") + mockClusterService.EXPECT(). + GetClusterProvisionShard(gomock.Any(), nodepool.ServiceProviderProperties.ClusterServiceID). + Return(provisionShard, nil) + + csCluster, err := arohcpv1alpha1.NewCluster().DomainPrefix("test-domain").Build() + require.NoError(t, err) + mockClusterService.EXPECT(). + GetCluster(gomock.Any(), nodepool.ServiceProviderProperties.ClusterServiceID). + Return(csCluster, nil) + + restEndpoint := provisionShard.MaestroConfig().RestApiConfig().Url() + grpcEndpoint := provisionShard.MaestroConfig().GrpcApiConfig().Url() + consumerName := provisionShard.MaestroConfig().ConsumerName() + sourceID := maestro.GenerateMaestroSourceID("test-env", provisionShard.ID()) + mockMaestroBuilder.EXPECT(). + NewClient(gomock.Any(), restEndpoint, grpcEndpoint, consumerName, sourceID). + Return(mockMaestroClient, nil) + + // Only the partially-synced bundle triggers a sync; maestro Get fails so we get an error + mockMaestroClient.EXPECT(). + Get(gomock.Any(), "nodepool-bundle-name", gomock.Any()). + Return(nil, fmt.Errorf("maestro API error")) + + err = syncer.SyncOnce(ctx, key) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to sync Maestro Bundle") + assert.Contains(t, err.Error(), "maestro API error") +} diff --git a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go index 6c563947560..4676d810e33 100644 --- a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go @@ -26,7 +26,6 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/workqueue" - "k8s.io/utils/ptr" "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" "github.com/Azure/ARO-HCP/backend/pkg/maestro" @@ -52,7 +51,8 @@ type deleteOrphanedMaestroReadonlyBundles struct { maestroSourceEnvironmentIdentifier string } -// NewDeleteOrphanedMaestroReadonlyBundlesController periodically looks for cosmos objs that don't have an owning cluster and deletes them. +// NewDeleteOrphanedMaestroReadonlyBundlesController periodically looks for Maestro readonly bundles in the Maestro API that are not referenced +// by any of the supported cosmos resources by this controller and deletes them. func NewDeleteOrphanedMaestroReadonlyBundlesController(cosmosClient database.DBClient, csClient ocm.ClusterServiceClientSpec, maestroClientBuilder maestro.MaestroClientBuilder, maestroSourceEnvironmentIdentifier string) controllerutils.Controller { c := &deleteOrphanedMaestroReadonlyBundles{ name: "DeleteOrphanedMaestroReadonlyBundles", @@ -83,16 +83,24 @@ func NewDeleteOrphanedMaestroReadonlyBundlesController(cosmosClient database.DBC // 2.4 List all documents again (fresh snapshot), rebuild the map from it in the same way as 2.2. // 2.5 For each candidate, if the bundle name is still not referenced on that shard in the fresh snapshot, delete it via Maestro // bundle deletion API. -// -// Cross-store: The fresh Cosmos snapshots and per-shard reference sets prevent deleting a bundle that is already referenced -// in committed documents by the time that snapshot is built, so a stale initial list alone does not cause accidental delete. +// Cross-store: The fresh SPC list and per-shard reference set (steps 5-6) prevent deleting a bundle that is already referenced +// in committed Cosmos documents by the time that snapshot is built, so a stale initial list alone does not cause accidental +// delete. +// // IMPORTANT NOTE: This assumes that the maestro server associated to the provision shard // has resources with always the same source ID. If it turns out we cannot have this assumption this logic would not // be good enough. In that case it might be necessary to store to what source ID a Maestro Bundle/set of Maestro Bundles // belongs to but then the instantiation of the Maestro client needs to be done differently as its scoped to // Maestro Consumer Name + Maestro Source ID. We know for example that in the CSPR environment different CS instances // have different Maestro source IDs using the same Maestro Server. +// +// Note: We considered using the Maestro API Maestro UID which is globally unique but it's possible that there's a scenario +// where a maestro create readonly bundles controller creates a bundle, then creates the Maestro bundle using the Maestro API +// but then for some reason fails to persist it in the database, which in that case the cluster ended up being deleted by the +// orphan controller accidentally. In that scenario we would not have the Maestro UID to identify the Maestro Bundle and +// we would not be able to delete it. Furthermore we should not use the fact of the UID being empty as the trigger +// to delete because it could be that it's being created and not yet persisted in Cosmos. func (c *deleteOrphanedMaestroReadonlyBundles) SyncOnce(ctx context.Context, _ any) error { logger := utils.LoggerFromContext(ctx) logger.Info("Syncing orphaned Maestro Readonly Bundles") @@ -106,40 +114,15 @@ func (c *deleteOrphanedMaestroReadonlyBundles) SyncOnce(ctx context.Context, _ a } logger.Info(fmt.Sprintf("Built Maestro clients for %d provision shards", len(maestroClientsByShard))) - initialServiceProviderClusters, err := c.getAllServiceProviderClusters(ctx) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to get all ServiceProviderClusters: %w", err)) - } - logger.Info(fmt.Sprintf("Found %d ServiceProviderClusters (initial)", len(initialServiceProviderClusters))) - - logger.Info("Mapping initial ServiceProviderClusters to provision shards") - initialShardToSPCs, err := c.mapServiceProviderClustersByProvisionShard(ctx, initialServiceProviderClusters, maestroClientsByShard) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to map ServiceProviderClusters to provision shards: %w", err)) - } - logger.Info(fmt.Sprintf("Initial ServiceProviderClusters mapped to %d provision shards", len(initialShardToSPCs))) - - allServiceProviderNodePools, err := c.getAllServiceProviderNodePools(ctx) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to get all ServiceProviderNodePools: %w", err)) - } - logger.Info(fmt.Sprintf("Found %d ServiceProviderNodePools (initial)", len(allServiceProviderNodePools))) - - logger.Info("Mapping initial ServiceProviderNodePools to provision shards") - initialShardToSPNPs, err := c.mapServiceProviderNodePoolsByProvisionShard(ctx, allServiceProviderNodePools, maestroClientsByShard) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to map ServiceProviderNodePools to provision shards: %w", err)) - } - logger.Info(fmt.Sprintf("Initial ServiceProviderNodePools mapped to %d provision shards", len(initialShardToSPNPs))) - var syncErrors []error - logger.Info("Ensuring orphaned cluster-scoped Maestro Readonly Bundles are deleted") - if err := c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, maestroClientsByShard, initialShardToSPCs); err != nil { + + logger.Info("Ensuring orphaned cluster scoped Maestro Readonly Bundles are deleted") + if err := c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, maestroClientsByShard); err != nil { syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to ensure orphaned cluster-scoped Maestro Bundles are deleted: %w", err))) } - logger.Info("Ensuring orphaned nodepool-scoped Maestro Readonly Bundles are deleted") - if err := c.ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted(ctx, maestroClientsByShard, initialShardToSPNPs); err != nil { + logger.Info("Ensuring orphaned nodepool scoped Maestro Readonly Bundles are deleted") + if err := c.ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted(ctx, maestroClientsByShard); err != nil { syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to ensure orphaned nodepool-scoped Maestro Bundles are deleted: %w", err))) } @@ -147,67 +130,44 @@ func (c *deleteOrphanedMaestroReadonlyBundles) SyncOnce(ctx context.Context, _ a return errors.Join(syncErrors...) } -// getAllServiceProviderClusters returns the list of all ServiceProviderClusters in the database. +// ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted ensures that Maestro readonly bundles managed by the cluster-scoped +// controller are deleted when no ServiceProviderCluster on that provision shard references them. +func (c *deleteOrphanedMaestroReadonlyBundles) ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient) error { + logger := utils.LoggerFromContext(ctx) + logger = logger.WithValues("maestroReadonlyBundleReferencesResourceType", api.ServiceProviderClusterResourceType) + ctx = utils.ContextWithLogger(ctx, logger) + + return c.ensureOrphanedReadonlyBundlesDeleted(ctx, maestroClientsByShard, readonlyBundleManagedByK8sLabelValueClusterScoped, c.clusterScopedPersistedMaestroBundleRefsByShardFromCosmos) +} + +// ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted ensures that Maestro readonly bundles managed by the +// nodepool-scoped controller are deleted when no ServiceProviderNodePool on that provision shard references them. +func (c *deleteOrphanedMaestroReadonlyBundles) ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient) error { + logger := utils.LoggerFromContext(ctx) + logger = logger.WithValues("maestroReadonlyBundleReferencesResourceType", api.ServiceProviderNodePoolResourceType) + ctx = utils.ContextWithLogger(ctx, logger) + + return c.ensureOrphanedReadonlyBundlesDeleted(ctx, maestroClientsByShard, readonlyBundleManagedByK8sLabelValueNodePoolScoped, c.nodePoolScopedPersistedMaestroBundleRefsByShardFromCosmos) +} + +// getAllServiceProviderClusters returns all ServiceProviderClusters via database.ListAll. func (c *deleteOrphanedMaestroReadonlyBundles) getAllServiceProviderClusters(ctx context.Context) ([]*api.ServiceProviderCluster, error) { - // We list all ServiceProviderClusters in chunks of 500 to avoid putting + // We list all ServiceProviderClusters in chunks of 500 at most to avoid putting // too much pressure on the Cosmos DB. // Any failure to iterate over the ServiceProviderclusters ends the sync process because otherwise // we would not have the complete information to evaluate the deletion and we could // accidentally delete Maestro Bundles that are still in use. - listOptions := &database.DBClientListResourceDocsOptions{ - PageSizeHint: ptr.To(int32(500)), - } - allServiceProviderClusters := []*api.ServiceProviderCluster{} - for { - iterator, err := c.cosmosClient.GlobalListers().ServiceProviderClusters().List(ctx, listOptions) - if err != nil { - return nil, utils.TrackError(fmt.Errorf("failed to list ServiceProviderClusters: %w", err)) - } - for _, spc := range iterator.Items(ctx) { - allServiceProviderClusters = append(allServiceProviderClusters, spc) - } - err = iterator.GetError() - if err != nil { - return nil, utils.TrackError(fmt.Errorf("failed iterating ServiceProviderClusters: %w", err)) - } - - continuationToken := iterator.GetContinuationToken() - if continuationToken == "" { - break - } - listOptions.ContinuationToken = &continuationToken - } - - return allServiceProviderClusters, nil + return database.ListAll(ctx, 500, c.cosmosClient.GlobalListers().ServiceProviderClusters().List) } -// getAllServiceProviderNodePools returns the list of all ServiceProviderNodePools in the database. +// getAllServiceProviderNodePools returns all ServiceProviderNodePools via database.ListAll. func (c *deleteOrphanedMaestroReadonlyBundles) getAllServiceProviderNodePools(ctx context.Context) ([]*api.ServiceProviderNodePool, error) { - listOptions := &database.DBClientListResourceDocsOptions{ - PageSizeHint: ptr.To(int32(500)), - } - allServiceproviderNodePools := []*api.ServiceProviderNodePool{} - for { - iterator, err := c.cosmosClient.GlobalListers().ServiceProviderNodePools().List(ctx, listOptions) - if err != nil { - return nil, utils.TrackError(fmt.Errorf("failed to list ServiceProviderNodePools: %w", err)) - } - for _, spnp := range iterator.Items(ctx) { - allServiceproviderNodePools = append(allServiceproviderNodePools, spnp) - } - err = iterator.GetError() - if err != nil { - return nil, utils.TrackError(fmt.Errorf("failed iterating ServiceProviderNodePools: %w", err)) - } - - continuationToken := iterator.GetContinuationToken() - if continuationToken == "" { - break - } - listOptions.ContinuationToken = &continuationToken - } - - return allServiceproviderNodePools, nil + // We list all ServiceProviderNodePools in chunks of 500 at most to avoid putting + // too much pressure on the Cosmos DB. + // Any failure to iterate over the ServiceProviderNodePools ends the sync process because otherwise + // we would not have the complete information to evaluate the deletion and we could + // accidentally delete Maestro Bundles that are still in use. + return database.ListAll(ctx, 500, c.cosmosClient.GlobalListers().ServiceProviderNodePools().List) } // shardMaestroClient holds a Maestro API client for one Cluster Service provision shard and its teardown cancel func. @@ -225,7 +185,6 @@ func cancelMaestroClientsByProvisionShard(maestroClientsByProvisionShard map[str // buildMaestroClientsByProvisionShard lists registered provision shards from Cluster Service and builds a map of // provision shard ID to Maestro client. The key of the map is the CS provision shard ID. -// // On error the returned map may be partial (clients created before the error). The caller must defer cancelMaestroClientsByProvisionShard unconditionally. func (c *deleteOrphanedMaestroReadonlyBundles) buildMaestroClientsByProvisionShard(ctx context.Context) (map[string]*shardMaestroClient, error) { maestroClientsByProvisionShard := map[string]*shardMaestroClient{} @@ -234,7 +193,8 @@ func (c *deleteOrphanedMaestroReadonlyBundles) buildMaestroClientsByProvisionSha // the information in Cosmos and this should be changed to use that instead. // TODO should we take into account the provision shard status on what to consider (active, maintenance, offline, ...)? // for now we consider all provision shards independently of their status. - for provisionShard := range c.clusterServiceClient.ListProvisionShards().Items(ctx) { + provisionShardIter := c.clusterServiceClient.ListProvisionShards() + for provisionShard := range provisionShardIter.Items(ctx) { // We create a new context with a cancel function so we can cancel the Maestro client when the sync is done. // This is important to avoid leaking resources when the sync is done. maestroClientCtx, cancel := context.WithCancel(ctx) @@ -248,6 +208,10 @@ func (c *deleteOrphanedMaestroReadonlyBundles) buildMaestroClientsByProvisionSha maestroClientCancelFunc: cancel, } } + err := provisionShardIter.GetError() + if err != nil { + return maestroClientsByProvisionShard, utils.TrackError(fmt.Errorf("failed to list Cluster Service provision shards: %w", err)) + } return maestroClientsByProvisionShard, nil } @@ -286,6 +250,14 @@ func (c *deleteOrphanedMaestroReadonlyBundles) mapServiceProviderNodePoolsByProv return res, nil } +// maestroBundleNamesByShard maps Cluster Service provision shard IDs to a set of Maestro API Maestro bundle names +// The outer map key is the Cluster Service provision shard ID. The inner map key is the Maestro API Maestro bundle name. +// The inner map value is a struct{} to indicate the presence of the bundle name. +type maestroBundleNamesByShard map[string]map[string]struct{} + +// maestroBundleNamesByShardRetrieverFunc retrieves a map of Cluster Service provision shard IDs to a set of Maestro API Maestro bundle names. +type maestroBundleNamesByShardRetrieverFunc func(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient) (bundleNamesByShard maestroBundleNamesByShard, err error) + // orphanReadonlyBundleDeleteCandidate is a Maestro bundle listed on a provision shard that was not referenced by the // initial snapshot for that shard; delete still requires a fresh snapshot check. type orphanReadonlyBundleDeleteCandidate struct { @@ -293,204 +265,164 @@ type orphanReadonlyBundleDeleteCandidate struct { bundle *workv1.ManifestWork } -// ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted ensures that Maestro readonly bundles managed by the cluster-scoped -// controller are deleted when no ServiceProviderCluster on that provision shard references them. -// -// 1. From initialShardToSPCs, build per-shard sets of referenced Maestro bundle names. -// 2. For each shard with a Maestro client, list bundles (paginated) and add candidates when the bundle is not referenced on that shard. -// 3. List all ServiceProviderClusters again (fresh), map them by shard, rebuild referenced sets. -// 4. Delete each candidate that is still unreferenced on its shard in the fresh snapshot. -func (c *deleteOrphanedMaestroReadonlyBundles) ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient, initialShardToSPCs map[string][]*api.ServiceProviderCluster) error { +// listOrphanReadonlyBundleCandidates lists Maestro bundles for each shard using the Maestro API +// and it returns a list of candidate orphan maestro readonly bundles for deletion. The criteria to consider a +// maestro readonly bundle as a candidate for deletion is that it matches the managedByLabelValue and is not +// referenced in bundleNamesByShard for that shard. +func (c *deleteOrphanedMaestroReadonlyBundles) listOrphanReadonlyBundleCandidates(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient, + bundleNamesByShard maestroBundleNamesByShard, managedByLabelValue string, +) ([]orphanReadonlyBundleDeleteCandidate, error) { logger := utils.LoggerFromContext(ctx) - var syncErrors []error - - referencedByShardInitial, err := referencedMaestroAPIMaestroBundleNamesByShard(initialShardToSPCs) - if err != nil { - return utils.TrackError(fmt.Errorf("error building referenced Maestro API Maestro bundle names by shard (initial snapshot): %w", err)) - } - var deleteCandidates []orphanReadonlyBundleDeleteCandidate for csShardID, shardEntry := range maestroClientsByShard { shardLogger := logger.WithValues("csProvisionShardID", csShardID) ctxShard := utils.ContextWithLogger(ctx, shardLogger) - initialOnShard := initialShardToSPCs[csShardID] - shardLogger.Info(fmt.Sprintf("listing cluster-scoped Maestro bundles on cluster service provision shard %s (%d ServiceProviderClusters in initial shard map)", csShardID, len(initialOnShard))) maestroClient := shardEntry.maestroClient - listOptions := metav1.ListOptions{Limit: 400, Continue: "", LabelSelector: fmt.Sprintf("%s=%s", readonlyBundleManagedByK8sLabelKey, readonlyBundleManagedByK8sLabelValueClusterScoped)} - for { - maestroBundles, err := maestroClient.List(ctxShard, listOptions) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to list Maestro Bundles for shard %s: %w", csShardID, err)) + // We list all the Maestro Bundles in chunks of 400 to avoid putting too much pressure on the Maestro API. + // We filter by the K8s label that identifies which controller manages the bundle. + listOpts := metav1.ListOptions{ + Limit: 400, + LabelSelector: fmt.Sprintf("%s=%s", readonlyBundleManagedByK8sLabelKey, managedByLabelValue), + } + err := maestro.ForEachMaestroBundle(ctxShard, maestroClient, listOpts, func(maestroBundle *workv1.ManifestWork) error { + // Even though Maestro should filter by the K8s label we specified we double check it here to be sure. + if maestroBundle.Labels[readonlyBundleManagedByK8sLabelKey] != managedByLabelValue { + return nil } - for i := range maestroBundles.Items { - maestroBundle := &maestroBundles.Items[i] - // Even though Maestro should filter by the K8s label we specified we double check it here to be sure - if maestroBundle.Labels[readonlyBundleManagedByK8sLabelKey] != readonlyBundleManagedByK8sLabelValueClusterScoped { - continue - } - // We check if the Maestro bundle is referenced by any of the ServiceProviderClusters on the shard in the initial snapshot. - // If it is referenced we skip it as it is not an orphan. - // The Maestro API Maestro Bundle Name should be unique within a given Maestro Consumer Name and Maestro Source ID. - if shardRefSet := referencedByShardInitial[csShardID]; shardRefSet != nil { - if _, referenced := shardRefSet[maestroBundle.Name]; referenced { - continue - } + // Check if the bundle is referenced by any resource allocated to this shard. + if shardRefSet := bundleNamesByShard[csShardID]; shardRefSet != nil { + if _, referenced := shardRefSet[maestroBundle.Name]; referenced { + // The Maestro API Maestro Bundle Name should be unique within a given Maestro Consumer Name and Maestro Source ID. + // If we find a match, it means the Maestro Bundle is referenced and we should not delete it. + return nil } - deleteCandidates = append(deleteCandidates, orphanReadonlyBundleDeleteCandidate{ - csShardID: csShardID, - bundle: maestroBundle, - }) } - continuationToken := maestroBundles.GetContinue() - if continuationToken == "" { - break - } - listOptions.Continue = continuationToken + deleteCandidates = append(deleteCandidates, orphanReadonlyBundleDeleteCandidate{ + csShardID: csShardID, + bundle: maestroBundle, + }) + return nil + }) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to list Maestro Bundles for shard %s: %w", csShardID, err)) } } - freshServiceProviderClusters, err := c.getAllServiceProviderClusters(ctx) + return deleteCandidates, nil +} + +// clusterScopedPersistedMaestroBundleRefsByShardFromCosmos lists ServiceProviderClusters from Cosmos, maps them by +// provision shard, and returns referenced Maestro API bundle names per shard. +func (c *deleteOrphanedMaestroReadonlyBundles) clusterScopedPersistedMaestroBundleRefsByShardFromCosmos(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient) (maestroBundleNamesByShard, error) { + clusters, err := c.getAllServiceProviderClusters(ctx) if err != nil { - return utils.TrackError(fmt.Errorf("error getting all ServiceProviderClusters (fresh snapshot): %w", err)) + return nil, utils.TrackError(fmt.Errorf("failed to get all ServiceProviderClusters: %w", err)) } - freshShardToSPCs, err := c.mapServiceProviderClustersByProvisionShard(ctx, freshServiceProviderClusters, maestroClientsByShard) + shardDocs, err := c.mapServiceProviderClustersByProvisionShard(ctx, clusters, maestroClientsByShard) if err != nil { - return utils.TrackError(fmt.Errorf("error mapping fresh ServiceProviderClusters to provision shards (fresh snapshot): %w", err)) + return nil, utils.TrackError(fmt.Errorf("failed to map ServiceProviderClusters to provision shards: %w", err)) } - referencedByShardFresh, err := referencedMaestroAPIMaestroBundleNamesByShard(freshShardToSPCs) + refs, err := c.buildClusterScopedMaestroAPIMaestroBundleNamesByShard(shardDocs) if err != nil { - return utils.TrackError(fmt.Errorf("error building referenced Maestro API Maestro bundle names by shard (fresh snapshot): %w", err)) + return nil, utils.TrackError(fmt.Errorf("error building cluster scoped Maestro API Maestro bundle names by shard: %w", err)) } - - for _, cand := range deleteCandidates { - csShardID := cand.csShardID - candidateMaestroBundle := cand.bundle - shardEntry, ok := maestroClientsByShard[csShardID] - if !ok { - syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("no Maestro client for shard %s when deleting bundle %q", csShardID, candidateMaestroBundle.Name))) - continue - } - maestroClient := shardEntry.maestroClient - - shardLogger := utils.LoggerFromContext(ctx).WithValues("csProvisionShardID", csShardID) - ctxShard := utils.ContextWithLogger(ctx, shardLogger) - if shardRefSet := referencedByShardFresh[csShardID]; shardRefSet != nil { - if _, referenced := shardRefSet[candidateMaestroBundle.Name]; referenced { - // If the Maestro bundle is referenced by any of the ServiceProviderClusters on the shard in the fresh snapshot we skip it as it is not an orphan. - continue - } - } - - shardLogger.Info("Deleting orphaned Maestro readonly Bundle", "maestroConsumerName", candidateMaestroBundle.Namespace, "maestroAPIMaestroBundleName", candidateMaestroBundle.Name, "maestroAPIMaestroBundleID", candidateMaestroBundle.UID) - err = maestroClient.Delete(ctxShard, candidateMaestroBundle.Name, metav1.DeleteOptions{}) - if err != nil { - // Failure to delete does not end the sync process. We log the error and we continue with the processing of other Maestro bundle deletion candidates. - syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to delete Maestro Bundle: %w", err))) - } else { - shardLogger.Info("Deleted orphaned Maestro readonly Bundle", "maestroConsumerName", candidateMaestroBundle.Namespace, "maestroAPIMaestroBundleName", candidateMaestroBundle.Name, "maestroAPIMaestroBundleID", candidateMaestroBundle.UID) - } - } - - return errors.Join(syncErrors...) + return refs, nil } -// ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted ensures that Maestro readonly bundles managed by the -// nodepool-scoped controller are deleted when no ServiceProviderNodePool on that provision shard references them. -// It uses the same two-snapshot pattern as ensureOrphanedMaestroReadonlyBundlesAreDeleted. -func (c *deleteOrphanedMaestroReadonlyBundles) ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient, initialShardToSPNPs map[string][]*api.ServiceProviderNodePool) error { - logger := utils.LoggerFromContext(ctx) - var syncErrors []error - - referencedByShardInitial, err := referencedMaestroAPIMaestroBundleNamesByShardFromNodePools(initialShardToSPNPs) +// nodePoolScopedPersistedMaestroBundleRefsByShardFromCosmos lists ServiceProviderNodePools from Cosmos, maps them by +// provision shard, and returns referenced Maestro API bundle names per shard. +func (c *deleteOrphanedMaestroReadonlyBundles) nodePoolScopedPersistedMaestroBundleRefsByShardFromCosmos(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient) (maestroBundleNamesByShard, error) { + pools, err := c.getAllServiceProviderNodePools(ctx) if err != nil { - return utils.TrackError(fmt.Errorf("error building referenced Maestro API Maestro bundle names by shard from node pools (initial snapshot): %w", err)) + return nil, utils.TrackError(fmt.Errorf("failed to get all ServiceProviderNodePools: %w", err)) } + shardDocs, err := c.mapServiceProviderNodePoolsByProvisionShard(ctx, pools, maestroClientsByShard) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to map ServiceProviderNodePools to provision shards: %w", err)) + } + refs, err := c.buildNodePoolScopedMaestroAPIMaestroBundleNamesByShard(shardDocs) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("error building nodepool scoped Maestro API Maestro bundle names by shard: %w", err)) + } + return refs, nil +} - var deleteCandidates []orphanReadonlyBundleDeleteCandidate - - for csShardID, shardEntry := range maestroClientsByShard { - shardLogger := logger.WithValues("csProvisionShardID", csShardID) - ctxShard := utils.ContextWithLogger(ctx, shardLogger) - initialOnShard := initialShardToSPNPs[csShardID] - shardLogger.Info(fmt.Sprintf("listing nodepool-scoped Maestro bundles on cluster service provision shard %s (%d ServiceProviderNodePools in initial shard map)", csShardID, len(initialOnShard))) - maestroClient := shardEntry.maestroClient - listOptions := metav1.ListOptions{Limit: 400, Continue: "", LabelSelector: fmt.Sprintf("%s=%s", readonlyBundleManagedByK8sLabelKey, readonlyBundleManagedByK8sLabelValueNodePoolScoped)} - for { - maestroBundles, err := maestroClient.List(ctxShard, listOptions) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to list nodepool-scoped Maestro Bundles for shard %s: %w", csShardID, err)) - } - for i := range maestroBundles.Items { - maestroBundle := &maestroBundles.Items[i] - // Even though Maestro should filter by the K8s label we specified we double check it here to be sure - if maestroBundle.Labels[readonlyBundleManagedByK8sLabelKey] != readonlyBundleManagedByK8sLabelValueNodePoolScoped { - continue - } - // We check if the Maestro bundle is referenced by any of the ServiceProviderNodePools on the shard in the initial snapshot. - // If it is referenced we skip it as it is not an orphan. - // The Maestro API Maestro Bundle Name should be unique within a given Maestro Consumer Name and Maestro Source ID. - if shardRefSet := referencedByShardInitial[csShardID]; shardRefSet != nil { - if _, referenced := shardRefSet[maestroBundle.Name]; referenced { - continue - } - } - deleteCandidates = append(deleteCandidates, orphanReadonlyBundleDeleteCandidate{ - csShardID: csShardID, - bundle: maestroBundle, - }) - } - continuationToken := maestroBundles.GetContinue() - if continuationToken == "" { - break - } - listOptions.Continue = continuationToken - } +// ensureOrphanedReadonlyBundlesDeleted runs the two-phase list/compare/delete flow shared by every Cosmos-backed resource type +// that can reference Maestro readonly bundles: +// 1. Initial Cosmos snapshot of all the instances of that resource type that can reference Maestro readonly bundles +// 2. For each shard, use its Maestro client and list the maestro bundles including them as deletion candidates +// when the bundle is not referenced on that shard and the readonlyBundleManagedByK8sLabelKey label matches the managedByLabelValue +// 3. Retrieve a fresh snapshot of all the instances of that resource type that can reference Maestro readonly bundles +// 4. Delete each candidate that is still unreferenced on that shard in the fresh snapshot +func (c *deleteOrphanedMaestroReadonlyBundles) ensureOrphanedReadonlyBundlesDeleted( + ctx context.Context, + maestroClientsByShard map[string]*shardMaestroClient, + managedByLabelValue string, + persistedMaestroBundleRefsByShardRetriever maestroBundleNamesByShardRetrieverFunc, +) error { + initialPersistedMaestroBundlesByShard, err := persistedMaestroBundleRefsByShardRetriever(ctx, maestroClientsByShard) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to retrieve initial persisted Maestro bundle references by shard: %w", err)) } - freshServiceProviderNodePools, err := c.getAllServiceProviderNodePools(ctx) + deleteCandidates, err := c.listOrphanReadonlyBundleCandidates(ctx, maestroClientsByShard, initialPersistedMaestroBundlesByShard, managedByLabelValue) if err != nil { - return utils.TrackError(fmt.Errorf("error getting all ServiceProviderNodePools (fresh snapshot): %w", err)) + return utils.TrackError(fmt.Errorf("failed to list orphaned Maestro readonly bundle candidates: %w", err)) } - freshShardToSPNPs, err := c.mapServiceProviderNodePoolsByProvisionShard(ctx, freshServiceProviderNodePools, maestroClientsByShard) + + freshPersistedMaestroBundlesByShard, err := persistedMaestroBundleRefsByShardRetriever(ctx, maestroClientsByShard) if err != nil { - return utils.TrackError(fmt.Errorf("error mapping fresh ServiceProviderNodePools to provision shards (fresh snapshot): %w", err)) + return utils.TrackError(fmt.Errorf("failed to retrieve fresh persisted Maestro bundle references by shard: %w", err)) } - referencedByShardFresh, err := referencedMaestroAPIMaestroBundleNamesByShardFromNodePools(freshShardToSPNPs) + + err = c.conditionallyDeleteOrphanReadonlyBundleCandidates(ctx, maestroClientsByShard, deleteCandidates, freshPersistedMaestroBundlesByShard) if err != nil { - return utils.TrackError(fmt.Errorf("error building referenced Maestro API Maestro bundle names by shard from node pools (fresh snapshot): %w", err)) + return utils.TrackError(fmt.Errorf("failed to delete orphaned Maestro readonly bundle candidates: %w", err)) } - for _, cand := range deleteCandidates { + return nil +} + +// conditionallyDeleteOrphanReadonlyBundleCandidates processes a list of Maestro readonly bundle delete candidates. An +// orphan Maestro readonly bundle delete candidate is deleted only if it is not referenced in persistedMaestroBundlesByShard +// for the same shard. +func (c *deleteOrphanedMaestroReadonlyBundles) conditionallyDeleteOrphanReadonlyBundleCandidates( + ctx context.Context, + maestroClientsByShard map[string]*shardMaestroClient, + candidates []orphanReadonlyBundleDeleteCandidate, + persistedMaestroBundlesByShard maestroBundleNamesByShard, +) error { + var syncErrors []error + for _, cand := range candidates { csShardID := cand.csShardID candidateMaestroBundle := cand.bundle shardEntry, ok := maestroClientsByShard[csShardID] if !ok { - syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("no Maestro client for shard %s when deleting nodepool bundle %q", csShardID, candidateMaestroBundle.Name))) + syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("no Maestro client for shard %s when deleting bundle %q", csShardID, candidateMaestroBundle.Name))) continue } maestroClient := shardEntry.maestroClient shardLogger := utils.LoggerFromContext(ctx).WithValues("csProvisionShardID", csShardID) ctxShard := utils.ContextWithLogger(ctx, shardLogger) - if shardRefSet := referencedByShardFresh[csShardID]; shardRefSet != nil { + if shardRefSet := persistedMaestroBundlesByShard[csShardID]; shardRefSet != nil { if _, referenced := shardRefSet[candidateMaestroBundle.Name]; referenced { - // If the Maestro bundle is referenced by any of the ServiceProviderClusters on the shard in the fresh snapshot we skip it as it is not an orphan. + // If the Maestro bundle is referenced by any persisted Maestro bundle references for that shard, skip it. continue } } - shardLogger.Info("Deleting orphaned nodepool-scoped Maestro readonly Bundle", "maestroConsumerName", candidateMaestroBundle.Namespace, "maestroAPIMaestroBundleName", candidateMaestroBundle.Name, "maestroAPIMaestroBundleID", candidateMaestroBundle.UID) - err = maestroClient.Delete(ctxShard, candidateMaestroBundle.Name, metav1.DeleteOptions{}) + shardLogger.Info("Deleting orphaned Maestro readonly Bundle", "maestroConsumerName", candidateMaestroBundle.Namespace, "maestroAPIMaestroBundleName", candidateMaestroBundle.Name, "maestroAPIMaestroBundleID", candidateMaestroBundle.UID) + err := maestroClient.Delete(ctxShard, candidateMaestroBundle.Name, metav1.DeleteOptions{}) if err != nil { // Failure to delete does not end the sync process. We log the error and we continue with the processing of other Maestro bundle deletion candidates. - syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to delete nodepool-scoped Maestro Bundle: %w", err))) + syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to delete Maestro Bundle: %w", err))) } else { - shardLogger.Info("Deleted orphaned nodepool-scoped Maestro readonly Bundle", "maestroConsumerName", candidateMaestroBundle.Namespace, "maestroAPIMaestroBundleName", candidateMaestroBundle.Name, "maestroAPIMaestroBundleID", candidateMaestroBundle.UID) + shardLogger.Info("Deleted orphaned Maestro readonly Bundle", "maestroConsumerName", candidateMaestroBundle.Namespace, "maestroAPIMaestroBundleName", candidateMaestroBundle.Name, "maestroAPIMaestroBundleID", candidateMaestroBundle.UID) } } - - return errors.Join(syncErrors...) + return utils.TrackError(errors.Join(syncErrors...)) } // clusterProvisionShardIDForServiceProviderCluster returns the Cluster Service provision shard ID for the cluster that owns the SPC. @@ -529,11 +461,6 @@ func (c *deleteOrphanedMaestroReadonlyBundles) clusterProvisionShardIDForService if err != nil { return "", utils.TrackError(fmt.Errorf("failed to get Cluster: %w", err)) } - // TODO We get the provision shard ID from CS but at some point we should have - // the information in Cosmos and this should be changed to use that instead. - // TODO should we take into account that at some point in the future we will implement migration between management - // clusters, where a cluster could have bundles allocated to different provision shards at the same time? For now - // we assume that the cluster is associated to a single provision shard at a time. clusterCSShard, err := c.clusterServiceClient.GetClusterProvisionShard(ctx, cluster.ServiceProviderProperties.ClusterServiceID) if err != nil { return "", utils.TrackError(fmt.Errorf("failed to get Cluster Provision Shard: %w", err)) @@ -541,11 +468,11 @@ func (c *deleteOrphanedMaestroReadonlyBundles) clusterProvisionShardIDForService return clusterCSShard.ID(), nil } -// referencedMaestroAPIMaestroBundleNamesByShard maps provision shard ID to the set of Maestro API bundle names referenced by -// SPCs grouped under that shard (shard assignment is already resolved in spcsByShard). Nil list entries or empty +// buildClusterScopedMaestroAPIMaestroBundleNamesByShard maps provision shard ID to the set of Maestro API bundle names referenced by +// ServiceProviderClusters grouped under that shard (shard assignment is already resolved in spcsByShard). Nil list entries or empty // maestroAPIMaestroBundleName return an error so the reference set cannot silently omit in-use bundles. -func referencedMaestroAPIMaestroBundleNamesByShard(spcsByShard map[string][]*api.ServiceProviderCluster) (map[string]map[string]struct{}, error) { - out := make(map[string]map[string]struct{}) +func (c *deleteOrphanedMaestroReadonlyBundles) buildClusterScopedMaestroAPIMaestroBundleNamesByShard(spcsByShard map[string][]*api.ServiceProviderCluster) (maestroBundleNamesByShard, error) { + out := make(maestroBundleNamesByShard) for shardID, spcs := range spcsByShard { // If it is the first time we are processing this shard we initialize the map entry for it @@ -572,10 +499,11 @@ func referencedMaestroAPIMaestroBundleNamesByShard(spcsByShard map[string][]*api return out, nil } -// referencedMaestroAPIMaestroBundleNamesByShardFromNodePools maps provision shard ID to the set of Maestro API bundle names -// referenced by ServiceProviderNodePools on that shard. -func referencedMaestroAPIMaestroBundleNamesByShardFromNodePools(spnpsByShard map[string][]*api.ServiceProviderNodePool) (map[string]map[string]struct{}, error) { - out := make(map[string]map[string]struct{}) +// buildNodePoolScopedMaestroAPIMaestroBundleNamesByShard builds a map of provision shard ID to the set of Maestro API bundle names referenced by +// ServiceProviderNodePools grouped under that shard (shard assignment is already resolved in spnpsByShard). Nil list entries or empty +// maestroAPIMaestroBundleName return an error so the reference set cannot silently omit in-use bundles. +func (c *deleteOrphanedMaestroReadonlyBundles) buildNodePoolScopedMaestroAPIMaestroBundleNamesByShard(spnpsByShard map[string][]*api.ServiceProviderNodePool) (maestroBundleNamesByShard, error) { + out := make(maestroBundleNamesByShard) for shardID, spnps := range spnpsByShard { // If it is the first time we are processing this shard we initialize the map entry for it diff --git a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller_test.go b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller_test.go index 9e790e18a8c..f6938067a31 100644 --- a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller_test.go +++ b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller_test.go @@ -41,55 +41,7 @@ import ( "github.com/Azure/ARO-HCP/internal/utils" ) -func TestDeleteOrphanedMaestroReadonlyBundles_getAllServiceProviderClusters(t *testing.T) { - ctx := context.Background() - clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) - spcResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/serviceProviderClusters/default")) - - tests := []struct { - name string - setupDB func(t *testing.T, ctx context.Context, mockDB *databasetesting.MockDBClient) - wantLen int - wantFirstResourceID string - }{ - { - name: "empty DB returns no SPCs", - setupDB: nil, - wantLen: 0, - }, - { - name: "returns SPCs created via CRUD", - setupDB: func(t *testing.T, ctx context.Context, mockDB *databasetesting.MockDBClient) { - spc := &api.ServiceProviderCluster{ - CosmosMetadata: arm.CosmosMetadata{ResourceID: spcResourceID}, - ResourceID: *spcResourceID, - } - spcCRUD := mockDB.ServiceProviderClusters(clusterResourceID.SubscriptionID, clusterResourceID.ResourceGroupName, clusterResourceID.Name) - _, err := spcCRUD.Create(ctx, spc, nil) - require.NoError(t, err) - }, - wantLen: 1, - wantFirstResourceID: spcResourceID.String(), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mockDB := databasetesting.NewMockDBClient() - if tt.setupDB != nil { - tt.setupDB(t, ctx, mockDB) - } - c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB} - all, err := c.getAllServiceProviderClusters(ctx) - require.NoError(t, err) - require.Len(t, all, tt.wantLen) - if tt.wantFirstResourceID != "" { - assert.Equal(t, tt.wantFirstResourceID, all[0].ResourceID.String()) - } - }) - } -} - -func TestReferencedMaestroAPIMaestroBundleNamesByShard(t *testing.T) { +func TestBuildClusterScopedMaestroAPIMaestroBundleNamesByShard(t *testing.T) { spcRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/serviceProviderClusters/default")) tests := []struct { @@ -158,9 +110,10 @@ func TestReferencedMaestroAPIMaestroBundleNamesByShard(t *testing.T) { wantBundleName: "bundle-one", }, } + c := &deleteOrphanedMaestroReadonlyBundles{} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - out, err := referencedMaestroAPIMaestroBundleNamesByShard(tt.spcsByShard) + out, err := c.buildClusterScopedMaestroAPIMaestroBundleNamesByShard(tt.spcsByShard) if tt.wantErr { require.Error(t, err) assert.Contains(t, err.Error(), tt.errSubstr) @@ -175,7 +128,7 @@ func TestReferencedMaestroAPIMaestroBundleNamesByShard(t *testing.T) { } } -func TestReferencedMaestroAPIMaestroBundleNamesByShardFromNodePools(t *testing.T) { +func TestBuildNodePoolScopedMaestroAPIMaestroBundleNamesByShard(t *testing.T) { spnpRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/nodePools/np/serviceProviderNodePools/default")) tests := []struct { @@ -244,9 +197,10 @@ func TestReferencedMaestroAPIMaestroBundleNamesByShardFromNodePools(t *testing.T wantBundleName: "bundle-np", }, } + c := &deleteOrphanedMaestroReadonlyBundles{} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - out, err := referencedMaestroAPIMaestroBundleNamesByShardFromNodePools(tt.spnpsByShard) + out, err := c.buildNodePoolScopedMaestroAPIMaestroBundleNamesByShard(tt.spnpsByShard) if tt.wantErr { require.Error(t, err) assert.Contains(t, err.Error(), tt.errSubstr) @@ -261,36 +215,6 @@ func TestReferencedMaestroAPIMaestroBundleNamesByShardFromNodePools(t *testing.T } } -// paginationSPCGlobalLister returns different iterators based on ContinuationToken to simulate pagination. -type paginationSPCGlobalLister struct { - iter1 database.DBClientIterator[api.ServiceProviderCluster] - iter2 database.DBClientIterator[api.ServiceProviderCluster] - token string -} - -func (p *paginationSPCGlobalLister) List(ctx context.Context, opts *database.DBClientListResourceDocsOptions) (database.DBClientIterator[api.ServiceProviderCluster], error) { - tok := "" - if opts != nil && opts.ContinuationToken != nil { - tok = *opts.ContinuationToken - } - if tok == "" { - return p.iter1, nil - } - if tok == p.token { - return p.iter2, nil - } - return nil, fmt.Errorf("unexpected continuation token %q", tok) -} - -// iteratorErrorSPCGlobalLister returns an iterator that reports an error from GetError(). -type iteratorErrorSPCGlobalLister struct { - iter database.DBClientIterator[api.ServiceProviderCluster] -} - -func (i *iteratorErrorSPCGlobalLister) List(ctx context.Context, opts *database.DBClientListResourceDocsOptions) (database.DBClientIterator[api.ServiceProviderCluster], error) { - return i.iter, nil -} - // panicGlobalLister is a GlobalLister that panics if List is called (used for unused listers in test doubles). type panicGlobalLister[T any] struct{} @@ -298,46 +222,45 @@ func (n *panicGlobalLister[T]) List(context.Context, *database.DBClientListResou panic("panicGlobalLister.List should not be called") } -// paginationGlobalListers implements GlobalListers with pagination only for ServiceProviderClusters. -type paginationGlobalListers struct { - spcLister database.GlobalLister[api.ServiceProviderCluster] -} +// defaultPanicGlobalListers implements database.GlobalListers with panic-on-List listers for every resource type. +// Embed it in a test double and override only the accessors the test cares about. +type defaultPanicGlobalListers struct{} -func (p *paginationGlobalListers) Subscriptions() database.GlobalLister[arm.Subscription] { +func (defaultPanicGlobalListers) Subscriptions() database.GlobalLister[arm.Subscription] { return &panicGlobalLister[arm.Subscription]{} } -func (p *paginationGlobalListers) Clusters() database.GlobalLister[api.HCPOpenShiftCluster] { +func (defaultPanicGlobalListers) Clusters() database.GlobalLister[api.HCPOpenShiftCluster] { return &panicGlobalLister[api.HCPOpenShiftCluster]{} } -func (p *paginationGlobalListers) NodePools() database.GlobalLister[api.HCPOpenShiftClusterNodePool] { +func (defaultPanicGlobalListers) NodePools() database.GlobalLister[api.HCPOpenShiftClusterNodePool] { return &panicGlobalLister[api.HCPOpenShiftClusterNodePool]{} } -func (p *paginationGlobalListers) ExternalAuths() database.GlobalLister[api.HCPOpenShiftClusterExternalAuth] { +func (defaultPanicGlobalListers) ExternalAuths() database.GlobalLister[api.HCPOpenShiftClusterExternalAuth] { return &panicGlobalLister[api.HCPOpenShiftClusterExternalAuth]{} } -func (p *paginationGlobalListers) ServiceProviderClusters() database.GlobalLister[api.ServiceProviderCluster] { - return p.spcLister -} -func (p *paginationGlobalListers) Operations() database.GlobalLister[api.Operation] { - return &panicGlobalLister[api.Operation]{} +func (defaultPanicGlobalListers) ServiceProviderClusters() database.GlobalLister[api.ServiceProviderCluster] { + return &panicGlobalLister[api.ServiceProviderCluster]{} } -func (p *paginationGlobalListers) ActiveOperations() database.GlobalLister[api.Operation] { - return &panicGlobalLister[api.Operation]{} +func (defaultPanicGlobalListers) ServiceProviderNodePools() database.GlobalLister[api.ServiceProviderNodePool] { + return &panicGlobalLister[api.ServiceProviderNodePool]{} } -func (p *paginationGlobalListers) Controllers() database.GlobalLister[api.Controller] { +func (defaultPanicGlobalListers) Controllers() database.GlobalLister[api.Controller] { return &panicGlobalLister[api.Controller]{} } -func (p *paginationGlobalListers) ManagementClusterContents() database.GlobalLister[api.ManagementClusterContent] { +func (defaultPanicGlobalListers) ManagementClusterContents() database.GlobalLister[api.ManagementClusterContent] { return &panicGlobalLister[api.ManagementClusterContent]{} } -func (p *paginationGlobalListers) ServiceProviderNodePools() database.GlobalLister[api.ServiceProviderNodePool] { - return &panicGlobalLister[api.ServiceProviderNodePool]{} +func (defaultPanicGlobalListers) Operations() database.GlobalLister[api.Operation] { + return &panicGlobalLister[api.Operation]{} +} +func (defaultPanicGlobalListers) ActiveOperations() database.GlobalLister[api.Operation] { + return &panicGlobalLister[api.Operation]{} } -func (p *paginationGlobalListers) BillingDocs() database.GlobalLister[database.BillingDocument] { +func (defaultPanicGlobalListers) BillingDocs() database.GlobalLister[database.BillingDocument] { return &panicGlobalLister[database.BillingDocument]{} } -var _ database.GlobalListers = (*paginationGlobalListers)(nil) +var _ database.GlobalListers = defaultPanicGlobalListers{} // simpleIterator is a simple iterator implementation for testing that doesn't use gomock. type simpleIterator[T any] struct { @@ -367,71 +290,55 @@ func (s *simpleIterator[T]) GetError() error { var _ database.DBClientIterator[api.ServiceProviderCluster] = &simpleIterator[api.ServiceProviderCluster]{} -var _ database.DBClientIterator[api.ServiceProviderNodePool] = &simpleIterator[api.ServiceProviderNodePool]{} - -func TestDeleteOrphanedMaestroReadonlyBundles_getAllServiceProviderClusters_Pagination(t *testing.T) { +func TestDeleteOrphanedMaestroReadonlyBundles_getAllServiceProviderClusters(t *testing.T) { ctx := context.Background() - spc1ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/serviceProviderClusters/default")) - spc2ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub2/resourceGroups/rg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster2/serviceProviderClusters/default")) - page1SPC := &api.ServiceProviderCluster{ - CosmosMetadata: arm.CosmosMetadata{ResourceID: spc1ResourceID}, - ResourceID: *spc1ResourceID, - } - page2SPC := &api.ServiceProviderCluster{ - CosmosMetadata: arm.CosmosMetadata{ResourceID: spc2ResourceID}, - ResourceID: *spc2ResourceID, - } - - iter1 := &simpleIterator[api.ServiceProviderCluster]{ - ids: []string{"id1"}, - items: []*api.ServiceProviderCluster{page1SPC}, - continuationToken: "token1", - err: nil, - } + clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) + spcResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/serviceProviderClusters/default")) - iter2 := &simpleIterator[api.ServiceProviderCluster]{ - ids: []string{"id2"}, - items: []*api.ServiceProviderCluster{page2SPC}, - continuationToken: "", - err: nil, + tests := []struct { + name string + setupDB func(t *testing.T, ctx context.Context, mockDB *databasetesting.MockDBClient) + wantLen int + wantFirstResourceID string + }{ + { + name: "empty DB returns no SPCs", + setupDB: nil, + wantLen: 0, + }, + { + name: "returns SPCs created via CRUD", + setupDB: func(t *testing.T, ctx context.Context, mockDB *databasetesting.MockDBClient) { + spc := &api.ServiceProviderCluster{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spcResourceID}, + ResourceID: *spcResourceID, + } + spcCRUD := mockDB.ServiceProviderClusters(clusterResourceID.SubscriptionID, clusterResourceID.ResourceGroupName, clusterResourceID.Name) + _, err := spcCRUD.Create(ctx, spc, nil) + require.NoError(t, err) + }, + wantLen: 1, + wantFirstResourceID: spcResourceID.String(), + }, } - - paginationListers := &paginationGlobalListers{ - spcLister: &paginationSPCGlobalLister{iter1: iter1, iter2: iter2, token: "token1"}, + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockDB := databasetesting.NewMockDBClient() + if tt.setupDB != nil { + tt.setupDB(t, ctx, mockDB) + } + c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB} + all, err := c.getAllServiceProviderClusters(ctx) + require.NoError(t, err) + require.Len(t, all, tt.wantLen) + if tt.wantFirstResourceID != "" { + assert.Equal(t, tt.wantFirstResourceID, all[0].ResourceID.String()) + } + }) } - mockDB := databasetesting.NewMockDBClient() - mockDB.SetGlobalListers(paginationListers) - - c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB} - all, err := c.getAllServiceProviderClusters(ctx) - require.NoError(t, err) - require.Len(t, all, 2) - assert.Equal(t, spc1ResourceID.String(), all[0].ResourceID.String()) - assert.Equal(t, spc2ResourceID.String(), all[1].ResourceID.String()) } -func TestDeleteOrphanedMaestroReadonlyBundles_getAllServiceProviderClusters_IteratorError(t *testing.T) { - ctx := context.Background() - iterErr := fmt.Errorf("iteration error") - iter := &simpleIterator[api.ServiceProviderCluster]{ - ids: []string{}, - items: []*api.ServiceProviderCluster{}, - continuationToken: "", - err: iterErr, - } - - mockDB := databasetesting.NewMockDBClient() - mockDB.SetGlobalListers(&paginationGlobalListers{ - spcLister: &iteratorErrorSPCGlobalLister{iter: iter}, - }) - - c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB} - all, err := c.getAllServiceProviderClusters(ctx) - require.Error(t, err) - assert.Nil(t, all) - assert.Contains(t, err.Error(), "failed iterating ServiceProviderClusters") - assert.Contains(t, err.Error(), "iteration error") -} +var _ database.DBClientIterator[api.ServiceProviderNodePool] = &simpleIterator[api.ServiceProviderNodePool]{} func TestDeleteOrphanedMaestroReadonlyBundles_getAllServiceProviderNodePools(t *testing.T) { ctx := context.Background() @@ -541,7 +448,7 @@ func (e *emptyGlobalLister[T]) List(ctx context.Context, opts *database.DBClient return &simpleIterator[T]{}, nil } -// failOnSecondSPNPGlobalLister fails ServiceProviderNodePools.List starting on the second call (initial SyncOnce list succeeds, fresh list inside nodepool orphan pass fails). +// failOnSecondSPNPGlobalLister fails ServiceProviderNodePools.List starting on the second call (initial list inside nodepool ensure succeeds, fresh list inside nodepool delete pass fails). type failOnSecondSPNPGlobalLister struct { call int err error @@ -555,46 +462,93 @@ func (f *failOnSecondSPNPGlobalLister) List(ctx context.Context, opts *database. return &simpleIterator[api.ServiceProviderNodePool]{}, nil } -// syncOnceSPCOKFailSecondSPNPGlobalListers lists empty SPCs always, and fails the second global SPNP list (for SyncOnce integration). +// syncOnceSPCOKFailSecondSPNPGlobalListers lists empty SPCs always, and fails the second global SPNP list during nodepool ensure (for SyncOnce integration). type syncOnceSPCOKFailSecondSPNPGlobalListers struct { + defaultPanicGlobalListers spnp *failOnSecondSPNPGlobalLister } -func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) Subscriptions() database.GlobalLister[arm.Subscription] { - return &panicGlobalLister[arm.Subscription]{} -} -func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) Clusters() database.GlobalLister[api.HCPOpenShiftCluster] { - return &panicGlobalLister[api.HCPOpenShiftCluster]{} -} -func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) NodePools() database.GlobalLister[api.HCPOpenShiftClusterNodePool] { - return &panicGlobalLister[api.HCPOpenShiftClusterNodePool]{} -} -func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) ExternalAuths() database.GlobalLister[api.HCPOpenShiftClusterExternalAuth] { - return &panicGlobalLister[api.HCPOpenShiftClusterExternalAuth]{} -} func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) ServiceProviderClusters() database.GlobalLister[api.ServiceProviderCluster] { return &emptyGlobalLister[api.ServiceProviderCluster]{} } + func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) ServiceProviderNodePools() database.GlobalLister[api.ServiceProviderNodePool] { return g.spnp } -func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) Controllers() database.GlobalLister[api.Controller] { - return &panicGlobalLister[api.Controller]{} + +var _ database.GlobalListers = (*syncOnceSPCOKFailSecondSPNPGlobalListers)(nil) + +// failOnSecondServiceProviderClusterGlobalLister fails ServiceProviderClusters.List starting on the second call. +type failOnSecondServiceProviderClusterGlobalLister struct { + call int + err error } -func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) ManagementClusterContents() database.GlobalLister[api.ManagementClusterContent] { - return &panicGlobalLister[api.ManagementClusterContent]{} + +func (f *failOnSecondServiceProviderClusterGlobalLister) List(ctx context.Context, opts *database.DBClientListResourceDocsOptions) (database.DBClientIterator[api.ServiceProviderCluster], error) { + f.call++ + if f.call >= 2 { + return nil, f.err + } + return &simpleIterator[api.ServiceProviderCluster]{}, nil } -func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) Operations() database.GlobalLister[api.Operation] { - return &panicGlobalLister[api.Operation]{} + +// emptyFirstThenServiceProviderClusterGlobalLister returns an empty first ServiceProviderClusters list, then yields items on subsequent calls. +type emptyFirstThenServiceProviderClusterGlobalLister struct { + call int + items []*api.ServiceProviderCluster } -func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) ActiveOperations() database.GlobalLister[api.Operation] { - return &panicGlobalLister[api.Operation]{} + +func (e *emptyFirstThenServiceProviderClusterGlobalLister) List(ctx context.Context, opts *database.DBClientListResourceDocsOptions) (database.DBClientIterator[api.ServiceProviderCluster], error) { + e.call++ + if e.call == 1 { + return &simpleIterator[api.ServiceProviderCluster]{}, nil + } + ids := make([]string, len(e.items)) + for i, spc := range e.items { + ids[i] = spc.ResourceID.String() + } + return &simpleIterator[api.ServiceProviderCluster]{ids: ids, items: e.items}, nil } -func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) BillingDocs() database.GlobalLister[database.BillingDocument] { - return &panicGlobalLister[database.BillingDocument]{} + +// orphanTestGlobalListersSPCOnly is a GlobalListers test double that only customizes ServiceProviderClusters(). +type orphanTestGlobalListersSPCOnly struct { + defaultPanicGlobalListers + spc database.GlobalLister[api.ServiceProviderCluster] } -var _ database.GlobalListers = (*syncOnceSPCOKFailSecondSPNPGlobalListers)(nil) +func newOrphanTestGlobalListersSPCOnly(spc database.GlobalLister[api.ServiceProviderCluster]) *orphanTestGlobalListersSPCOnly { + return &orphanTestGlobalListersSPCOnly{spc: spc} +} + +func (g *orphanTestGlobalListersSPCOnly) ServiceProviderClusters() database.GlobalLister[api.ServiceProviderCluster] { + return g.spc +} + +func (g *orphanTestGlobalListersSPCOnly) ServiceProviderNodePools() database.GlobalLister[api.ServiceProviderNodePool] { + return &emptyGlobalLister[api.ServiceProviderNodePool]{} +} + +var _ database.GlobalListers = (*orphanTestGlobalListersSPCOnly)(nil) + +// orphanTestGlobalListersSPNPOnly is a GlobalListers test double that only customizes ServiceProviderNodePools(). +type orphanTestGlobalListersSPNPOnly struct { + defaultPanicGlobalListers + spnp database.GlobalLister[api.ServiceProviderNodePool] +} + +func newOrphanTestGlobalListersSPNPOnly(spnp database.GlobalLister[api.ServiceProviderNodePool]) *orphanTestGlobalListersSPNPOnly { + return &orphanTestGlobalListersSPNPOnly{spnp: spnp} +} + +func (g *orphanTestGlobalListersSPNPOnly) ServiceProviderClusters() database.GlobalLister[api.ServiceProviderCluster] { + return &emptyGlobalLister[api.ServiceProviderCluster]{} +} + +func (g *orphanTestGlobalListersSPNPOnly) ServiceProviderNodePools() database.GlobalLister[api.ServiceProviderNodePool] { + return g.spnp +} + +var _ database.GlobalListers = (*orphanTestGlobalListersSPNPOnly)(nil) func TestDeleteOrphanedMaestroReadonlyBundles_SyncOnce_ListServiceProviderClustersError(t *testing.T) { ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) @@ -628,7 +582,7 @@ func TestDeleteOrphanedMaestroReadonlyBundles_SyncOnce_ListServiceProviderNodePo c := NewDeleteOrphanedMaestroReadonlyBundlesController(mockDB, mockCS, nil, "test-env") err := c.SyncOnce(ctx, nil) require.Error(t, err) - assert.Contains(t, err.Error(), "error getting all ServiceProviderNodePools (fresh snapshot)") + assert.Contains(t, err.Error(), "failed to get all ServiceProviderNodePools") assert.Contains(t, err.Error(), "list SPNPs error") } @@ -665,6 +619,17 @@ func TestDeleteOrphanedMaestroReadonlyBundles_buildMaestroClientsByProvisionShar require.Empty(t, clients) }, }, + { + name: "list provision shards iterator reports error after iteration", + setup: func(ctrl *gomock.Controller) *deleteOrphanedMaestroReadonlyBundles { + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + listErr := fmt.Errorf("CS list provision shards failed") + mockCS.EXPECT().ListProvisionShards().Return(ocm.NewSimpleProvisionShardListIterator(nil, listErr)) + return &deleteOrphanedMaestroReadonlyBundles{clusterServiceClient: mockCS} + }, + wantErr: true, + errSubstr: "failed to list Cluster Service provision shards", + }, { name: "success single shard", setup: func(ctrl *gomock.Controller) *deleteOrphanedMaestroReadonlyBundles { @@ -984,6 +949,7 @@ func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderClustersByProvis func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderNodePoolsByProvisionShard(t *testing.T) { ctx := context.Background() + // mapServiceProviderNodePoolsByProvisionShard does not use the Maestro client; tests only need shard IDs in the map. noopMaestroShardClient := &shardMaestroClient{ maestroClient: nil, maestroClientCancelFunc: func() {}, @@ -991,34 +957,250 @@ func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderNodePoolsByProvi clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/nodePools/worker/serviceProviderNodePools/default")) - t.Run("success single shard", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockDB := databasetesting.NewMockDBClient() - mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) - cluster := &api.HCPOpenShiftCluster{ - TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterResourceID}}, - ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ - ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid")), + tests := []struct { + name string + setup func(ctrl *gomock.Controller) (c *deleteOrphanedMaestroReadonlyBundles, clients map[string]*shardMaestroClient, spnps []*api.ServiceProviderNodePool) + wantErr bool + errSubstr string + validateOut func(t *testing.T, shardToSPNPs map[string][]*api.ServiceProviderNodePool) + }{ + { + name: "no parent resource ID", + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderNodePool) { + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + broken := *spnpResourceID + broken.Parent = nil + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: &broken}, + ResourceID: broken, + } + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, + map[string]*shardMaestroClient{"unused-shard": noopMaestroShardClient}, + []*api.ServiceProviderNodePool{spnp} }, - } - _, err := mockDB.HCPClusters("sub", "rg").Create(ctx, cluster, nil) - require.NoError(t, err) - spnp := &api.ServiceProviderNodePool{ - CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, - ResourceID: *spnpResourceID, - } - provisionShard := buildTestProvisionShard("test-consumer") - mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster.ServiceProviderProperties.ClusterServiceID).Return(provisionShard, nil) - clients := map[string]*shardMaestroClient{provisionShard.ID(): noopMaestroShardClient} - c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} - defer cancelMaestroClientsByProvisionShard(clients) - shardToSPNPs, err := c.mapServiceProviderNodePoolsByProvisionShard(ctx, []*api.ServiceProviderNodePool{spnp}, clients) - require.NoError(t, err) - require.Len(t, shardToSPNPs, 1) - spnps := shardToSPNPs[provisionShard.ID()] - require.Len(t, spnps, 1) - assert.Equal(t, spnpResourceID.String(), spnps[0].ResourceID.String()) - }) + wantErr: true, + errSubstr: "has no parent resource ID", + }, + { + name: "no grandparent cluster resource ID", + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderNodePool) { + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + noGrand := *spnpResourceID + npOnly := *noGrand.Parent + npOnly.Parent = nil + noGrand.Parent = &npOnly + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: &noGrand}, + ResourceID: noGrand, + } + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, + map[string]*shardMaestroClient{"unused-shard": noopMaestroShardClient}, + []*api.ServiceProviderNodePool{spnp} + }, + wantErr: true, + errSubstr: "has no grandparent cluster resource ID", + }, + { + name: "Get cluster error", + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderNodePool) { + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + } + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, + map[string]*shardMaestroClient{"unused-shard": noopMaestroShardClient}, + []*api.ServiceProviderNodePool{spnp} + }, + wantErr: true, + errSubstr: "failed to get Cluster", + }, + { + name: "Get provision shard error", + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderNodePool) { + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + cluster := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterResourceID}}, + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid")), + }, + } + _, err := mockDB.HCPClusters("sub", "rg").Create(ctx, cluster, nil) + require.NoError(t, err) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + } + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster.ServiceProviderProperties.ClusterServiceID).Return(nil, fmt.Errorf("provision shard error")) + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, + map[string]*shardMaestroClient{"unused-shard": noopMaestroShardClient}, + []*api.ServiceProviderNodePool{spnp} + }, + wantErr: true, + errSubstr: "failed to get Cluster Provision Shard", + }, + { + name: "provision shard not in clients map", + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderNodePool) { + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + cluster := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterResourceID}}, + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid")), + }, + } + _, err := mockDB.HCPClusters("sub", "rg").Create(ctx, cluster, nil) + require.NoError(t, err) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + } + shardInClientsMap := buildTestProvisionShard("consumer-in-list") + shard2ID := "33333333333333333333333333333333" + shardReturnedByCS, err := arohcpv1alpha1.NewProvisionShard(). + ID(shard2ID). + MaestroConfig( + arohcpv1alpha1.NewProvisionShardMaestroConfig(). + ConsumerName("other-consumer"). + RestApiConfig(arohcpv1alpha1.NewProvisionShardMaestroRestApiConfig().Url("https://other.example.com:443")). + GrpcApiConfig(arohcpv1alpha1.NewProvisionShardMaestroGrpcApiConfig().Url("https://other.example.com:444")), + ). + Build() + require.NoError(t, err) + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster.ServiceProviderProperties.ClusterServiceID).Return(shardReturnedByCS, nil) + clients := map[string]*shardMaestroClient{ + shardInClientsMap.ID(): noopMaestroShardClient, + } + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, clients, []*api.ServiceProviderNodePool{spnp} + }, + wantErr: true, + errSubstr: "not present in provision shards map", + }, + { + name: "success single shard", + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderNodePool) { + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + cluster := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterResourceID}}, + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid")), + }, + } + _, err := mockDB.HCPClusters("sub", "rg").Create(ctx, cluster, nil) + require.NoError(t, err) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + } + provisionShard := buildTestProvisionShard("test-consumer") + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster.ServiceProviderProperties.ClusterServiceID).Return(provisionShard, nil) + clients := map[string]*shardMaestroClient{provisionShard.ID(): noopMaestroShardClient} + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, clients, []*api.ServiceProviderNodePool{spnp} + }, + validateOut: func(t *testing.T, shardToSPNPs map[string][]*api.ServiceProviderNodePool) { + provisionShard := buildTestProvisionShard("test-consumer") + require.Len(t, shardToSPNPs, 1) + spnps := shardToSPNPs[provisionShard.ID()] + require.Len(t, spnps, 1) + assert.Equal(t, spnpResourceID.String(), spnps[0].ResourceID.String()) + }, + }, + { + name: "multiple provision shards each get own map entry", + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderNodePool) { + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + + cluster1ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub1/resourceGroups/rg1/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster1")) + cluster2ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub2/resourceGroups/rg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster2")) + cluster1 := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: cluster1ResourceID}}, + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid1")), + }, + } + cluster2 := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: cluster2ResourceID}}, + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid2")), + }, + } + _, err := mockDB.HCPClusters("sub1", "rg1").Create(ctx, cluster1, nil) + require.NoError(t, err) + _, err = mockDB.HCPClusters("sub2", "rg2").Create(ctx, cluster2, nil) + require.NoError(t, err) + + spnp1ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub1/resourceGroups/rg1/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster1/nodePools/worker/serviceProviderNodePools/default")) + spnp2ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub2/resourceGroups/rg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster2/nodePools/worker/serviceProviderNodePools/default")) + spnp1 := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnp1ResourceID}, + ResourceID: *spnp1ResourceID, + } + spnp2 := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnp2ResourceID}, + ResourceID: *spnp2ResourceID, + } + + shard1 := buildTestProvisionShard("consumer1") + shard2ID := "33333333333333333333333333333333" + shard2, err := arohcpv1alpha1.NewProvisionShard(). + ID(shard2ID). + MaestroConfig( + arohcpv1alpha1.NewProvisionShardMaestroConfig(). + ConsumerName("consumer2"). + RestApiConfig(arohcpv1alpha1.NewProvisionShardMaestroRestApiConfig().Url("https://maestro2.example.com:443")). + GrpcApiConfig(arohcpv1alpha1.NewProvisionShardMaestroGrpcApiConfig().Url("https://maestro2.example.com:444")), + ). + Build() + require.NoError(t, err) + + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster1.ServiceProviderProperties.ClusterServiceID).Return(shard1, nil) + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster2.ServiceProviderProperties.ClusterServiceID).Return(shard2, nil) + + clients := map[string]*shardMaestroClient{ + shard1.ID(): noopMaestroShardClient, + shard2.ID(): noopMaestroShardClient, + } + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, clients, []*api.ServiceProviderNodePool{spnp1, spnp2} + }, + validateOut: func(t *testing.T, shardToSPNPs map[string][]*api.ServiceProviderNodePool) { + require.Len(t, shardToSPNPs, 2) + shard1 := buildTestProvisionShard("consumer1") + spnps1 := shardToSPNPs[shard1.ID()] + require.Len(t, spnps1, 1) + spnp1RID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub1/resourceGroups/rg1/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster1/nodePools/worker/serviceProviderNodePools/default")) + assert.Equal(t, spnp1RID.String(), spnps1[0].ResourceID.String()) + shard2ID := "33333333333333333333333333333333" + spnps2 := shardToSPNPs[shard2ID] + require.Len(t, spnps2, 1) + spnp2RID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub2/resourceGroups/rg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster2/nodePools/worker/serviceProviderNodePools/default")) + assert.Equal(t, spnp2RID.String(), spnps2[0].ResourceID.String()) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + c, clients, spnps := tt.setup(ctrl) + defer cancelMaestroClientsByProvisionShard(clients) + shardToSPNPs, err := c.mapServiceProviderNodePoolsByProvisionShard(ctx, spnps, clients) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errSubstr) + return + } + require.NoError(t, err) + if tt.validateOut != nil { + tt.validateOut(t, shardToSPNPs) + } + }) + } } func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(t *testing.T) { @@ -1027,45 +1209,44 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestro tests := []struct { name string - setupMock func(*testing.T, *maestro.MockClient, *databasetesting.MockDBClient, *ocm.MockClusterServiceClientSpec, *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) + setupMock func(*testing.T, *maestro.MockClient, *databasetesting.MockDBClient, *ocm.MockClusterServiceClientSpec, *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient wantErr bool errSubstr string }{ { name: "empty provision shards map does not perform anything", - setupMock: func(t *testing.T, _ *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, _ *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) { - return nil, nil + setupMock: func(t *testing.T, _ *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, _ *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { + return nil }, wantErr: false, }, { name: "second global SPC list error", - setupMock: func(_ *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) { + setupMock: func(_ *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { listErr := fmt.Errorf("fresh list SPCs error") - mockDB.SetGlobalListers(&alwaysErrorGlobalListers{err: listErr}) - // We mock the Maestro client list to return an empty list to avoid processing any Maestro bundles and going straight to the SPC list call. + mockDB.SetGlobalListers(newOrphanTestGlobalListersSPCOnly(&failOnSecondServiceProviderClusterGlobalLister{err: listErr})) m.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{}, nil) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderCluster{} + } }, wantErr: true, - errSubstr: "error getting all ServiceProviderClusters (fresh snapshot)", + errSubstr: "failed to get all ServiceProviderClusters", }, { name: "an error is returned when listing Maestro bundles fails", - setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) { + setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { m.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("maestro list error")) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderCluster{} + } }, wantErr: true, errSubstr: "failed to list Maestro Bundles", }, { name: "skips bundle without readonly managed-by label", - setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) { + setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { bundleList := &workv1.ManifestWorkList{ Items: []workv1.ManifestWork{ { @@ -1080,12 +1261,12 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestro m.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderCluster{} + } }, }, { name: "skips Maestro bundle that is referenced by a ServiceProviderCluster on the shard", - setupMock: func(t *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, mockCS *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) { + setupMock: func(t *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, mockCS *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { clusterRID := spcResourceID.Parent cluster := &api.HCPOpenShiftCluster{ TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterRID}}, @@ -1121,12 +1302,12 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestro m.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderCluster{shard.ID(): {spc}} + } }, }, { name: "deletes Maestro bundle that is not referenced by any ServiceProviderCluster on the shard", - setupMock: func(t *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, mockCS *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) { + setupMock: func(t *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, mockCS *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { clusterRID := spcResourceID.Parent cluster := &api.HCPOpenShiftCluster{ TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterRID}}, @@ -1163,14 +1344,14 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestro m.EXPECT().Delete(gomock.Any(), "orphaned-bundle", metav1.DeleteOptions{}).Return(nil) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderCluster{shard.ID(): {spc}} + } }, }, { // Two orphans: first Delete fails (appended to syncErrors, loop continues), second Delete succeeds; // gomock call order proves the second delete was still attempted; errors.Join still returns an error. name: "continues deleting remaining orphans after a delete failure", - setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) { + setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { metaA := metav1.ObjectMeta{ Name: "orphan-a", Namespace: "consumer", @@ -1194,14 +1375,14 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestro m.EXPECT().Delete(gomock.Any(), "orphan-b", metav1.DeleteOptions{}).Return(nil) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderCluster{} + } }, wantErr: true, errSubstr: "failed to delete Maestro Bundle", }, { name: "pagination lists and deletes across pages", - setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) { + setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { page1Meta := metav1.ObjectMeta{ Name: "orphan-page1", Namespace: "consumer", @@ -1221,7 +1402,7 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestro m.EXPECT().Delete(gomock.Any(), "orphan-page1", metav1.DeleteOptions{}).Return(nil) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderCluster{} + } }, }, } @@ -1232,9 +1413,9 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestro mockDB := databasetesting.NewMockDBClient() mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) shard := buildTestProvisionShard("test-consumer") - clients, initialShardToSPCs := tt.setupMock(t, mockMaestro, mockDB, mockCS, shard) + clients := tt.setupMock(t, mockMaestro, mockDB, mockCS, shard) c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} - err := c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPCs) + err := c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients) if tt.wantErr { require.Error(t, err) assert.Contains(t, err.Error(), tt.errSubstr) @@ -1251,44 +1432,44 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedNodePoolScopedMaestr tests := []struct { name string - setupMock func(*testing.T, *maestro.MockClient, *databasetesting.MockDBClient, *ocm.MockClusterServiceClientSpec, *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderNodePool) + setupMock func(*testing.T, *maestro.MockClient, *databasetesting.MockDBClient, *ocm.MockClusterServiceClientSpec, *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient wantErr bool errSubstr string }{ { name: "empty maestro clients map does not perform anything", - setupMock: func(t *testing.T, _ *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, _ *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderNodePool) { - return nil, nil + setupMock: func(t *testing.T, _ *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, _ *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { + return nil }, wantErr: false, }, { name: "second global ServiceProviderNodePools list error", - setupMock: func(_ *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderNodePool) { + setupMock: func(_ *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { listErr := fmt.Errorf("fresh list SPNPs error") - mockDB.SetGlobalListers(&alwaysErrorGlobalListers{err: listErr}) + mockDB.SetGlobalListers(newOrphanTestGlobalListersSPNPOnly(&failOnSecondSPNPGlobalLister{err: listErr})) m.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{}, nil) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderNodePool{} + } }, wantErr: true, - errSubstr: "error getting all ServiceProviderNodePools (fresh snapshot)", + errSubstr: "failed to get all ServiceProviderNodePools", }, { name: "an error is returned when listing Maestro bundles fails", - setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderNodePool) { + setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { m.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("maestro list error")) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderNodePool{} + } }, wantErr: true, - errSubstr: "failed to list nodepool-scoped Maestro Bundles", + errSubstr: "failed to list Maestro Bundles for shard", }, { name: "skips bundle without nodepool readonly managed-by label", - setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderNodePool) { + setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { bundleList := &workv1.ManifestWorkList{ Items: []workv1.ManifestWork{ { @@ -1303,12 +1484,12 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedNodePoolScopedMaestr m.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderNodePool{} + } }, }, { name: "skips Maestro bundle referenced by a ServiceProviderNodePool on the shard", - setupMock: func(t *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, mockCS *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderNodePool) { + setupMock: func(t *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, mockCS *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { clusterRID := spnpResourceID.Parent.Parent cluster := &api.HCPOpenShiftCluster{ TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterRID}}, @@ -1344,12 +1525,12 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedNodePoolScopedMaestr m.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderNodePool{shard.ID(): {spnp}} + } }, }, { name: "deletes Maestro bundle not referenced by any ServiceProviderNodePool on the shard", - setupMock: func(t *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, mockCS *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderNodePool) { + setupMock: func(t *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, mockCS *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { clusterRID := spnpResourceID.Parent.Parent cluster := &api.HCPOpenShiftCluster{ TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterRID}}, @@ -1382,7 +1563,7 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedNodePoolScopedMaestr m.EXPECT().Delete(gomock.Any(), "orphaned-np-bundle", metav1.DeleteOptions{}).Return(nil) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderNodePool{shard.ID(): {spnp}} + } }, }, } @@ -1393,9 +1574,9 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedNodePoolScopedMaestr mockDB := databasetesting.NewMockDBClient() mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) shard := buildTestProvisionShard("test-consumer") - clients, initialShardToSPNPs := tt.setupMock(t, mockMaestro, mockDB, mockCS, shard) + clients := tt.setupMock(t, mockMaestro, mockDB, mockCS, shard) c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} - err := c.ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPNPs) + err := c.ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted(ctx, clients) if tt.wantErr { require.Error(t, err) assert.Contains(t, err.Error(), tt.errSubstr) @@ -1463,23 +1644,10 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestro _, err = mockDB.ServiceProviderClusters(parent.SubscriptionID, parent.ResourceGroupName, parent.Name).Create(ctx, spcOnShard1, nil) require.NoError(t, err) - spcOnShard1Lite := &api.ServiceProviderCluster{ - ResourceID: *spcOnShard1ResourceID, - Status: api.ServiceProviderClusterStatus{ - MaestroReadonlyBundles: api.MaestroBundleReferenceList{ - {MaestroAPIMaestroBundleName: "bundle-X"}, - }, - }, - } - clients := map[string]*shardMaestroClient{ shard1.ID(): {maestroClient: mockShard1, maestroClientCancelFunc: func() {}}, shard2.ID(): {maestroClient: mockShard2, maestroClientCancelFunc: func() {}}, } - initialShardToSPCs := map[string][]*api.ServiceProviderCluster{ - shard1.ID(): {spcOnShard1Lite}, - shard2.ID(): {}, - } mockShard1.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{Items: []workv1.ManifestWork{}}, nil) @@ -1498,7 +1666,7 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestro mockShard2.EXPECT().Delete(gomock.Any(), "bundle-X", metav1.DeleteOptions{}).Return(nil) c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} - err = c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPCs) + err = c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients) require.NoError(t, err) } @@ -1559,23 +1727,10 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestro _, err = mockDB.ServiceProviderClusters(parent.SubscriptionID, parent.ResourceGroupName, parent.Name).Create(ctx, spc, nil) require.NoError(t, err) - spcLite := &api.ServiceProviderCluster{ - ResourceID: *spcResourceID, - Status: api.ServiceProviderClusterStatus{ - MaestroReadonlyBundles: api.MaestroBundleReferenceList{ - {MaestroAPIMaestroBundleName: "bundle-N"}, - }, - }, - } - clients := map[string]*shardMaestroClient{ shardA.ID(): {maestroClient: mockShardA, maestroClientCancelFunc: func() {}}, shardB.ID(): {maestroClient: mockShardB, maestroClientCancelFunc: func() {}}, } - initialShardToSPCs := map[string][]*api.ServiceProviderCluster{ - shardA.ID(): {spcLite}, - shardB.ID(): {}, - } bundleNMeta := metav1.ObjectMeta{ Name: "bundle-N", @@ -1590,12 +1745,12 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestro mockShardB.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{Items: []workv1.ManifestWork{}}, nil) c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} - err = c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPCs) + err = c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients) require.NoError(t, err) } // TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted_ReferenceOnlyOnFreshGlobalList -// verifies that an SPC document present only on the second global Cosmos list (not in the cached SPC snapshot) +// verifies that an SPC document present only on the second global Cosmos list (not on the first) // still prevents deletion of the referenced bundle. func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted_ReferenceOnlyOnFreshGlobalList(t *testing.T) { ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) @@ -1629,15 +1784,14 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestro _, err = mockDB.ServiceProviderClusters(clusterRID.SubscriptionID, clusterRID.ResourceGroupName, clusterRID.Name).Create(ctx, spc, nil) require.NoError(t, err) + mockDB.SetGlobalListers(newOrphanTestGlobalListersSPCOnly(&emptyFirstThenServiceProviderClusterGlobalLister{items: []*api.ServiceProviderCluster{spc}})) + clients := map[string]*shardMaestroClient{ shard.ID(): { maestroClient: mockMaestro, maestroClientCancelFunc: func() {}, }, } - initialShardToSPCs := map[string][]*api.ServiceProviderCluster{ - shard.ID(): {}, // stale: no SPC rows for this shard in the initial map - } bundleList := &workv1.ManifestWorkList{ Items: []workv1.ManifestWork{ @@ -1653,7 +1807,7 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestro mockMaestro.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} - err = c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPCs) + err = c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients) require.NoError(t, err) } diff --git a/backend/pkg/controllers/maestro_readonly_bundle_helpers.go b/backend/pkg/controllers/maestro_readonly_bundle_helpers.go new file mode 100644 index 00000000000..4ffd238ee74 --- /dev/null +++ b/backend/pkg/controllers/maestro_readonly_bundle_helpers.go @@ -0,0 +1,318 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package controllers + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + workv1 "open-cluster-management.io/api/work/v1" + + "k8s.io/apimachinery/pkg/api/equality" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + + "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" + "github.com/Azure/ARO-HCP/backend/pkg/maestro" + "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/database" + "github.com/Azure/ARO-HCP/internal/utils" +) + +const ( + // readonlyBundleManagedByK8sLabelKey is the key of the K8s label that is used to identify the controller that manages the readonly Maestro bundle. + readonlyBundleManagedByK8sLabelKey = "aro-hcp.azure.com/readonly-bundle-managed-by" +) + +// kubeContentMaxSizeBytes is the maximum serialized size (in bytes) stored as KubeContent in Cosmos. +// 2MB is the maximum size of a Cosmos DB item (https://learn.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-item-limits). +const kubeContentMaxSizeBytes = 1887436 // 2MB * 0.9 + +// buildInitialReadonlyMaestroBundle builds an initial readonly Maestro Bundle for a given resource specified in obj. +// objResourceIdentifier is the resource identifier of the resource specified in obj. +// maestroBundleNamespacedName is the namespaced name of the Maestro Bundle. +// managedByLabelValue is the value of the readonlyBundleManagedByK8sLabelKey label to apply to the bundle. +// Used to create the readonly Maestro bundle associated to the resource specified in obj. Some controllers consider +// the readonlyBundleManagedByK8sLabelKey label to perform their own filtering of Maestro Bundles. +func buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName types.NamespacedName, objResourceIdentifier workv1.ResourceIdentifier, obj runtime.Object, managedByLabelValue string) *workv1.ManifestWork { + maestroBundleObjMeta := metav1.ObjectMeta{ + Name: maestroBundleNamespacedName.Name, + Namespace: maestroBundleNamespacedName.Namespace, + ResourceVersion: "0", // TODO is this needed when creating a maestro bundle? + Labels: map[string]string{ + // We define it as a K8s label because Maestro supports server-side filtering based on K8s labels. + // We can define it as a K8s label because for this specific use case we can comply with + // K8s labels length and charset restrictions https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set. + readonlyBundleManagedByK8sLabelKey: managedByLabelValue, + }, + } + + // We build the Maestro Bundle that will contain the resource specified in obj. + // Aside from putting the resource (manifest) previously built above, we + // also define a FeedbackRule that will allow us to retrieve the whole content + // from the management cluster + maestroBundle := &workv1.ManifestWork{ + ObjectMeta: maestroBundleObjMeta, + Spec: workv1.ManifestWorkSpec{ + Workload: workv1.ManifestsTemplate{ + Manifests: []workv1.Manifest{ + { + RawExtension: runtime.RawExtension{ + // We put the resource (manifest) specified in obj. + // In Maestro only the desired `spec` as defined in the bundle can be retrieved + // from here when querying the Maestro Bundle. + // To retrieve another section other than the desired spec Maestro + // requires defining FeedbackRule(s) in the Maestro bundle. + // For maestro readonly resources, not even the desired spec can be retrieved from here. For + // those type of resources it needs to be retrieved via status feedback rule(s) too. + // For owned resources, here the desired spec can be retrieved but that + // is not necessarily the actual spec in the management cluster side. If that is + // desired it is again necessary to get the spec via FeedbackRule(s). + Object: obj, + }, + }, + }, + }, + ManifestConfigs: []workv1.ManifestConfigOption{ + // We also need to define the ManifestConfig associated to the resource(manifest) + // that is being put within the Maestro Bundle. + { + // ResourceIdentifier needs to be specified and it is the information + // associated to the manifest that is being put within the Maestro Bundle. + ResourceIdentifier: objResourceIdentifier, + // We need to set the UpdateStrategy to read only. This + // creates a "readonly maestro bundle". + UpdateStrategy: &workv1.UpdateStrategy{ + Type: workv1.UpdateStrategyTypeReadOnly, + }, + // We define a feedbackrule based on JSONPath. We alias the name + // of this JSONPath as "resource" and its real JSONPath is "@" which + // signals the whole object is retrieved. This includes both spec + // and status. + FeedbackRules: []workv1.FeedbackRule{ + { + Type: workv1.JSONPathsType, + JsonPaths: []workv1.JsonPath{ + { + Name: "resource", + Path: "@", + }, + }, + }, + }, + }, + }, + }, + } + + return maestroBundle +} + +// buildInitialMaestroBundleReference builds an initial Maestro Bundle reference for a given maestro bundle internal name. +func buildInitialMaestroBundleReference(internalName api.MaestroBundleInternalName, generator maestro.MaestroAPIMaestroBundleNameGenerator) (*api.MaestroBundleReference, error) { + maestroAPIMaestroBundleName, err := generator.NewMaestroAPIMaestroBundleName() + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to generate Maestro API Maestro Bundle name: %w", err)) + } + return &api.MaestroBundleReference{ + Name: internalName, + MaestroAPIMaestroBundleName: maestroAPIMaestroBundleName, + MaestroAPIMaestroBundleID: "", + }, nil +} + +// buildObjectsFromUnstructuredObj builds the list of objects from the given unstructured object. +// If the unstructured object is a list, it flattens the list of objects from the list of items. Nested lists are not flattened. +// If the unstructured object is not a list, it returns a list with a single item being the single object. +func buildObjectsFromUnstructuredObj(unstructuredObj *unstructured.Unstructured) ([]runtime.RawExtension, error) { + if !unstructuredObj.IsList() { + return []runtime.RawExtension{{Object: unstructuredObj}}, nil + } + + objs := []runtime.RawExtension{} + err := unstructuredObj.EachListItem(func(o runtime.Object) error { + objs = append(objs, runtime.RawExtension{Object: o}) + return nil + }) + if err != nil { + return nil, utils.TrackError(err) + } + + return objs, nil +} + +func buildDegradedCondition(conditionStatus api.ConditionStatus, conditionReason string, conditionMessage string) api.Condition { + return api.Condition{ + Type: "Degraded", + Status: conditionStatus, + Reason: conditionReason, + Message: conditionMessage, + } +} + +// getSingleResourceStatusFeedbackRawJSONFromMaestroBundle gets the single resource status feedback raw JSON from a Maestro Bundle. +// Used to extract the content of the resource from the Maestro Bundle. +// An error is returned if the Maestro Bundle does not contain a single resource or if the resource does not contain a single status feedback value +// with its name being "resource" and its type being JsonRaw. +func getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(maestroBundle *workv1.ManifestWork) (json.RawMessage, error) { + resourceStatusManifests := maestroBundle.Status.ResourceStatus.Manifests + if len(resourceStatusManifests) != 1 { + return nil, utils.TrackError(fmt.Errorf("expected exactly one resource within the Maestro Bundle, got %d", len(resourceStatusManifests))) + } + + statusFeedbackValues := resourceStatusManifests[0].StatusFeedbacks.Values + if len(statusFeedbackValues) == 0 { + return nil, utils.TrackError(fmt.Errorf("expected exactly one status feedback value within the Maestro Bundle resource, got %d", len(statusFeedbackValues))) + } + if len(statusFeedbackValues) > 1 { + return nil, utils.TrackError(fmt.Errorf("expected exactly one status feedback value within the Maestro Bundle resource, got %d", len(statusFeedbackValues))) + } + statusFeedbackValue := statusFeedbackValues[0] + if statusFeedbackValue.Name != "resource" { + return nil, utils.TrackError(fmt.Errorf("expected status feedback value name to be 'resource', got %s", statusFeedbackValue.Name)) + } + if statusFeedbackValue.Value.Type != workv1.JsonRaw { + return nil, utils.TrackError(fmt.Errorf("expected status feedback value type to be JsonRaw, got %s", statusFeedbackValue.Value.Type)) + } + if statusFeedbackValue.Value.JsonRaw == nil { + return nil, utils.TrackError(fmt.Errorf("expected status feedback value JsonRaw to be not nil")) + } + + return []byte(*statusFeedbackValue.Value.JsonRaw), nil +} + +// calculateManagementClusterContentFromMaestroBundle builds the desired ManagementClusterContent from a Maestro +// bundle reference. clusterResourceID is the HCP cluster ARM resource ID (for node pool–scoped bundles, pass the +// node pool resource's parent cluster ID). +func calculateManagementClusterContentFromMaestroBundle( + ctx context.Context, + clusterResourceID *azcorearm.ResourceID, + maestroBundleReference *api.MaestroBundleReference, + maestroClient maestro.Client, +) (*api.ManagementClusterContent, error) { + managementClusterContentResourceID := controllerutils.ManagementClusterContentResourceIDFromClusterResourceID(clusterResourceID, maestroBundleReference.Name) + desired := controllerutils.NewInitialManagementClusterContent(managementClusterContentResourceID) + + existingMaestroBundle, err := maestroClient.Get(ctx, maestroBundleReference.MaestroAPIMaestroBundleName, metav1.GetOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return nil, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle: %w", err)) + } + if k8serrors.IsNotFound(err) { + degradedCondition := buildDegradedCondition(api.ConditionTrue, "MaestroBundleNotFound", err.Error()) + controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) + return desired, nil + } + + rawBytes, err := getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(existingMaestroBundle) + if err != nil { + degradedCondition := buildDegradedCondition(api.ConditionTrue, "MaestroBundleStatusFeedbackNotAvailable", err.Error()) + controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) + return desired, nil + } + + kubeContentMaxSizeExceeded := len(rawBytes) > kubeContentMaxSizeBytes + var kubeContextMaxSizeExceededConditionMessage string + // We only set the retrieved content if it is within the size limit. If it + // is outside the limit we set the Degraded condition communicating the issue. + // We use unstructuredObj.Unstructured to deserialize the content so we can + // implement logic agnostic to the type of the content being retrieved. + unstructuredObj := &unstructured.Unstructured{} + err = json.Unmarshal(rawBytes, unstructuredObj) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to unmarshal object from status feedback value: %w", err)) + } + kind := unstructuredObj.GetKind() + if kind == "" { + return nil, utils.TrackError(fmt.Errorf("expected kind to be not empty")) + } + + objs, err := buildObjectsFromUnstructuredObj(unstructuredObj) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to build objects from unstructured object: %w", err)) + } + var degradedCondition api.Condition + if !kubeContentMaxSizeExceeded { + // TODO is ListMeta or TypeMeta required at the metav1.List level? + desired.Status.KubeContent = &metav1.List{Items: objs} + degradedCondition = buildDegradedCondition(api.ConditionFalse, "NoErrors", "As expected.") + } else { + kubeContextMaxSizeExceededConditionMessage = fmt.Sprintf("%s serialized size %.2f MiB exceeds Kube content max size %.2f MiB;", kind, float64(len(rawBytes))/(1024*1024), float64(kubeContentMaxSizeBytes)/(1024*1024)) + degradedCondition = buildDegradedCondition(api.ConditionTrue, "KubeContentMaxSizeExceeded", kubeContextMaxSizeExceededConditionMessage) + } + controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) + + return desired, nil +} + +// readAndPersistMaestroReadonlyBundleContent reads a Maestro readonly bundle and creates or updates the corresponding +// ManagementClusterContent in Cosmos. hcpClusterResourceID must be the HCP OpenShift cluster ARM resource ID (for +// node pool–scoped bundles, pass the node pool resource's parent cluster ID). +func readAndPersistMaestroReadonlyBundleContent( + ctx context.Context, + cosmosClient database.DBClient, + hcpClusterResourceID *azcorearm.ResourceID, + maestroBundleReference *api.MaestroBundleReference, + maestroClient maestro.Client, +) error { + desired, err := calculateManagementClusterContentFromMaestroBundle(ctx, hcpClusterResourceID, maestroBundleReference, maestroClient) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to calculate ManagementClusterContent from Maestro Bundle: %w", err)) + } + + managementClusterContentsDBClient := cosmosClient.ManagementClusterContents( + hcpClusterResourceID.SubscriptionID, + hcpClusterResourceID.ResourceGroupName, + hcpClusterResourceID.Name, + ) + + existing, err := managementClusterContentsDBClient.Get(ctx, desired.CosmosMetadata.ResourceID.Name) + if err != nil && !database.IsResponseError(err, http.StatusNotFound) { + return utils.TrackError(fmt.Errorf("failed to get ManagementClusterContent: %w", err)) + } + if database.IsResponseError(err, http.StatusNotFound) { + _, err := managementClusterContentsDBClient.Create(ctx, desired, nil) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to create ManagementClusterContent: %w", err)) + } + return nil + } + + // We set the Cosmos ETag to the existing one to avoid conflicts when replacing the document + // unless someone else has modified the document since we last read it. + desired.CosmosETag = existing.CosmosETag + + // If we haven't been able to retrieve the content but there was already content + // stored we keep the previously existing stored content. + if desired.Status.KubeContent == nil && existing.Status.KubeContent != nil { + desired.Status.KubeContent = existing.Status.KubeContent + } + + if equality.Semantic.DeepEqual(existing, desired) { + return nil + } + + _, err = managementClusterContentsDBClient.Replace(ctx, desired, nil) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to replace ManagementClusterContent: %w", err)) + } + + return nil +} diff --git a/backend/pkg/controllers/maestro_readonly_bundle_helpers_test.go b/backend/pkg/controllers/maestro_readonly_bundle_helpers_test.go new file mode 100644 index 00000000000..e4b3602373a --- /dev/null +++ b/backend/pkg/controllers/maestro_readonly_bundle_helpers_test.go @@ -0,0 +1,229 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + workv1 "open-cluster-management.io/api/work/v1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + hsv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + + "github.com/Azure/ARO-HCP/internal/api" +) + +// buildTestMaestroBundleWithStatusFeedback builds a ManifestWork with exactly one resource status manifest +// and one status feedback value named "resource" with JsonRaw type. +func buildTestMaestroBundleWithStatusFeedback(name, namespace, rawJSON string) *workv1.ManifestWork { + jsonRaw := rawJSON + return &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Status: workv1.ManifestWorkStatus{ + ResourceStatus: workv1.ManifestResourceStatus{ + Manifests: []workv1.ManifestCondition{ + { + ResourceMeta: workv1.ManifestResourceMeta{ + Group: "hypershift.openshift.io", + Version: "v1beta1", + Kind: "HostedCluster", + Name: "test-hc", + Namespace: "test-ns", + }, + StatusFeedbacks: workv1.StatusFeedbackResult{ + Values: []workv1.FeedbackValue{ + { + Name: "resource", + Value: workv1.FieldValue{ + Type: workv1.JsonRaw, + JsonRaw: &jsonRaw, + }, + }, + }, + }, + Conditions: []metav1.Condition{}, + }, + }, + }, + }, + } +} + +func TestMaestroReadonlyBundleHelpers_buildDegradedCondition(t *testing.T) { + cond := buildDegradedCondition(api.ConditionTrue, "MaestroBundleNotFound", "bundle not found") + assert.Equal(t, "Degraded", cond.Type) + assert.Equal(t, api.ConditionTrue, cond.Status) + assert.Equal(t, "MaestroBundleNotFound", cond.Reason) + assert.Equal(t, "bundle not found", cond.Message) + + condFalse := buildDegradedCondition(api.ConditionFalse, "", "") + assert.Equal(t, api.ConditionFalse, condFalse.Status) + assert.Empty(t, condFalse.Reason) + assert.Empty(t, condFalse.Message) +} + +func TestMaestroReadonlyBundleHelpers_buildObjectsFromUnstructuredObj(t *testing.T) { + t.Run("single object returns one item", func(t *testing.T) { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "hypershift.openshift.io", Version: "v1beta1", Kind: "HostedCluster"}) + obj.SetName("test-hc") + obj.SetNamespace("test-ns") + + objs, err := buildObjectsFromUnstructuredObj(obj) + require.NoError(t, err) + require.Len(t, objs, 1) + assert.Equal(t, obj, objs[0].Object) + }) + + t.Run("list object flattens items", func(t *testing.T) { + // Build an Unstructured that represents a K8s ConfigMapList with two ConfigMap items. + item1 := map[string]interface{}{"kind": "HostedClusterList", "metadata": map[string]interface{}{"name": "cm1"}} + item2 := map[string]interface{}{"kind": "HostedClusterList", "metadata": map[string]interface{}{"name": "cm2"}} + listObj := &unstructured.Unstructured{} + listObj.SetUnstructuredContent(map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMapList", + "items": []interface{}{item1, item2}, + }) + objs, err := buildObjectsFromUnstructuredObj(listObj) + require.NoError(t, err) + require.Len(t, objs, 2) + + // RawExtension has Object set (not Raw) when coming from buildObjectsFromUnstructuredObj; unmarshal to typed. + require.NotNil(t, objs[0].Object, "Object should be set") + u1 := objs[0].Object.(*unstructured.Unstructured) + cm1 := &hsv1beta1.HostedCluster{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(u1.UnstructuredContent(), cm1) + require.NoError(t, err) + assert.Equal(t, "cm1", cm1.Name) + + u2 := objs[1].Object.(*unstructured.Unstructured) + cm2 := &hsv1beta1.HostedCluster{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(u2.UnstructuredContent(), cm2) + require.NoError(t, err) + assert.Equal(t, "cm2", cm2.Name) + }) +} + +func TestMaestroReadonlyBundleHelpers_getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(t *testing.T) { + validJSON := `{"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"test"}}` + + tests := []struct { + name string + bundle *workv1.ManifestWork + want string + wantErr bool + errSub string + }{ + { + name: "success - returns raw JSON", + bundle: buildTestMaestroBundleWithStatusFeedback("bundle-1", "ns", validJSON), + want: validJSON, + }, + { + name: "error - zero manifests", + bundle: &workv1.ManifestWork{ + Status: workv1.ManifestWorkStatus{ + ResourceStatus: workv1.ManifestResourceStatus{ + Manifests: []workv1.ManifestCondition{}, + }, + }, + }, + wantErr: true, + errSub: "expected exactly one resource within the Maestro Bundle, got 0", + }, + { + name: "error - two manifests", + bundle: &workv1.ManifestWork{ + Status: workv1.ManifestWorkStatus{ + ResourceStatus: workv1.ManifestResourceStatus{ + Manifests: []workv1.ManifestCondition{ + {ResourceMeta: workv1.ManifestResourceMeta{}, Conditions: []metav1.Condition{}}, + {ResourceMeta: workv1.ManifestResourceMeta{}, Conditions: []metav1.Condition{}}, + }, + }, + }, + }, + wantErr: true, + errSub: "expected exactly one resource within the Maestro Bundle, got 2", + }, + { + name: "error - zero status feedback values", + bundle: &workv1.ManifestWork{ + Status: workv1.ManifestWorkStatus{ + ResourceStatus: workv1.ManifestResourceStatus{ + Manifests: []workv1.ManifestCondition{ + { + ResourceMeta: workv1.ManifestResourceMeta{}, + StatusFeedbacks: workv1.StatusFeedbackResult{Values: []workv1.FeedbackValue{}}, + Conditions: []metav1.Condition{}, + }, + }, + }, + }, + }, + wantErr: true, + errSub: "expected exactly one status feedback value", + }, + { + name: "error - wrong feedback name", + bundle: func() *workv1.ManifestWork { + b := buildTestMaestroBundleWithStatusFeedback("b", "ns", validJSON) + b.Status.ResourceStatus.Manifests[0].StatusFeedbacks.Values[0].Name = "wrong" + return b + }(), + wantErr: true, + errSub: "expected status feedback value name to be 'resource', got wrong", + }, + { + name: "error - wrong feedback type", + bundle: func() *workv1.ManifestWork { + b := buildTestMaestroBundleWithStatusFeedback("b", "ns", validJSON) + b.Status.ResourceStatus.Manifests[0].StatusFeedbacks.Values[0].Value.Type = workv1.String + return b + }(), + wantErr: true, + errSub: "expected status feedback value type to be JsonRaw", + }, + { + name: "error - nil JsonRaw", + bundle: func() *workv1.ManifestWork { + b := buildTestMaestroBundleWithStatusFeedback("b", "ns", validJSON) + b.Status.ResourceStatus.Manifests[0].StatusFeedbacks.Values[0].Value.JsonRaw = nil + return b + }(), + wantErr: true, + errSub: "expected status feedback value JsonRaw to be not nil", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(tt.bundle) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errSub) + } else { + require.NoError(t, err) + assert.Equal(t, tt.want, string(got)) + } + }) + } +} diff --git a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go index 61593e7b65e..edb75cfe8bd 100644 --- a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go +++ b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go @@ -15,25 +15,15 @@ package controllers import ( "context" - "encoding/json" "errors" "fmt" "net/http" "time" - workv1 "open-cluster-management.io/api/work/v1" - - "k8s.io/apimachinery/pkg/api/equality" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" "github.com/Azure/ARO-HCP/backend/pkg/informers" "github.com/Azure/ARO-HCP/backend/pkg/listers" "github.com/Azure/ARO-HCP/backend/pkg/maestro" - "github.com/Azure/ARO-HCP/internal/api" "github.com/Azure/ARO-HCP/internal/database" "github.com/Azure/ARO-HCP/internal/ocm" "github.com/Azure/ARO-HCP/internal/utils" @@ -131,7 +121,7 @@ func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) SyncOnc var syncErrors []error for _, maestroBundleReference := range existingServiceProviderCluster.Status.MaestroReadonlyBundles { - err = c.readAndPersistMaestroBundleContent(ctx, existingCluster, maestroBundleReference, maestroClient) + err = readAndPersistMaestroReadonlyBundleContent(ctx, c.cosmosClient, existingCluster.ID, maestroBundleReference, maestroClient) if err != nil { syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to read and persist HostedCluster: %w", err))) } @@ -141,187 +131,6 @@ func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) SyncOnc return utils.TrackError(errors.Join(syncErrors...)) } -// calculateManagementClusterContentFromMaestroBundle calculates the desired ManagementClusterContent from the given Maestro Bundle reference. -// It returns the desired ManagementClusterContent or an error if the calculation fails. -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) calculateManagementClusterContentFromMaestroBundle( - ctx context.Context, cluster *api.HCPOpenShiftCluster, hostedClusterMaestroBundleReference *api.MaestroBundleReference, - maestroClient maestro.Client, -) (*api.ManagementClusterContent, error) { - managementClusterContentResourceID := controllerutils.ManagementClusterContentResourceIDFromClusterResourceID(cluster.ID, hostedClusterMaestroBundleReference.Name) - desired := controllerutils.NewInitialManagementClusterContent(managementClusterContentResourceID) - - existingMaestroBundle, err := maestroClient.Get(ctx, hostedClusterMaestroBundleReference.MaestroAPIMaestroBundleName, metav1.GetOptions{}) - if err != nil && !k8serrors.IsNotFound(err) { - return nil, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle: %w", err)) - } - if k8serrors.IsNotFound(err) { - degradedCondition := c.buildDegradedCondition(api.ConditionTrue, "MaestroBundleNotFound", err.Error()) - controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) - return desired, nil - } - - rawBytes, err := c.getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(existingMaestroBundle) - if err != nil { - degradedCondition := c.buildDegradedCondition(api.ConditionTrue, "MaestroBundleStatusFeedbackNotAvailable", err.Error()) - controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) - return desired, nil - } - - kubeContentMaxSizeExceeded := len(rawBytes) > c.kubeContentMaxSizeBytes() - var kubeContextMaxSizeExceededConditionMessage string - // We only set the retrieved content if it is within the size limit. If it - // is outside the limit we set the Degraded condition communicating the issue. - // We use unstructuredObj.Unstructured to deserialize the content so we can - // implement logic agnostic to the type of the content being retrieved. - unstructuredObj := &unstructured.Unstructured{} - err = json.Unmarshal(rawBytes, unstructuredObj) - if err != nil { - return nil, utils.TrackError(fmt.Errorf("failed to unmarshal object from status feedback value: %w", err)) - } - kind := unstructuredObj.GetKind() - if kind == "" { - return nil, utils.TrackError(fmt.Errorf("expected kind to be not empty")) - } - - objs, err := c.buildObjectsFromUnstructuredObj(unstructuredObj) - if err != nil { - return nil, utils.TrackError(fmt.Errorf("failed to build objects from unstructured object: %w", err)) - } - var degradedCondition api.Condition - if !kubeContentMaxSizeExceeded { - // TODO is ListMeta or TypeMeta required at the metav1.List level? - desired.Status.KubeContent = &metav1.List{Items: objs} - degradedCondition = c.buildDegradedCondition(api.ConditionFalse, "NoErrors", "As expected.") - } else { - kubeContextMaxSizeExceededConditionMessage = fmt.Sprintf("%s serialized size %.2f MiB exceeds Kube content max size %.2f MiB;", kind, float64(len(rawBytes))/(1024*1024), float64(c.kubeContentMaxSizeBytes())/(1024*1024)) - degradedCondition = c.buildDegradedCondition(api.ConditionTrue, "KubeContentMaxSizeExceeded", kubeContextMaxSizeExceededConditionMessage) - } - controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) - - return desired, nil -} - -// buildObjectsFromUnstructuredObj builds the list of objects from the given unstructured object. -// If the unstructured object is a list, it flattens the list of objects from the list of items. Nested lists are not flattened. -// If the unstructured object is not a list, it returns a list with a single item being the single object. -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) buildObjectsFromUnstructuredObj(unstructuredObj *unstructured.Unstructured) ([]runtime.RawExtension, error) { - if !unstructuredObj.IsList() { - return []runtime.RawExtension{{Object: unstructuredObj}}, nil - } - - objs := []runtime.RawExtension{} - err := unstructuredObj.EachListItem(func(o runtime.Object) error { - objs = append(objs, runtime.RawExtension{Object: o}) - return nil - }) - if err != nil { - return nil, utils.TrackError(err) - } - - return objs, nil -} - -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) buildDegradedCondition(conditionStatus api.ConditionStatus, conditionReason string, conditionMessage string) api.Condition { - return api.Condition{ - Type: "Degraded", - Status: conditionStatus, - Reason: conditionReason, - Message: conditionMessage, - } -} - -// readAndPersistMaestroBundleContent reads the Maestro Bundle content from the given Maestro Bundle reference -// and persists it in Cosmos. -// To achieve that, it gets the Maestro readonly bundle pointing to the Cluster's HostedCluster, it extracts the -// returned content by Maestro by taking it from the Maestro bundles's status feedback rule that contains the whole object and then it persists it -// in Cosmos. -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) readAndPersistMaestroBundleContent( - ctx context.Context, cluster *api.HCPOpenShiftCluster, hostedClusterMaestroBundleReference *api.MaestroBundleReference, - maestroClient maestro.Client, -) error { - - desired, err := c.calculateManagementClusterContentFromMaestroBundle(ctx, cluster, hostedClusterMaestroBundleReference, maestroClient) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to calculate ManagementClusterContent from Maestro Bundle: %w", err)) - } - - managementClusterContentsDBClient := c.cosmosClient.ManagementClusterContents( - cluster.ID.SubscriptionID, - cluster.ID.ResourceGroupName, - cluster.ID.Name, - ) - - existing, err := managementClusterContentsDBClient.Get(ctx, desired.CosmosMetadata.ResourceID.Name) - if err != nil && !database.IsResponseError(err, http.StatusNotFound) { - return utils.TrackError(fmt.Errorf("failed to get ManagementClusterContent: %w", err)) - } - if database.IsResponseError(err, http.StatusNotFound) { - _, err := managementClusterContentsDBClient.Create(ctx, desired, nil) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to create ManagementClusterContent: %w", err)) - } - return nil - } - - // We set the Cosmos ETag to the existing one to avoid conflicts when replacing the document - // unless someone else has modified the document since we last read it. - desired.CosmosETag = existing.CosmosETag - - // If we haven't been able to retrieve the content but there was already content - // stored we keep the previously existing stored content. - if desired.Status.KubeContent == nil && existing.Status.KubeContent != nil { - desired.Status.KubeContent = existing.Status.KubeContent - } - - if equality.Semantic.DeepEqual(existing, desired) { - return nil - } - - _, err = managementClusterContentsDBClient.Replace(ctx, desired, nil) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to replace ManagementClusterContent: %w", err)) - } - - return nil -} - -// kubeContentMaxSizeBytes returns the maximum size of a Cosmos DB item in bytes. -// 2MB is the maximum size of a Cosmos DB item (https://learn.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-item-limits). -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) kubeContentMaxSizeBytes() int { - return 1887436 // 2MB * 0.9 -} - -// getSingleResourceStatusFeedbackRawJSONFromMaestroBundle gets the single resource status feedback raw JSON from a Maestro Bundle. -// Used to extract the content of the resource from the Maestro Bundle. -// An error is returned if the Maestro Bundle does not contain a single resource or if the resource does not contain a single status feedback value -// with its name being "resource" and its type being JsonRaw. -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(maestroBundle *workv1.ManifestWork) (json.RawMessage, error) { - resourceStatusManifests := maestroBundle.Status.ResourceStatus.Manifests - if len(resourceStatusManifests) != 1 { - return nil, utils.TrackError(fmt.Errorf("expected exactly one resource within the Maestro Bundle, got %d", len(resourceStatusManifests))) - } - - statusFeedbackValues := resourceStatusManifests[0].StatusFeedbacks.Values - if len(statusFeedbackValues) == 0 { - return nil, utils.TrackError(fmt.Errorf("expected exactly one status feedback value within the Maestro Bundle resource, got %d", len(statusFeedbackValues))) - } - if len(statusFeedbackValues) > 1 { - return nil, utils.TrackError(fmt.Errorf("expected exactly one status feedback value within the Maestro Bundle resource, got %d", len(statusFeedbackValues))) - } - statusFeedbackValue := statusFeedbackValues[0] - if statusFeedbackValue.Name != "resource" { - return nil, utils.TrackError(fmt.Errorf("expected status feedback value name to be 'resource', got %s", statusFeedbackValue.Name)) - } - if statusFeedbackValue.Value.Type != workv1.JsonRaw { - return nil, utils.TrackError(fmt.Errorf("expected status feedback value type to be JsonRaw, got %s", statusFeedbackValue.Value.Type)) - } - if statusFeedbackValue.Value.JsonRaw == nil { - return nil, utils.TrackError(fmt.Errorf("expected status feedback value JsonRaw to be not nil")) - } - - return []byte(*statusFeedbackValue.Value.JsonRaw), nil -} - func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) CooldownChecker() controllerutils.CooldownChecker { return c.cooldownChecker } diff --git a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go index dfc0197e968..e029306e6ab 100644 --- a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go +++ b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go @@ -138,208 +138,6 @@ func (e *errorInjectingSPCCRUD) Get(ctx context.Context, resourceID string) (*ap var _ database.ServiceProviderClusterCRUD = &errorInjectingSPCCRUD{} -func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_buildDegradedCondition(t *testing.T) { - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{} - - cond := syncer.buildDegradedCondition(api.ConditionTrue, "MaestroBundleNotFound", "bundle not found") - assert.Equal(t, "Degraded", cond.Type) - assert.Equal(t, api.ConditionTrue, cond.Status) - assert.Equal(t, "MaestroBundleNotFound", cond.Reason) - assert.Equal(t, "bundle not found", cond.Message) - - condFalse := syncer.buildDegradedCondition(api.ConditionFalse, "", "") - assert.Equal(t, api.ConditionFalse, condFalse.Status) - assert.Empty(t, condFalse.Reason) - assert.Empty(t, condFalse.Message) -} - -func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_buildObjectsFromUnstructuredObj(t *testing.T) { - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{} - - t.Run("single object returns one item", func(t *testing.T) { - obj := &unstructured.Unstructured{} - obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "hypershift.openshift.io", Version: "v1beta1", Kind: "HostedCluster"}) - obj.SetName("test-hc") - obj.SetNamespace("test-ns") - - objs, err := syncer.buildObjectsFromUnstructuredObj(obj) - require.NoError(t, err) - require.Len(t, objs, 1) - assert.Equal(t, obj, objs[0].Object) - }) - - t.Run("list object flattens items", func(t *testing.T) { - // Build an Unstructured that represents a K8s ConfigMapList with two ConfigMap items. - item1 := map[string]interface{}{"kind": "HostedClusterList", "metadata": map[string]interface{}{"name": "cm1"}} - item2 := map[string]interface{}{"kind": "HostedClusterList", "metadata": map[string]interface{}{"name": "cm2"}} - listObj := &unstructured.Unstructured{} - listObj.SetUnstructuredContent(map[string]interface{}{ - "apiVersion": "v1", - "kind": "ConfigMapList", - "items": []interface{}{item1, item2}, - }) - objs, err := syncer.buildObjectsFromUnstructuredObj(listObj) - require.NoError(t, err) - require.Len(t, objs, 2) - - // RawExtension has Object set (not Raw) when coming from buildObjectsFromUnstructuredObj; unmarshal to typed. - require.NotNil(t, objs[0].Object, "Object should be set") - u1 := objs[0].Object.(*unstructured.Unstructured) - cm1 := &hsv1beta1.HostedCluster{} - err = runtime.DefaultUnstructuredConverter.FromUnstructured(u1.UnstructuredContent(), cm1) - require.NoError(t, err) - assert.Equal(t, "cm1", cm1.Name) - - u2 := objs[1].Object.(*unstructured.Unstructured) - cm2 := &hsv1beta1.HostedCluster{} - err = runtime.DefaultUnstructuredConverter.FromUnstructured(u2.UnstructuredContent(), cm2) - require.NoError(t, err) - assert.Equal(t, "cm2", cm2.Name) - }) -} - -// buildTestMaestroBundleWithStatusFeedback builds a ManifestWork with exactly one resource status manifest -// and one status feedback value named "resource" with JsonRaw type. -func buildTestMaestroBundleWithStatusFeedback(name, namespace, rawJSON string) *workv1.ManifestWork { - jsonRaw := rawJSON - return &workv1.ManifestWork{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, - Status: workv1.ManifestWorkStatus{ - ResourceStatus: workv1.ManifestResourceStatus{ - Manifests: []workv1.ManifestCondition{ - { - ResourceMeta: workv1.ManifestResourceMeta{ - Group: "hypershift.openshift.io", - Version: "v1beta1", - Kind: "HostedCluster", - Name: "test-hc", - Namespace: "test-ns", - }, - StatusFeedbacks: workv1.StatusFeedbackResult{ - Values: []workv1.FeedbackValue{ - { - Name: "resource", - Value: workv1.FieldValue{ - Type: workv1.JsonRaw, - JsonRaw: &jsonRaw, - }, - }, - }, - }, - Conditions: []metav1.Condition{}, - }, - }, - }, - }, - } -} - -func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(t *testing.T) { - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{} - validJSON := `{"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"test"}}` - - tests := []struct { - name string - bundle *workv1.ManifestWork - want string - wantErr bool - errSub string - }{ - { - name: "success - returns raw JSON", - bundle: buildTestMaestroBundleWithStatusFeedback("bundle-1", "ns", validJSON), - want: validJSON, - }, - { - name: "error - zero manifests", - bundle: &workv1.ManifestWork{ - Status: workv1.ManifestWorkStatus{ - ResourceStatus: workv1.ManifestResourceStatus{ - Manifests: []workv1.ManifestCondition{}, - }, - }, - }, - wantErr: true, - errSub: "expected exactly one resource within the Maestro Bundle, got 0", - }, - { - name: "error - two manifests", - bundle: &workv1.ManifestWork{ - Status: workv1.ManifestWorkStatus{ - ResourceStatus: workv1.ManifestResourceStatus{ - Manifests: []workv1.ManifestCondition{ - {ResourceMeta: workv1.ManifestResourceMeta{}, Conditions: []metav1.Condition{}}, - {ResourceMeta: workv1.ManifestResourceMeta{}, Conditions: []metav1.Condition{}}, - }, - }, - }, - }, - wantErr: true, - errSub: "expected exactly one resource within the Maestro Bundle, got 2", - }, - { - name: "error - zero status feedback values", - bundle: &workv1.ManifestWork{ - Status: workv1.ManifestWorkStatus{ - ResourceStatus: workv1.ManifestResourceStatus{ - Manifests: []workv1.ManifestCondition{ - { - ResourceMeta: workv1.ManifestResourceMeta{}, - StatusFeedbacks: workv1.StatusFeedbackResult{Values: []workv1.FeedbackValue{}}, - Conditions: []metav1.Condition{}, - }, - }, - }, - }, - }, - wantErr: true, - errSub: "expected exactly one status feedback value", - }, - { - name: "error - wrong feedback name", - bundle: func() *workv1.ManifestWork { - b := buildTestMaestroBundleWithStatusFeedback("b", "ns", validJSON) - b.Status.ResourceStatus.Manifests[0].StatusFeedbacks.Values[0].Name = "wrong" - return b - }(), - wantErr: true, - errSub: "expected status feedback value name to be 'resource', got wrong", - }, - { - name: "error - wrong feedback type", - bundle: func() *workv1.ManifestWork { - b := buildTestMaestroBundleWithStatusFeedback("b", "ns", validJSON) - b.Status.ResourceStatus.Manifests[0].StatusFeedbacks.Values[0].Value.Type = workv1.String - return b - }(), - wantErr: true, - errSub: "expected status feedback value type to be JsonRaw", - }, - { - name: "error - nil JsonRaw", - bundle: func() *workv1.ManifestWork { - b := buildTestMaestroBundleWithStatusFeedback("b", "ns", validJSON) - b.Status.ResourceStatus.Manifests[0].StatusFeedbacks.Values[0].Value.JsonRaw = nil - return b - }(), - wantErr: true, - errSub: "expected status feedback value JsonRaw to be not nil", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := syncer.getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(tt.bundle) - if tt.wantErr { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.errSub) - } else { - require.NoError(t, err) - assert.Equal(t, tt.want, string(got)) - } - }) - } -} - func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_calculateManagementClusterContentFromMaestroBundle(t *testing.T) { clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) cluster := &api.HCPOpenShiftCluster{ @@ -413,8 +211,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_calculat mockMaestro := maestro.NewMockClient(ctrl) tt.maestroGet(mockMaestro) - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{} - got, err := syncer.calculateManagementClusterContentFromMaestroBundle(context.Background(), cluster, ref, mockMaestro) + got, err := calculateManagementClusterContentFromMaestroBundle(context.Background(), cluster.ID, ref, mockMaestro) if tt.wantErr { require.Error(t, err) assert.Contains(t, err.Error(), tt.errSub) @@ -453,8 +250,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP mockDB := databasetesting.NewMockDBClient() mccCRUD := mockDB.ManagementClusterContents("sub", "rg", "cluster") - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{cosmosClient: mockDB} - err := syncer.readAndPersistMaestroBundleContent(ctx, cluster, ref, mockMaestro) + err := readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, cluster.ID, ref, mockMaestro) require.NoError(t, err) // Content should have been created (name = bundle internal name) @@ -483,8 +279,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP _, err := mccCRUD.Create(ctx, existing, nil) require.NoError(t, err) - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{cosmosClient: mockDB} - err = syncer.readAndPersistMaestroBundleContent(ctx, cluster, ref, mockMaestro) + err = readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, cluster.ID, ref, mockMaestro) require.NoError(t, err) got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) @@ -520,8 +315,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP _, err := mccCRUD.Create(ctx, existing, nil) require.NoError(t, err) - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{cosmosClient: mockDB} - err = syncer.readAndPersistMaestroBundleContent(ctx, cluster, ref, mockMaestro) + err = readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, cluster.ID, ref, mockMaestro) require.NoError(t, err) got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) @@ -535,7 +329,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP ctrl := gomock.NewController(t) mockMaestro := maestro.NewMockClient(ctrl) b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) - // Get is called once when building desired for pre-create, and once inside readAndPersistMaestroBundleContent. + // Get is called once when building desired for pre-create, and once inside readAndPersistMaestroReadonlyBundleContent. mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil).Times(2) mockDB := databasetesting.NewMockDBClient() @@ -543,8 +337,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) // Pre-create content that matches exactly what the syncer would compute (same KubeContent and Degraded=False condition) // so that DeepEqual(existing, desired) is true and Replace is not called. - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{cosmosClient: mockDB} - desired, err := syncer.calculateManagementClusterContentFromMaestroBundle(ctx, cluster, ref, mockMaestro) + desired, err := calculateManagementClusterContentFromMaestroBundle(ctx, cluster.ID, ref, mockMaestro) require.NoError(t, err) require.NotNil(t, desired) existing := &api.ManagementClusterContent{ @@ -555,7 +348,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP _, err = mccCRUD.Create(ctx, existing, nil) require.NoError(t, err) - err = syncer.readAndPersistMaestroBundleContent(ctx, cluster, ref, mockMaestro) + err = readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, cluster.ID, ref, mockMaestro) require.NoError(t, err) // Document should still exist with same content (no Replace was needed) @@ -587,8 +380,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP }, } - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{cosmosClient: mockDB} - err := syncer.readAndPersistMaestroBundleContent(ctx, cluster, ref, mockMaestro) + err := readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, cluster.ID, ref, mockMaestro) require.Error(t, err) assert.Contains(t, err.Error(), "failed to replace ManagementClusterContent") assert.True(t, database.IsResponseError(err, http.StatusPreconditionFailed), "expected 412 Precondition Failed") @@ -609,8 +401,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP }, } - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{cosmosClient: mockDB} - err := syncer.readAndPersistMaestroBundleContent(ctx, cluster, ref, mockMaestro) + err := readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, cluster.ID, ref, mockMaestro) require.Error(t, err) assert.Contains(t, err.Error(), "failed to get ManagementClusterContent") assert.Contains(t, err.Error(), "cosmos connection error") diff --git a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go index 56bd42bc735..9451dfe27b7 100644 --- a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go +++ b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go @@ -15,25 +15,15 @@ package controllers import ( "context" - "encoding/json" "errors" "fmt" "net/http" "time" - workv1 "open-cluster-management.io/api/work/v1" - - "k8s.io/apimachinery/pkg/api/equality" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" "github.com/Azure/ARO-HCP/backend/pkg/informers" "github.com/Azure/ARO-HCP/backend/pkg/listers" "github.com/Azure/ARO-HCP/backend/pkg/maestro" - "github.com/Azure/ARO-HCP/internal/api" "github.com/Azure/ARO-HCP/internal/database" "github.com/Azure/ARO-HCP/internal/ocm" "github.com/Azure/ARO-HCP/internal/utils" @@ -130,7 +120,7 @@ func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) SyncOn var syncErrors []error for _, maestroBundleReference := range existingServiceProviderNodePool.Status.MaestroReadonlyBundles { - err = c.readAndPersistMaestroBundleContent(ctx, existingNodePool, maestroBundleReference, maestroClient) + err = readAndPersistMaestroReadonlyBundleContent(ctx, c.cosmosClient, existingNodePool.ID.Parent, maestroBundleReference, maestroClient) if err != nil { syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to read and persist NodePool: %w", err))) } @@ -140,187 +130,6 @@ func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) SyncOn return utils.TrackError(errors.Join(syncErrors...)) } -// calculateManagementClusterContentFromMaestroBundle calculates the desired ManagementClusterContent from the given Maestro Bundle reference. -// It returns the desired ManagementClusterContent or an error if the calculation fails. -func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) calculateManagementClusterContentFromMaestroBundle( - ctx context.Context, nodePool *api.HCPOpenShiftClusterNodePool, nodePoolMaestroBundleReference *api.MaestroBundleReference, - maestroClient maestro.Client, -) (*api.ManagementClusterContent, error) { - managementClusterContentResourceID := controllerutils.ManagementClusterContentResourceIDFromClusterResourceID(nodePool.ID.Parent, nodePoolMaestroBundleReference.Name) - desired := controllerutils.NewInitialManagementClusterContent(managementClusterContentResourceID) - - existingMaestroBundle, err := maestroClient.Get(ctx, nodePoolMaestroBundleReference.MaestroAPIMaestroBundleName, metav1.GetOptions{}) - if err != nil && !k8serrors.IsNotFound(err) { - return nil, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle: %w", err)) - } - if k8serrors.IsNotFound(err) { - degradedCondition := c.buildDegradedCondition(api.ConditionTrue, "MaestroBundleNotFound", err.Error()) - controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) - return desired, nil - } - - rawBytes, err := c.getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(existingMaestroBundle) - if err != nil { - degradedCondition := c.buildDegradedCondition(api.ConditionTrue, "MaestroBundleStatusFeedbackNotAvailable", err.Error()) - controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) - return desired, nil - } - - kubeContentMaxSizeExceeded := len(rawBytes) > c.kubeContentMaxSizeBytes() - var kubeContextMaxSizeExceededConditionMessage string - // We only set the retrieved content if it is within the size limit. If it - // is outside the limit we set the Degraded condition communicating the issue. - // We use unstructuredObj.Unstructured to deserialize the content so we can - // implement logic agnostic to the type of the content being retrieved. - unstructuredObj := &unstructured.Unstructured{} - err = json.Unmarshal(rawBytes, unstructuredObj) - if err != nil { - return nil, utils.TrackError(fmt.Errorf("failed to unmarshal object from status feedback value: %w", err)) - } - kind := unstructuredObj.GetKind() - if kind == "" { - return nil, utils.TrackError(fmt.Errorf("expected kind to be not empty")) - } - - objs, err := c.buildObjectsFromUnstructuredObj(unstructuredObj) - if err != nil { - return nil, utils.TrackError(fmt.Errorf("failed to build objects from unstructured object: %w", err)) - } - var degradedCondition api.Condition - if !kubeContentMaxSizeExceeded { - // TODO is ListMeta or TypeMeta required at the metav1.List level? - desired.Status.KubeContent = &metav1.List{Items: objs} - degradedCondition = c.buildDegradedCondition(api.ConditionFalse, "NoErrors", "As expected.") - } else { - kubeContextMaxSizeExceededConditionMessage = fmt.Sprintf("%s serialized size %.2f MiB exceeds Kube content max size %.2f MiB;", kind, float64(len(rawBytes))/(1024*1024), float64(c.kubeContentMaxSizeBytes())/(1024*1024)) - degradedCondition = c.buildDegradedCondition(api.ConditionTrue, "KubeContentMaxSizeExceeded", kubeContextMaxSizeExceededConditionMessage) - } - controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) - - return desired, nil -} - -// buildObjectsFromUnstructuredObj builds the list of objects from the given unstructured object. -// If the unstructured object is a list, it flattens the list of objects from the list of items. Nested lists are not flattened. -// If the unstructured object is not a list, it returns a list with a single item being the single object. -func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) buildObjectsFromUnstructuredObj(unstructuredObj *unstructured.Unstructured) ([]runtime.RawExtension, error) { - if !unstructuredObj.IsList() { - return []runtime.RawExtension{{Object: unstructuredObj}}, nil - } - - objs := []runtime.RawExtension{} - err := unstructuredObj.EachListItem(func(o runtime.Object) error { - objs = append(objs, runtime.RawExtension{Object: o}) - return nil - }) - if err != nil { - return nil, utils.TrackError(err) - } - - return objs, nil -} - -func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) buildDegradedCondition(conditionStatus api.ConditionStatus, conditionReason string, conditionMessage string) api.Condition { - return api.Condition{ - Type: "Degraded", - Status: conditionStatus, - Reason: conditionReason, - Message: conditionMessage, - } -} - -// readAndPersistMaestroBundleContent reads the Maestro Bundle content from the given Maestro Bundle reference -// and persists it in Cosmos. -// To achieve that, it gets the Maestro readonly bundle pointing to the Cluster's NodePool, it extracts the -// returned content by Maestro by taking it from the Maestro bundles's status feedback rule that contains the whole object and then it persists it -// in Cosmos. -func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) readAndPersistMaestroBundleContent( - ctx context.Context, nodePool *api.HCPOpenShiftClusterNodePool, nodePoolMaestroBundleReference *api.MaestroBundleReference, - maestroClient maestro.Client, -) error { - - desired, err := c.calculateManagementClusterContentFromMaestroBundle(ctx, nodePool, nodePoolMaestroBundleReference, maestroClient) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to calculate ManagementClusterContent from Maestro Bundle: %w", err)) - } - - managementClusterContentsDBClient := c.cosmosClient.ManagementClusterContents( - nodePool.ID.SubscriptionID, - nodePool.ID.ResourceGroupName, - nodePool.ID.Parent.Name, - ) - - existing, err := managementClusterContentsDBClient.Get(ctx, desired.CosmosMetadata.ResourceID.Name) - if err != nil && !database.IsResponseError(err, http.StatusNotFound) { - return utils.TrackError(fmt.Errorf("failed to get ManagementClusterContent: %w", err)) - } - if database.IsResponseError(err, http.StatusNotFound) { - _, err := managementClusterContentsDBClient.Create(ctx, desired, nil) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to create ManagementClusterContent: %w", err)) - } - return nil - } - - // We set the Cosmos ETag to the existing one to avoid conflicts when replacing the document - // unless someone else has modified the document since we last read it. - desired.CosmosETag = existing.CosmosETag - - // If we haven't been able to retrieve the content but there was already content - // stored we keep the previously existing stored content. - if desired.Status.KubeContent == nil && existing.Status.KubeContent != nil { - desired.Status.KubeContent = existing.Status.KubeContent - } - - if equality.Semantic.DeepEqual(existing, desired) { - return nil - } - - _, err = managementClusterContentsDBClient.Replace(ctx, desired, nil) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to replace ManagementClusterContent: %w", err)) - } - - return nil -} - -// kubeContentMaxSizeBytes returns the maximum size of a Cosmos DB item in bytes. -// 2MB is the maximum size of a Cosmos DB item (https://learn.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-item-limits). -func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) kubeContentMaxSizeBytes() int { - return 1887436 // 2MB * 0.9 -} - -// getSingleResourceStatusFeedbackRawJSONFromMaestroBundle gets the single resource status feedback raw JSON from a Maestro Bundle. -// Used to extract the content of the resource from the Maestro Bundle. -// An error is returned if the Maestro Bundle does not contain a single resource or if the resource does not contain a single status feedback value -// with its name being "resource" and its type being JsonRaw. -func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(maestroBundle *workv1.ManifestWork) (json.RawMessage, error) { - resourceStatusManifests := maestroBundle.Status.ResourceStatus.Manifests - if len(resourceStatusManifests) != 1 { - return nil, utils.TrackError(fmt.Errorf("expected exactly one resource within the Maestro Bundle, got %d", len(resourceStatusManifests))) - } - - statusFeedbackValues := resourceStatusManifests[0].StatusFeedbacks.Values - if len(statusFeedbackValues) == 0 { - return nil, utils.TrackError(fmt.Errorf("expected exactly one status feedback value within the Maestro Bundle resource, got %d", len(statusFeedbackValues))) - } - if len(statusFeedbackValues) > 1 { - return nil, utils.TrackError(fmt.Errorf("expected exactly one status feedback value within the Maestro Bundle resource, got %d", len(statusFeedbackValues))) - } - statusFeedbackValue := statusFeedbackValues[0] - if statusFeedbackValue.Name != "resource" { - return nil, utils.TrackError(fmt.Errorf("expected status feedback value name to be 'resource', got %s", statusFeedbackValue.Name)) - } - if statusFeedbackValue.Value.Type != workv1.JsonRaw { - return nil, utils.TrackError(fmt.Errorf("expected status feedback value type to be JsonRaw, got %s", statusFeedbackValue.Value.Type)) - } - if statusFeedbackValue.Value.JsonRaw == nil { - return nil, utils.TrackError(fmt.Errorf("expected status feedback value JsonRaw to be not nil")) - } - - return []byte(*statusFeedbackValue.Value.JsonRaw), nil -} - func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) CooldownChecker() controllerutils.CooldownChecker { return c.cooldownChecker } diff --git a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go new file mode 100644 index 00000000000..971332b14b2 --- /dev/null +++ b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go @@ -0,0 +1,648 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + workv1 "open-cluster-management.io/api/work/v1" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos" + + hsv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + + "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" + "github.com/Azure/ARO-HCP/backend/pkg/maestro" + "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/api/arm" + "github.com/Azure/ARO-HCP/internal/database" + "github.com/Azure/ARO-HCP/internal/databasetesting" + "github.com/Azure/ARO-HCP/internal/ocm" +) + +// errorInjectingDBClientForNodePoolReadPersist wraps MockDBClient to return error-injecting CRUDs. +type errorInjectingDBClientForNodePoolReadPersist struct { + *databasetesting.MockDBClient + mccCRUD database.ManagementClusterContentCRUD + spnpCRUD database.ServiceProviderNodePoolCRUD +} + +func (e *errorInjectingDBClientForNodePoolReadPersist) ManagementClusterContents(subscriptionID, resourceGroupName, clusterName string) database.ManagementClusterContentCRUD { + if e.mccCRUD != nil { + return e.mccCRUD + } + return e.MockDBClient.ManagementClusterContents(subscriptionID, resourceGroupName, clusterName) +} + +func (e *errorInjectingDBClientForNodePoolReadPersist) ServiceProviderNodePools(subscriptionID, resourceGroupName, clusterName, nodePoolName string) database.ServiceProviderNodePoolCRUD { + if e.spnpCRUD != nil { + return e.spnpCRUD + } + return e.MockDBClient.ServiceProviderNodePools(subscriptionID, resourceGroupName, clusterName, nodePoolName) +} + +var _ database.DBClient = &errorInjectingDBClientForNodePoolReadPersist{} + +// errorInjectingSPNPCRUD wraps ServiceProviderNodePoolCRUD to allow error injection. +type errorInjectingSPNPCRUD struct { + database.ServiceProviderNodePoolCRUD + getErr error +} + +func (e *errorInjectingSPNPCRUD) Get(ctx context.Context, resourceID string) (*api.ServiceProviderNodePool, error) { + if e.getErr != nil { + return nil, e.getErr + } + return e.ServiceProviderNodePoolCRUD.Get(ctx, resourceID) +} + +// errorInjectingMCCCRUDForNodePool wraps ManagementClusterContentCRUD to allow error injection. +type errorInjectingMCCCRUDForNodePool struct { + database.ManagementClusterContentCRUD + getResult *api.ManagementClusterContent + getErr error + replaceErr error +} + +func (e *errorInjectingMCCCRUDForNodePool) Get(ctx context.Context, resourceID string) (*api.ManagementClusterContent, error) { + if e.getErr != nil { + return nil, e.getErr + } + if e.getResult != nil { + return e.getResult, nil + } + return e.ManagementClusterContentCRUD.Get(ctx, resourceID) +} + +func (e *errorInjectingMCCCRUDForNodePool) Replace(ctx context.Context, obj *api.ManagementClusterContent, opts *azcosmos.ItemOptions) (*api.ManagementClusterContent, error) { + if e.replaceErr != nil { + return nil, e.replaceErr + } + return e.ManagementClusterContentCRUD.Replace(ctx, obj, opts) +} + +var _ database.ManagementClusterContentCRUD = &errorInjectingMCCCRUDForNodePool{} + +func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_calculateManagementClusterContentFromMaestroBundle(t *testing.T) { + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + } + bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "test-nodepool") + ref := &api.MaestroBundleReference{ + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "bundle-name", + } + + np := hsv1beta1.NodePool{TypeMeta: metav1.TypeMeta{APIVersion: "hypershift.openshift.io/v1beta1", Kind: "NodePool"}, ObjectMeta: metav1.ObjectMeta{Name: "np1", Namespace: "ns1"}} + npJSONBytes, err := json.Marshal(np) + require.NoError(t, err) + validNPJSON := string(npJSONBytes) + + tests := []struct { + name string + maestroGet func(*maestro.MockClient) + wantDegraded bool + wantKubeContent bool + wantErr bool + errSub string + }{ + { + name: "bundle not found - desired with degraded condition", + maestroGet: func(m *maestro.MockClient) { + m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "bundle-name")) + }, + wantDegraded: true, + wantKubeContent: false, + }, + { + name: "maestro bundle get error - returns error", + maestroGet: func(m *maestro.MockClient) { + m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(nil, fmt.Errorf("connection error")) + }, + wantErr: true, + errSub: "failed to get Maestro Bundle", + }, + { + name: "bundle has invalid status feedback - desired with degraded", + maestroGet: func(m *maestro.MockClient) { + b := &workv1.ManifestWork{ + Status: workv1.ManifestWorkStatus{ + ResourceStatus: workv1.ManifestResourceStatus{ + Manifests: []workv1.ManifestCondition{ + {ResourceMeta: workv1.ManifestResourceMeta{}, StatusFeedbacks: workv1.StatusFeedbackResult{Values: []workv1.FeedbackValue{}}, Conditions: []metav1.Condition{}}, + }, + }, + }, + } + m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + }, + wantDegraded: true, + wantKubeContent: false, + }, + { + name: "success - desired with kube content", + maestroGet: func(m *maestro.MockClient) { + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validNPJSON) + m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + }, + wantDegraded: false, + wantKubeContent: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + tt.maestroGet(mockMaestro) + + got, err := calculateManagementClusterContentFromMaestroBundle(context.Background(), nodepool.ID.Parent, ref, mockMaestro) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errSub) + } else { + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, tt.wantKubeContent, got.Status.KubeContent != nil && len(got.Status.KubeContent.Items) > 0) + hasDegradedTrue := controllerutils.IsConditionTrue(got.Status.Conditions, "Degraded") + assert.Equal(t, tt.wantDegraded, hasDegradedTrue) + } + }) + } +} + +func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_readAndPersistMaestroBundleContent(t *testing.T) { + ctx := context.Background() + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + } + bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "test-nodepool") + ref := &api.MaestroBundleReference{ + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "bundle-name", + } + + np := hsv1beta1.NodePool{TypeMeta: metav1.TypeMeta{APIVersion: "hypershift.openshift.io/v1beta1", Kind: "NodePool"}, ObjectMeta: metav1.ObjectMeta{Name: "np1", Namespace: "ns1"}} + npJSONBytes, err := json.Marshal(np) + require.NoError(t, err) + validNPJSON := string(npJSONBytes) + + // ManagementClusterContents is keyed by the cluster name (nodePool.ID.Parent.Name) + const mccContainerSub = "sub" + const mccContainerRG = "rg" + const mccContainerCluster = "cluster" + existingMCCResourceIDStr := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/%s/managementClusterContents/%s", + mccContainerSub, mccContainerRG, mccContainerCluster, bundleInternalName) + + t.Run("creates new ManagementClusterContent when not found", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validNPJSON) + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + + mockDB := databasetesting.NewMockDBClient() + mccCRUD := mockDB.ManagementClusterContents(mccContainerSub, mccContainerRG, mccContainerCluster) + + err := readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, nodepool.ID.Parent, ref, mockMaestro) + require.NoError(t, err) + + got, err := mccCRUD.Get(ctx, string(bundleInternalName)) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.Status.KubeContent) + require.Len(t, got.Status.KubeContent.Items, 1) + }) + + t.Run("replaces existing when content changed", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validNPJSON) + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + + mockDB := databasetesting.NewMockDBClient() + mccCRUD := mockDB.ManagementClusterContents(mccContainerSub, mccContainerRG, mccContainerCluster) + existingRID := api.Must(azcorearm.ParseResourceID(existingMCCResourceIDStr)) + existing := &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, + ResourceID: *existingRID, + Status: api.ManagementClusterContentStatus{KubeContent: &metav1.List{Items: []runtime.RawExtension{}}}, + } + _, err := mccCRUD.Create(ctx, existing, nil) + require.NoError(t, err) + + err = readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, nodepool.ID.Parent, ref, mockMaestro) + require.NoError(t, err) + + got, err := mccCRUD.Get(ctx, string(bundleInternalName)) + require.NoError(t, err) + require.NotNil(t, got.Status.KubeContent) + require.Len(t, got.Status.KubeContent.Items, 1) + }) + + t.Run("keeps existing kube content when desired has no content (degraded)", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + // Bundle with no valid status feedback -> desired has no KubeContent + b := &workv1.ManifestWork{ + Status: workv1.ManifestWorkStatus{ + ResourceStatus: workv1.ManifestResourceStatus{ + Manifests: []workv1.ManifestCondition{ + {ResourceMeta: workv1.ManifestResourceMeta{}, StatusFeedbacks: workv1.StatusFeedbackResult{Values: []workv1.FeedbackValue{}}, Conditions: []metav1.Condition{}}, + }, + }, + }, + } + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + + mockDB := databasetesting.NewMockDBClient() + mccCRUD := mockDB.ManagementClusterContents(mccContainerSub, mccContainerRG, mccContainerCluster) + existingRID := api.Must(azcorearm.ParseResourceID(existingMCCResourceIDStr)) + existingContent := &metav1.List{Items: []runtime.RawExtension{{Raw: []byte(`{}`)}}} + existing := &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, + ResourceID: *existingRID, + Status: api.ManagementClusterContentStatus{KubeContent: existingContent}, + } + _, err := mccCRUD.Create(ctx, existing, nil) + require.NoError(t, err) + + err = readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, nodepool.ID.Parent, ref, mockMaestro) + require.NoError(t, err) + + got, err := mccCRUD.Get(ctx, string(bundleInternalName)) + require.NoError(t, err) + require.NotNil(t, got.Status.KubeContent) + assert.Equal(t, existingContent.Items[0].Raw, got.Status.KubeContent.Items[0].Raw) + }) + + t.Run("no replace occurs when content has not changed", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validNPJSON) + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil).Times(2) + + mockDB := databasetesting.NewMockDBClient() + mccCRUD := mockDB.ManagementClusterContents(mccContainerSub, mccContainerRG, mccContainerCluster) + existingRID := api.Must(azcorearm.ParseResourceID(existingMCCResourceIDStr)) + desired, err := calculateManagementClusterContentFromMaestroBundle(ctx, nodepool.ID.Parent, ref, mockMaestro) + require.NoError(t, err) + existing := &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, + ResourceID: *existingRID, + Status: desired.Status, + } + _, err = mccCRUD.Create(ctx, existing, nil) + require.NoError(t, err) + + err = readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, nodepool.ID.Parent, ref, mockMaestro) + require.NoError(t, err) + + got, err := mccCRUD.Get(ctx, string(bundleInternalName)) + require.NoError(t, err) + require.NotNil(t, got.Status.KubeContent) + require.Len(t, got.Status.KubeContent.Items, 1) + }) + + t.Run("error occurs when object has been modified in Cosmos since we retrieved it", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validNPJSON) + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + + existingRID := api.Must(azcorearm.ParseResourceID(existingMCCResourceIDStr)) + existingDoc := &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, + ResourceID: *existingRID, + Status: api.ManagementClusterContentStatus{KubeContent: &metav1.List{Items: []runtime.RawExtension{{Raw: []byte(validNPJSON)}}}}, + } + + mockDB := &errorInjectingDBClientForNodePoolReadPersist{ + MockDBClient: databasetesting.NewMockDBClient(), + mccCRUD: &errorInjectingMCCCRUDForNodePool{ + getResult: existingDoc, + replaceErr: databasetesting.NewPreconditionFailedError(), + }, + } + + err := readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, nodepool.ID.Parent, ref, mockMaestro) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to replace ManagementClusterContent") + assert.True(t, database.IsResponseError(err, http.StatusPreconditionFailed), "expected 412 Precondition Failed") + }) + + t.Run("error occurs when managementClusterContentsDBClient.Get fails", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validNPJSON) + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + + mockDB := &errorInjectingDBClientForNodePoolReadPersist{ + MockDBClient: databasetesting.NewMockDBClient(), + mccCRUD: &errorInjectingMCCCRUDForNodePool{ + getErr: fmt.Errorf("cosmos connection error"), + }, + } + + err := readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, nodepool.ID.Parent, ref, mockMaestro) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to get ManagementClusterContent") + assert.Contains(t, err.Error(), "cosmos connection error") + }) +} + +func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnce_NodePoolNotFound(t *testing.T) { + mockDBClient := databasetesting.NewMockDBClient() + syncer := &readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + // No nodepool in DB -> Get returns NotFound -> SyncOnce returns nil (no work to do) + err := syncer.SyncOnce(context.Background(), key) + assert.NoError(t, err) +} + +func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnce_GetServiceProviderNodePoolError(t *testing.T) { + ctx := context.Background() + + baseMockDB := databasetesting.NewMockDBClient() + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + nodepoolsCRUD := baseMockDB.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + expectedError := fmt.Errorf("database error") + mockDBClient := &errorInjectingDBClientForNodePoolReadPersist{ + MockDBClient: baseMockDB, + spnpCRUD: &errorInjectingSPNPCRUD{ + getErr: expectedError, + }, + } + + syncer := &readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + } + + err = syncer.SyncOnce(ctx, key) + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to get or create ServiceProviderNodePool") +} + +func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnce_NoMaestroReadonlyBundlesRefs(t *testing.T) { + ctx := context.Background() + mockDBClient := databasetesting.NewMockDBClient() + syncer := &readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + nodepoolsCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + // SPNP with no bundle references -> SyncOnce returns nil (nothing to process) + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + } + spnpCRUD := mockDBClient.ServiceProviderNodePools(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName, key.HCPNodePoolName) + _, err = spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + + err = syncer.SyncOnce(ctx, key) + assert.NoError(t, err) +} + +func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnce_GetProvisionShardError(t *testing.T) { + ctrl := gomock.NewController(t) + ctx := context.Background() + + mockDBClient := databasetesting.NewMockDBClient() + mockClusterService := ocm.NewMockClusterServiceClientSpec(ctrl) + + syncer := &readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + clusterServiceClient: mockClusterService, + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + nodepoolsCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "test-nodepool") + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + {Name: bundleInternalName, MaestroAPIMaestroBundleName: "bundle-name"}, + }, + }, + } + spnpCRUD := mockDBClient.ServiceProviderNodePools(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName, key.HCPNodePoolName) + _, err = spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + + mockClusterService.EXPECT(). + GetClusterProvisionShard(gomock.Any(), nodepool.ServiceProviderProperties.ClusterServiceID). + Return(nil, fmt.Errorf("provision shard error")) + + err = syncer.SyncOnce(ctx, key) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to get Cluster Provision Shard") +} + +func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnce_ReadAndPersistFlow(t *testing.T) { + ctrl := gomock.NewController(t) + ctx := context.Background() + + mockDBClient := databasetesting.NewMockDBClient() + mockClusterService := ocm.NewMockClusterServiceClientSpec(ctrl) + mockMaestroBuilder := maestro.NewMockMaestroClientBuilder(ctrl) + mockMaestroClient := maestro.NewMockClient(ctrl) + + syncer := &readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + clusterServiceClient: mockClusterService, + maestroClientBuilder: mockMaestroBuilder, + maestroSourceEnvironmentIdentifier: "test-env", + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + nodepoolsCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "test-nodepool") + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + {Name: bundleInternalName, MaestroAPIMaestroBundleName: "bundle-name"}, + }, + }, + } + spnpCRUD := mockDBClient.ServiceProviderNodePools(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName, key.HCPNodePoolName) + _, err = spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + + provisionShard := buildTestProvisionShard("test-consumer") + mockClusterService.EXPECT(). + GetClusterProvisionShard(gomock.Any(), nodepool.ServiceProviderProperties.ClusterServiceID). + Return(provisionShard, nil) + + restEndpoint := provisionShard.MaestroConfig().RestApiConfig().Url() + grpcEndpoint := provisionShard.MaestroConfig().GrpcApiConfig().Url() + consumerName := provisionShard.MaestroConfig().ConsumerName() + sourceID := maestro.GenerateMaestroSourceID("test-env", provisionShard.ID()) + mockMaestroBuilder.EXPECT(). + NewClient(gomock.Any(), restEndpoint, grpcEndpoint, consumerName, sourceID). + Return(mockMaestroClient, nil) + + validNPJSON := `{"apiVersion":"hypershift.openshift.io/v1beta1","kind":"NodePool","metadata":{"name":"np1","namespace":"ns1"}}` + bundle := buildTestMaestroBundleWithStatusFeedback("bundle-name", "test-consumer", validNPJSON) + mockMaestroClient.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(bundle, nil) + + err = syncer.SyncOnce(ctx, key) + require.NoError(t, err) + + // ManagementClusterContents is keyed by cluster name (nodePool.ID.Parent.Name) + mccCRUD := mockDBClient.ManagementClusterContents(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName) + got, err := mccCRUD.Get(ctx, string(bundleInternalName)) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.Status.KubeContent) + require.Len(t, got.Status.KubeContent.Items, 1) + + var u unstructured.Unstructured + err = json.Unmarshal(got.Status.KubeContent.Items[0].Raw, &u) + require.NoError(t, err) + assert.Equal(t, "NodePool", u.GetKind()) + assert.Equal(t, "np1", u.GetName()) +} diff --git a/backend/pkg/maestro/maestro_bundle.go b/backend/pkg/maestro/maestro_bundle.go index 4536e34c5fa..e30fa8c42fe 100644 --- a/backend/pkg/maestro/maestro_bundle.go +++ b/backend/pkg/maestro/maestro_bundle.go @@ -53,3 +53,27 @@ func GetOrCreateMaestroBundle(ctx context.Context, maestroClient Client, maestro existingMaestroBundle, err = maestroClient.Get(ctx, maestroBundle.Name, metav1.GetOptions{}) return existingMaestroBundle, err } + +// ForEachMaestroBundle lists all Maestro Bundles across all pages matching opts using the Maestro client client and calls fn +// for each Maestro Bundle. +// Pagination can be controlled by setting the Limit attribute in opts. +// fn is called for each bundle in page order. If fn returns an error, iteration stops and the error is returned. +func ForEachMaestroBundle(ctx context.Context, client Client, opts metav1.ListOptions, fn func(*workv1.ManifestWork) error) error { + for { + bundles, err := client.List(ctx, opts) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to list Maestro Bundles: %w", err)) + } + for i := range bundles.Items { + if err := fn(&bundles.Items[i]); err != nil { + return err + } + } + token := bundles.GetContinue() + if token == "" { + break + } + opts.Continue = token + } + return nil +} diff --git a/backend/pkg/maestro/maestro_bundle_test.go b/backend/pkg/maestro/maestro_bundle_test.go index 7397e9d4da2..3b38b23cc1f 100644 --- a/backend/pkg/maestro/maestro_bundle_test.go +++ b/backend/pkg/maestro/maestro_bundle_test.go @@ -16,6 +16,7 @@ package maestro import ( "context" + "errors" "fmt" "testing" @@ -128,3 +129,109 @@ func TestGetOrCreateMaestroBundle(t *testing.T) { }) } } + +func TestForEachMaestroBundle(t *testing.T) { + ctx := context.Background() + callbackErr := errors.New("stop here") + + tests := []struct { + name string + setupMock func(*MockClient) + listOpts metav1.ListOptions + errAfterName string + wantNames []string + wantErr bool + errContains []string + errIs error + }{ + { + name: "empty list", + setupMock: func(m *MockClient) { + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{ + Items: []workv1.ManifestWork{}, + }, nil) + }, + wantNames: []string{}, + }, + { + name: "invokes fn for each bundle in order on one page", + setupMock: func(m *MockClient) { + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{ + Items: []workv1.ManifestWork{ + {ObjectMeta: metav1.ObjectMeta{Name: "a"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "b"}}, + }, + }, nil) + }, + wantNames: []string{"a", "b"}, + }, + { + name: "paginates using continue token", + setupMock: func(m *MockClient) { + m.EXPECT().List(gomock.Any(), metav1.ListOptions{Limit: 10}).Return(&workv1.ManifestWorkList{ + ListMeta: metav1.ListMeta{Continue: "next-page"}, + Items: []workv1.ManifestWork{{ObjectMeta: metav1.ObjectMeta{Name: "first"}}}, + }, nil) + m.EXPECT().List(gomock.Any(), metav1.ListOptions{Limit: 10, Continue: "next-page"}).Return(&workv1.ManifestWorkList{ + Items: []workv1.ManifestWork{{ObjectMeta: metav1.ObjectMeta{Name: "second"}}}, + }, nil) + }, + listOpts: metav1.ListOptions{Limit: 10}, + wantNames: []string{"first", "second"}, + }, + { + name: "returns wrapped error when list fails", + setupMock: func(m *MockClient) { + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("upstream failure")) + }, + wantNames: []string{}, + wantErr: true, + errContains: []string{"failed to list Maestro Bundles", "upstream failure"}, + }, + { + name: "returns callback error and stops iteration", + setupMock: func(m *MockClient) { + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{ + Items: []workv1.ManifestWork{ + {ObjectMeta: metav1.ObjectMeta{Name: "a"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "b"}}, + }, + }, nil) + }, + errAfterName: "a", + wantNames: []string{"a"}, + wantErr: true, + errIs: callbackErr, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := NewMockClient(ctrl) + tt.setupMock(mockMaestro) + + names := make([]string, 0) + err := ForEachMaestroBundle(ctx, mockMaestro, tt.listOpts, func(mw *workv1.ManifestWork) error { + names = append(names, mw.Name) + if tt.errAfterName != "" && mw.Name == tt.errAfterName { + return callbackErr + } + return nil + }) + + assert.Equal(t, tt.wantNames, names) + if tt.wantErr { + require.Error(t, err) + if tt.errIs != nil { + require.ErrorIs(t, err, tt.errIs) + } + for _, sub := range tt.errContains { + assert.Contains(t, err.Error(), sub) + } + } else { + require.NoError(t, err) + } + }) + } +} From 5a97c2a13cd16249426f0e46a68a6d165fdaf1f6 Mon Sep 17 00:00:00 2001 From: Miguel Soriano Date: Mon, 23 Mar 2026 18:26:00 +0100 Subject: [PATCH 08/16] fix: correctly calculate cs cluster href from nodepool href in maestro np controllers --- ...ped_maestro_readonly_bundles_controller.go | 33 +++++++++++++------ ...aestro_readonly_bundles_controller_test.go | 30 ++++++++++++----- ...tro_readonly_bundles_content_controller.go | 6 +++- frontend/pkg/frontend/frontend_test.go | 4 +-- frontend/pkg/frontend/helpers_test.go | 4 +-- internal/ocm/internalid.go | 24 +++++++++++--- 6 files changed, 74 insertions(+), 27 deletions(-) diff --git a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go index 0a4610d5ebf..9576dc55cfb 100644 --- a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go @@ -167,7 +167,11 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) SyncOnce(ctx context. // we are guaranteed to have a shard allocated for the cluster. If this changes in the future // we would need to change the logic in controllers to check that the retrieved cluster has a // shard allocated. - clusterProvisionShard, err := c.clusterServiceClient.GetClusterProvisionShard(ctx, existingNodePool.ServiceProviderProperties.ClusterServiceID) + + csClusterID := existingNodePool.ServiceProviderProperties.ClusterServiceID.ClusterID() + csClusterHREF := ocm.GenerateAROHCPClusterHREF(csClusterID) + csClusterInternalID := api.Must(api.NewInternalID(csClusterHREF)) + clusterProvisionShard, err := c.clusterServiceClient.GetClusterProvisionShard(ctx, csClusterInternalID) if err != nil { return utils.TrackError(fmt.Errorf("failed to get Cluster Provision Shard from Cluster Service: %w", err)) } @@ -181,7 +185,7 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) SyncOnce(ctx context. return utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) } - csCluster, err := c.clusterServiceClient.GetCluster(ctx, existingNodePool.ServiceProviderProperties.ClusterServiceID) + csCluster, err := c.clusterServiceClient.GetCluster(ctx, csClusterInternalID) if err != nil { return utils.TrackError(fmt.Errorf("failed to get Cluster from Cluster Service: %w", err)) } @@ -308,8 +312,8 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildClusterEmptyNode // decouple from this concept of CDNamespace and we would use the stored value when needed. However, if we want to // create resources in the same namespace as the old ones then we would still need to keep forever the concept of "env name part used to calculate // some k8s resource names/namespaces". - nodePoolNamespace := c.getNodePoolNamespace(c.maestroSourceEnvironmentIdentifier, csClusterID, csNodePoolID) - nodePoolName := csClusterDomainPrefix + nodePoolNamespace := c.getNodePoolNamespace(c.maestroSourceEnvironmentIdentifier, csClusterID) + nodePoolName := c.getNodePoolName(csClusterDomainPrefix, csNodePoolID) // We first build the resource (manifest) that we want to put within the Maestro Bundle. // The resource is empty and it only has the type information and the object meta @@ -331,7 +335,8 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildClusterEmptyNode // buildInitialReadonlyMaestroBundleForNodePool builds an initial readonly Maestro Bundle a the Cluster's Hypershift NodePool. // Used to create the readonly Maestro bundle associated to it. func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildInitialReadonlyMaestroBundleForNodePool(nodePool *api.HCPOpenShiftClusterNodePool, csClusterDomainPrefix string, maestroBundleNamespacedName types.NamespacedName) *workv1.ManifestWork { - hypershiftNodePool := c.buildClusterEmptyNodePool(nodePool.ServiceProviderProperties.ClusterServiceID.ID(), csClusterDomainPrefix, nodePool.ID.Name) + csClusterID := nodePool.ServiceProviderProperties.ClusterServiceID.ClusterID() + hypershiftNodePool := c.buildClusterEmptyNodePool(csClusterID, csClusterDomainPrefix, nodePool.ID.Name) maestroBundleResourceIdentifier := workv1.ResourceIdentifier{ Group: hsv1beta1.SchemeGroupVersion.Group, Resource: "nodepools", @@ -342,13 +347,21 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildInitialReadonlyM return buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName, maestroBundleResourceIdentifier, hypershiftNodePool, readonlyBundleManagedByK8sLabelValueNodePoolScoped) } -// getNodePoolNamespace gets the namespace for the node pool based on the environment name, the cluster service OCM Cluster ID and the node pool service OCM Node Pool ID. +// getNodePoolNamespace gets the namespace for the node pool based on the environment name and the cluster service OCM Cluster ID. // For example, if the Node Pool URL is /api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111/nodepools/XXXX then the -// cluster service OCM Cluster ID is 11111111111111111111111111111111 and the node pool service OCM Node Pool ID is XXXX. -// The namespace is of the format ocm---. This is how CS calculates Hypershift's NodePool namespace. +// cluster service OCM Cluster ID is 11111111111111111111111111111111. +// The namespace is of the format ocm--. This is how CS calculates Hypershift's NodePool namespace. // Internally in CS this is the "CDNamespace" attribute associated to the cluster. -func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) getNodePoolNamespace(envName string, csClusterID string, csNodePoolID string) string { - return fmt.Sprintf("ocm-%s-%s-%s", envName, csClusterID, csNodePoolID) +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) getNodePoolNamespace(envName string, csClusterID string) string { + return fmt.Sprintf("ocm-%s-%s", envName, csClusterID) +} + +// getNodePoolName gets the name for the node pool based on the cluster domain prefix and the node pool service OCM Node Pool ID. +// For example, if the Node Pool URL is /api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111/nodepools/XXXX and the +// cluster's domain prefix is test-domprefix then the name is test-domprefix-XXXX. +// The name is of the format -. +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) getNodePoolName(csClusterDomainPrefix string, csNodePoolID string) string { + return fmt.Sprintf("%s-%s", csClusterDomainPrefix, csNodePoolID) } func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) CooldownChecker() controllerutils.CooldownChecker { diff --git a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go index b02b0107cae..0b40acf6764 100644 --- a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go +++ b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go @@ -84,27 +84,40 @@ func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_buildClusterEmptyNodeP csClusterID := "11111111111111111111111111111111" csClusterDomainPrefix := "test-domprefix" csNodePoolID := "nodepool-id-1234" + expectedNodePoolName := fmt.Sprintf("%s-%s", csClusterDomainPrefix, csNodePoolID) np := syncer.buildClusterEmptyNodePool(csClusterID, csClusterDomainPrefix, csNodePoolID) assert.NotNil(t, np) assert.Equal(t, "NodePool", np.Kind) assert.Equal(t, hsv1beta1.SchemeGroupVersion.String(), np.APIVersion) - assert.Equal(t, csClusterDomainPrefix, np.Name) - assert.Equal(t, fmt.Sprintf("ocm-%s-%s-%s", syncer.maestroSourceEnvironmentIdentifier, csClusterID, csNodePoolID), np.Namespace) + assert.Equal(t, expectedNodePoolName, np.Name) + assert.Equal(t, fmt.Sprintf("ocm-%s-%s", syncer.maestroSourceEnvironmentIdentifier, csClusterID), np.Namespace) } func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_getNodePoolNamespace(t *testing.T) { envName := "testenv" csClusterID := "11111111111111111111111111111111" + expected := fmt.Sprintf("ocm-%s-%s", envName, csClusterID) + + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + maestroSourceEnvironmentIdentifier: envName, + } + + result := syncer.getNodePoolNamespace(envName, csClusterID) + assert.Equal(t, expected, result) +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_getNodePoolName(t *testing.T) { + envName := "testenv" + csClusterDomainPrefix := "test-domprefix" csNodePoolID := "nodepool-abc" - expected := fmt.Sprintf("ocm-%s-%s-%s", envName, csClusterID, csNodePoolID) + expected := fmt.Sprintf("%s-%s", csClusterDomainPrefix, csNodePoolID) syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ - maestroSourceEnvironmentIdentifier: envName, - maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + maestroSourceEnvironmentIdentifier: envName, } - result := syncer.getNodePoolNamespace(envName, csClusterID, csNodePoolID) + result := syncer.getNodePoolName(csClusterDomainPrefix, csNodePoolID) assert.Equal(t, expected, result) } @@ -149,6 +162,7 @@ func TestBuildInitialReadonlyMaestroBundleForNodePool(t *testing.T) { Name: "test-maestro-api-maestro-bundle-name", Namespace: "test-maestro-consumer", } + expectedNodePoolName := fmt.Sprintf("%s-%s", csClusterDomainPrefix, strings.ToLower(nodepool.Name)) bundle := syncer.buildInitialReadonlyMaestroBundleForNodePool(nodepool, csClusterDomainPrefix, maestroBundleNamespacedName) require.NotNil(t, bundle) @@ -162,8 +176,8 @@ func TestBuildInitialReadonlyMaestroBundleForNodePool(t *testing.T) { manifestConfig := bundle.Spec.ManifestConfigs[0] assert.Equal(t, "nodepools", manifestConfig.ResourceIdentifier.Resource) assert.Equal(t, hsv1beta1.SchemeGroupVersion.Group, manifestConfig.ResourceIdentifier.Group) - assert.Equal(t, csClusterDomainPrefix, manifestConfig.ResourceIdentifier.Name) - expectedNamespace := fmt.Sprintf("ocm-testenv-11111111111111111111111111111111-%s", strings.ToLower(nodepoolResourceID.Name)) + assert.Equal(t, expectedNodePoolName, manifestConfig.ResourceIdentifier.Name) + expectedNamespace := "ocm-testenv-11111111111111111111111111111111" assert.Equal(t, expectedNamespace, manifestConfig.ResourceIdentifier.Namespace) expectedNodePool := syncer.buildClusterEmptyNodePool( diff --git a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go index 9451dfe27b7..fdb3532447d 100644 --- a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go +++ b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go @@ -24,6 +24,7 @@ import ( "github.com/Azure/ARO-HCP/backend/pkg/informers" "github.com/Azure/ARO-HCP/backend/pkg/listers" "github.com/Azure/ARO-HCP/backend/pkg/maestro" + "github.com/Azure/ARO-HCP/internal/api" "github.com/Azure/ARO-HCP/internal/database" "github.com/Azure/ARO-HCP/internal/ocm" "github.com/Azure/ARO-HCP/internal/utils" @@ -105,7 +106,10 @@ func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) SyncOn // we are guaranteed to have a shard allocated for the cluster. If this changes in the future // we would need to change the logic in controllers to check that the retrieved cluster has a // shard allocated. - clusterProvisionShard, err := c.clusterServiceClient.GetClusterProvisionShard(ctx, existingNodePool.ServiceProviderProperties.ClusterServiceID) + csClusterID := existingNodePool.ServiceProviderProperties.ClusterServiceID.ClusterID() + csClusterHREF := ocm.GenerateAROHCPClusterHREF(csClusterID) + csClusterInternalID := api.Must(api.NewInternalID(csClusterHREF)) + clusterProvisionShard, err := c.clusterServiceClient.GetClusterProvisionShard(ctx, csClusterInternalID) if err != nil { return utils.TrackError(fmt.Errorf("failed to get Cluster Provision Shard from Cluster Service: %w", err)) } diff --git a/frontend/pkg/frontend/frontend_test.go b/frontend/pkg/frontend/frontend_test.go index 46ff678776d..7868a66b1db 100644 --- a/frontend/pkg/frontend/frontend_test.go +++ b/frontend/pkg/frontend/frontend_test.go @@ -54,7 +54,7 @@ func newClusterResourceID(t *testing.T) *azcorearm.ResourceID { } func newClusterInternalID(t *testing.T) ocm.InternalID { - internalID, err := api.NewInternalID(ocm.GenerateClusterHREF("myCluster")) + internalID, err := api.NewInternalID(ocm.GenerateOCMCommercialClusterHREF("myCluster")) require.NoError(t, err) return internalID } @@ -636,7 +636,7 @@ func TestRequestAdminCredential(t *testing.T) { mockCSClient.EXPECT(). PostBreakGlassCredential(gomock.Any(), clusterInternalID). Return(cmv1.NewBreakGlassCredential(). - HREF(ocm.GenerateBreakGlassCredentialHREF(clusterInternalID.String(), "0")).Build()) + HREF(ocm.GenerateOCMCommercialBreakGlassCredentialHREF(clusterInternalID.String(), "0")).Build()) } subs := map[string]*arm.Subscription{ diff --git a/frontend/pkg/frontend/helpers_test.go b/frontend/pkg/frontend/helpers_test.go index cc305f9f077..660bf8e0299 100644 --- a/frontend/pkg/frontend/helpers_test.go +++ b/frontend/pkg/frontend/helpers_test.go @@ -121,7 +121,7 @@ func TestCheckForProvisioningStateConflict(t *testing.T) { // Pre-populate the parent cluster in the database for nested resources (node pool, external auth) if tt.parentConflict != nil { parentResourceID := resourceID.Parent - clusterInternalID := api.Must(api.NewInternalID(ocm.GenerateClusterHREF("testCluster"))) + clusterInternalID := api.Must(api.NewInternalID(ocm.GenerateOCMCommercialClusterHREF("testCluster"))) parentCluster := &api.HCPOpenShiftCluster{ TrackedResource: arm.TrackedResource{ Resource: arm.Resource{ @@ -168,7 +168,7 @@ func TestCheckForProvisioningStateConflict(t *testing.T) { parentResourceID := resourceID.Parent if parentResourceID.ResourceType.Namespace == resourceID.ResourceType.Namespace { // Pre-populate the parent cluster with the test provisioning state - clusterInternalID := api.Must(api.NewInternalID(ocm.GenerateClusterHREF("testCluster"))) + clusterInternalID := api.Must(api.NewInternalID(ocm.GenerateOCMCommercialClusterHREF("testCluster"))) parentCluster := &api.HCPOpenShiftCluster{ TrackedResource: arm.TrackedResource{ Resource: arm.Resource{ diff --git a/internal/ocm/internalid.go b/internal/ocm/internalid.go index dc73c87752b..4d4145057a6 100644 --- a/internal/ocm/internalid.go +++ b/internal/ocm/internalid.go @@ -42,22 +42,38 @@ var ( aroHcpV1Alpha1ProvisionShardPattern = path.Join(aroHcpV1Alpha1ClusterPattern, provisionShardKey, "*") ) -func GenerateClusterHREF(clusterName string) string { +func GenerateOCMCommercialClusterHREF(clusterName string) string { return path.Join(v1Pattern, clusterKey, clusterName) } -func GenerateNodePoolHREF(clusterPath string, nodePoolName string) string { +func GenerateAROHCPClusterHREF(clusterName string) string { + return path.Join(aroHcpV1Alpha1Pattern, clusterKey, clusterName) +} + +func GenerateOCMCommercialNodePoolHREF(clusterPath string, nodePoolName string) string { return path.Join(clusterPath, nodePoolKey, nodePoolName) } -func GenerateExternalAuthHREF(clusterPath string, externalAuthName string) string { +func GenerateAROHCPNodePoolHREF(clusterPath string, nodePoolName string) string { + return path.Join(aroHcpV1Alpha1Pattern, clusterKey, clusterPath, nodePoolKey, nodePoolName) +} + +func GenerateOCMCommercialExternalAuthHREF(clusterPath string, externalAuthName string) string { return path.Join(clusterPath, externalAuthKey, externalAuthName) } -func GenerateBreakGlassCredentialHREF(clusterPath string, credentialName string) string { +func GenerateAROHCPExternalAuthHREF(clusterPath string, externalAuthName string) string { + return path.Join(aroHcpV1Alpha1Pattern, clusterKey, clusterPath, externalAuthKey, externalAuthName) +} + +func GenerateOCMCommercialBreakGlassCredentialHREF(clusterPath string, credentialName string) string { return path.Join(clusterPath, "break_glass_credentials", credentialName) } +func GenerateAROHCPBreakGlassCredentialHREF(clusterPath string, credentialName string) string { + return path.Join(aroHcpV1Alpha1Pattern, clusterKey, clusterPath, "break_glass_credentials", credentialName) +} + type InternalID = api.InternalID // getClusterClient returns a v1 ClusterClient from the InternalID. From 56605953e45f97e8c2d8961bd3033311502571d4 Mon Sep 17 00:00:00 2001 From: Miguel Soriano Date: Mon, 23 Mar 2026 21:17:23 +0100 Subject: [PATCH 09/16] fix: maestro client leaks on read and persist maestro readonly bundles controllers --- ..._scoped_maestro_readonly_bundles_content_controller.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go index fdb3532447d..9345d6cdce5 100644 --- a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go +++ b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go @@ -113,14 +113,14 @@ func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) SyncOn if err != nil { return utils.TrackError(fmt.Errorf("failed to get Cluster Provision Shard from Cluster Service: %w", err)) } - maestroClient, err := createMaestroClientFromCSProvisionShard(ctx, c.maestroSourceEnvironmentIdentifier, c.maestroClientBuilder, clusterProvisionShard) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) - } // We create a new context with a cancel function so we can cancel the Maestro client when the sync is done. // This is important to avoid leaking resources when the sync is done. ctx, cancel := context.WithCancel(ctx) defer cancel() + maestroClient, err := createMaestroClientFromCSProvisionShard(ctx, c.maestroSourceEnvironmentIdentifier, c.maestroClientBuilder, clusterProvisionShard) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) + } var syncErrors []error for _, maestroBundleReference := range existingServiceProviderNodePool.Status.MaestroReadonlyBundles { From 4cf4a03e8ff628d77646088239638feee6296d50 Mon Sep 17 00:00:00 2001 From: Miguel Soriano Date: Mon, 30 Mar 2026 13:38:08 +0200 Subject: [PATCH 10/16] feat: support cluster and nodepool scoped ManagementClusterContent cosmos resources Introduce distinct Cosmos resource types for managementClusterContents nested under an hcpOpenShiftCluster vs under a nodePool, and wire them through Cosmos CRUD, global listing (union query), and databasetesting mocks. We remove DBClient.ManagementClusterContents and we instead now provide its access through HCPClusters(...).ManagementClusterContents(...) or NodePools(...).ManagementClusterContents(...) instead. We now have a Cosmos global lister that retrieves all ManagementClusterContents at both cluster scope and nodepool scope. The same occurs for the corresponding informer. The cluster watching controller uses the ManagementclusterContent informer but it only ends up enqueuing cluster-scoped ManagementClusterContent documents. Now that NodePools have their own ManagementClusterContent subresource we update the maestro bundle internal name for node pools to have a fixed name of `readonlyHypershiftNodePool` instead of a prefix, as now there are no potential naming collisions anymore between nodepools of the same cluster or cross cluster. --- .../management_cluster_content_controller.go | 12 +-- ...agement_cluster_content_controller_test.go | 14 +++- .../nodepool_watching_controller.go | 3 +- ...ped_maestro_readonly_bundles_controller.go | 19 ++--- ...aestro_readonly_bundles_controller_test.go | 10 +-- .../maestro_readonly_bundle_helpers.go | 22 ++---- ...tro_readonly_bundles_content_controller.go | 4 +- ...eadonly_bundles_content_controller_test.go | 51 ++++++++----- ...tro_readonly_bundles_content_controller.go | 10 ++- ...eadonly_bundles_content_controller_test.go | 76 ++++++++----------- ...control_plane_active_version_controller.go | 2 +- ...ol_plane_active_version_controller_test.go | 2 +- backend/pkg/informers/informers.go | 11 +-- backend/pkg/informers/types.go | 2 + .../management_cluster_content_lister.go | 8 +- backend/pkg/listertesting/db_listers.go | 40 ++++++++++ backend/pkg/listertesting/db_listers_test.go | 55 ++++++++++++++ .../pkg/listertesting/slice_listers_test.go | 27 +++++++ backend/pkg/maestro/maestro_bundle.go | 2 +- internal/api/registry.go | 31 ++++---- .../api/types_management_cluster_content.go | 2 +- .../api/types_serviceprovider_nodepool.go | 8 +- internal/database/crud_hcpcluster.go | 33 ++++++++ .../crud_management_cluster_content.go | 4 + internal/database/database.go | 8 -- internal/database/global_lister.go | 44 ++++++++++- internal/databasetesting/mock_crud.go | 27 ++++++- internal/databasetesting/mock_dbclient.go | 6 -- .../databasetesting/mock_global_lister.go | 55 +++++++++++++- internal/databasetesting/mock_init.go | 32 ++++++++ 30 files changed, 459 insertions(+), 161 deletions(-) diff --git a/backend/pkg/controllers/controllerutils/management_cluster_content_controller.go b/backend/pkg/controllers/controllerutils/management_cluster_content_controller.go index 8f80e81de28..dbaf7a09955 100644 --- a/backend/pkg/controllers/controllerutils/management_cluster_content_controller.go +++ b/backend/pkg/controllers/controllerutils/management_cluster_content_controller.go @@ -23,8 +23,7 @@ import ( ) // NewInitialManagementClusterContent returns a new ManagementClusterContent with -// the given resource ID as its parent. The resource ID is assumed to be a -// cluster resource ID. +// the given full managementClusterContents ARM resource ID. // The returned value can be used to consistently initialize a new ManagementClusterContent func NewInitialManagementClusterContent(managementClusterContentResourceID *azcorearm.ResourceID) *api.ManagementClusterContent { return &api.ManagementClusterContent{ @@ -35,8 +34,9 @@ func NewInitialManagementClusterContent(managementClusterContentResourceID *azco } } -// ManagementClusterContentResourceIDFromClusterResourceID returns the resource ID for the -// ManagementClusterContent associated to the given cluster resource ID and maestro bundle internal name. -func ManagementClusterContentResourceIDFromClusterResourceID(clusterResourceID *azcorearm.ResourceID, maestroBundleInternalName api.MaestroBundleInternalName) *azcorearm.ResourceID { - return api.Must(azcorearm.ParseResourceID(fmt.Sprintf("%s/%s/%s", clusterResourceID.String(), api.ManagementClusterContentResourceTypeName, maestroBundleInternalName))) +// ManagementClusterContentResourceIDFromParentResourceID returns the resource ID for the +// ManagementClusterContent nested under parentResourceID with the given +// maestro bundle internal name. +func ManagementClusterContentResourceIDFromParentResourceID(parentResourceID *azcorearm.ResourceID, maestroBundleInternalName api.MaestroBundleInternalName) *azcorearm.ResourceID { + return api.Must(azcorearm.ParseResourceID(fmt.Sprintf("%s/%s/%s", parentResourceID.String(), api.ManagementClusterContentResourceTypeName, maestroBundleInternalName))) } diff --git a/backend/pkg/controllers/controllerutils/management_cluster_content_controller_test.go b/backend/pkg/controllers/controllerutils/management_cluster_content_controller_test.go index f5532cc597f..c3020fa6dd6 100644 --- a/backend/pkg/controllers/controllerutils/management_cluster_content_controller_test.go +++ b/backend/pkg/controllers/controllerutils/management_cluster_content_controller_test.go @@ -28,9 +28,19 @@ import ( func TestManagementClusterContentResourceIDFromClusterResourceID(t *testing.T) { clusterRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/mycluster")) - got := ManagementClusterContentResourceIDFromClusterResourceID(clusterRID, api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster) + got := ManagementClusterContentResourceIDFromParentResourceID(clusterRID, api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster) require.NotNil(t, got) - assert.Equal(t, got.ResourceType.Type, api.ManagementClusterContentResourceType.Type) + assert.Equal(t, got.ResourceType.Type, api.ClusterScopedManagementClusterContentResourceType.Type) // Name is the last segment of the resource ID (the management cluster content name) assert.Equal(t, got.Name, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) } + +func TestManagementClusterContentResourceIDFromNodePoolResourceID(t *testing.T) { + nodePoolRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/mycluster/nodePools/mynodepool")) + + got := ManagementClusterContentResourceIDFromParentResourceID(nodePoolRID, api.MaestroBundleInternalNameReadonlyHypershiftNodePool) + require.NotNil(t, got) + assert.Equal(t, got.ResourceType.Type, api.NodePoolScopedManagementClusterContentResourceType.Type) + // Name is the last segment of the resource ID (the management cluster content name) + assert.Equal(t, got.Name, string(api.MaestroBundleInternalNameReadonlyHypershiftNodePool)) +} diff --git a/backend/pkg/controllers/controllerutils/nodepool_watching_controller.go b/backend/pkg/controllers/controllerutils/nodepool_watching_controller.go index b99054fef42..139426d9f2e 100644 --- a/backend/pkg/controllers/controllerutils/nodepool_watching_controller.go +++ b/backend/pkg/controllers/controllerutils/nodepool_watching_controller.go @@ -63,7 +63,8 @@ func NewNodePoolWatchingController( if informers != nil { nodePoolInformer, _ := informers.NodePools() serviceProviderNodePoolInformer, _ := informers.ServiceProviderNodePools() - err := nodePoolController.QueueForInformers(resyncDuration, nodePoolInformer, serviceProviderNodePoolInformer) + managementClusterContentInformer, _ := informers.ManagementClusterContents() + err := nodePoolController.QueueForInformers(resyncDuration, nodePoolInformer, serviceProviderNodePoolInformer, managementClusterContentInformer) if err != nil { panic(err) // coding error } diff --git a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go index 9576dc55cfb..4d8fcea199b 100644 --- a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go @@ -18,7 +18,6 @@ import ( "errors" "fmt" "net/http" - "strings" "time" workv1 "open-cluster-management.io/api/work/v1" @@ -40,13 +39,13 @@ import ( ) const ( - // readonlyBundleManagedByK8sLabelValueClusterScoped is the K8s label associated to the readonlyBundleManagedByK8sLabelKey + // readonlyBundleManagedByK8sLabelValueNodePoolScoped is the K8s label associated to the readonlyBundleManagedByK8sLabelKey // key that indicates that the readonly Maestro bundle is managed by the create // nodepool scoped maestro readonly bundles controller. readonlyBundleManagedByK8sLabelValueNodePoolScoped = "create-nodepool-scoped-maestro-readonly-bundles-controller" ) -// createClusterScopedMaestroReadonlyBundlesSyncer is a controller that creates Maestro readonly bundles for the clusters. +// createNodePoolScopedMaestroReadonlyBundlesSyncer is a controller that creates Maestro readonly bundles for the node pools. // It is responsible for creating the Maestro readonly bundles and storing a reference to them in Cosmos. It does // not persist the content of the Maestro bundles themselves. That is the responsibility of the // readAndPersistMaestroReadonlyBundlesContentSyncer controller. @@ -101,14 +100,6 @@ func NewCreateNodePoolScopedMaestroReadonlyBundlesController( return controller } -// generateMaestroBundleInternalName generates a Maestro Bundle internal name for a given NodePool. -// Used to generate a Maestro Bundle internal name for a new Maestro Bundle reference. -// The generated name is a concatenation of the prefix with the lowercased name of the NodePool. -// Example: "readonlyHypershiftNodePool-my-nodepool" -func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) generateMaestroBundleInternalName(nodePool *api.HCPOpenShiftClusterNodePool) api.MaestroBundleInternalName { - return api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + strings.ToLower(nodePool.Name)) -} - func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) SyncOnce(ctx context.Context, key controllerutils.HCPNodePoolKey) error { existingNodePool, err := c.cosmosClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName).Get(ctx, key.HCPNodePoolName) if database.IsResponseError(err, http.StatusNotFound) { @@ -127,7 +118,7 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) SyncOnce(ctx context. // Any Maestro Bundle internal name that is not in this list will not be synced by the // controller and reported as an error. recognizedMaestroBundles := []api.MaestroBundleInternalName{ - c.generateMaestroBundleInternalName(existingNodePool), + api.MaestroBundleInternalNameReadonlyHypershiftNodePool, } var maestroBundlesToSync []api.MaestroBundleInternalName @@ -267,7 +258,7 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) syncMaestroBundle( var desiredMaestroBundle *workv1.ManifestWork switch maestroBundleInternalName { - case c.generateMaestroBundleInternalName(existingNodePool): + case api.MaestroBundleInternalNameReadonlyHypershiftNodePool: desiredMaestroBundle = c.buildInitialReadonlyMaestroBundleForNodePool(existingNodePool, csClusterDomainPrefix, maestroBundleNamespacedName) default: return lastPersistedSPNP, utils.TrackError(fmt.Errorf("unrecognized Maestro Bundle internal name: %s", maestroBundleInternalName)) @@ -296,7 +287,7 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) syncMaestroBundle( return lastPersistedSPNP, nil } -// buildClusterEmptyNodePool returns an empty hosted cluster representing a Cluster's Hypershift NodePool resource. +// buildClusterEmptyNodePool returns an empty node pool representing a Cluster's Hypershift NodePool resource. // It strictly contains the type information and the object meta information necessary to identify the resource in the management cluster. // It can be used to provide as the input of a Maestro resource bundle. func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildClusterEmptyNodePool(csClusterID string, csClusterDomainPrefix string, csNodePoolID string) *hsv1beta1.NodePool { diff --git a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go index 0b40acf6764..305114ebc07 100644 --- a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go +++ b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go @@ -123,7 +123,7 @@ func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_getNodePoolName(t *tes func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_buildInitialMaestroBundleReferenceForNodePool(t *testing.T) { generator := maestro.NewMaestroAPIMaestroBundleNameGenerator() - bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "my-nodepool") + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool ref, err := buildInitialMaestroBundleReference(bundleInternalName, generator) require.NoError(t, err) @@ -194,7 +194,7 @@ func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_syncMaestroBundle(t *t spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) - bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "test-nodepool") + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool tests := []struct { name string @@ -578,7 +578,7 @@ func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_AllBundlesAlr _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) require.NoError(t, err) - bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "test-nodepool") + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) spnp := &api.ServiceProviderNodePool{ CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, @@ -695,7 +695,7 @@ func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_SyncLoopExecu require.NoError(t, err) require.NotNil(t, updatedSPNP) - bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "test-nodepool") + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool bundleRef, err := updatedSPNP.Status.MaestroReadonlyBundles.Get(bundleInternalName) require.NoError(t, err) require.NotNil(t, bundleRef) @@ -744,7 +744,7 @@ func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_ProcessesPart _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) require.NoError(t, err) - bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "test-nodepool") + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) spnp := &api.ServiceProviderNodePool{ CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, diff --git a/backend/pkg/controllers/maestro_readonly_bundle_helpers.go b/backend/pkg/controllers/maestro_readonly_bundle_helpers.go index 4ffd238ee74..4fbc59f8276 100644 --- a/backend/pkg/controllers/maestro_readonly_bundle_helpers.go +++ b/backend/pkg/controllers/maestro_readonly_bundle_helpers.go @@ -200,15 +200,14 @@ func getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(maestroBundle *work } // calculateManagementClusterContentFromMaestroBundle builds the desired ManagementClusterContent from a Maestro -// bundle reference. clusterResourceID is the HCP cluster ARM resource ID (for node pool–scoped bundles, pass the -// node pool resource's parent cluster ID). +// bundle reference. parentResourceID is the ARM ID of the document parent func calculateManagementClusterContentFromMaestroBundle( ctx context.Context, - clusterResourceID *azcorearm.ResourceID, + parentResourceID *azcorearm.ResourceID, maestroBundleReference *api.MaestroBundleReference, maestroClient maestro.Client, ) (*api.ManagementClusterContent, error) { - managementClusterContentResourceID := controllerutils.ManagementClusterContentResourceIDFromClusterResourceID(clusterResourceID, maestroBundleReference.Name) + managementClusterContentResourceID := controllerutils.ManagementClusterContentResourceIDFromParentResourceID(parentResourceID, maestroBundleReference.Name) desired := controllerutils.NewInitialManagementClusterContent(managementClusterContentResourceID) existingMaestroBundle, err := maestroClient.Get(ctx, maestroBundleReference.MaestroAPIMaestroBundleName, metav1.GetOptions{}) @@ -263,26 +262,19 @@ func calculateManagementClusterContentFromMaestroBundle( } // readAndPersistMaestroReadonlyBundleContent reads a Maestro readonly bundle and creates or updates the corresponding -// ManagementClusterContent in Cosmos. hcpClusterResourceID must be the HCP OpenShift cluster ARM resource ID (for -// node pool–scoped bundles, pass the node pool resource's parent cluster ID). +// ManagementClusterContent in Cosmos. parentResourceID is the resource id of the parent resource of the ManagementClusterContent. func readAndPersistMaestroReadonlyBundleContent( ctx context.Context, - cosmosClient database.DBClient, - hcpClusterResourceID *azcorearm.ResourceID, + parentResourceID *azcorearm.ResourceID, maestroBundleReference *api.MaestroBundleReference, maestroClient maestro.Client, + managementClusterContentsDBClient database.ManagementClusterContentCRUD, ) error { - desired, err := calculateManagementClusterContentFromMaestroBundle(ctx, hcpClusterResourceID, maestroBundleReference, maestroClient) + desired, err := calculateManagementClusterContentFromMaestroBundle(ctx, parentResourceID, maestroBundleReference, maestroClient) if err != nil { return utils.TrackError(fmt.Errorf("failed to calculate ManagementClusterContent from Maestro Bundle: %w", err)) } - managementClusterContentsDBClient := cosmosClient.ManagementClusterContents( - hcpClusterResourceID.SubscriptionID, - hcpClusterResourceID.ResourceGroupName, - hcpClusterResourceID.Name, - ) - existing, err := managementClusterContentsDBClient.Get(ctx, desired.CosmosMetadata.ResourceID.Name) if err != nil && !database.IsResponseError(err, http.StatusNotFound) { return utils.TrackError(fmt.Errorf("failed to get ManagementClusterContent: %w", err)) diff --git a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go index edb75cfe8bd..c8fc6e3ad44 100644 --- a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go +++ b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go @@ -119,9 +119,11 @@ func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) SyncOnc return utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) } + managementClusterContentsDBClient := c.cosmosClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).ManagementClusterContents(key.HCPClusterName) + var syncErrors []error for _, maestroBundleReference := range existingServiceProviderCluster.Status.MaestroReadonlyBundles { - err = readAndPersistMaestroReadonlyBundleContent(ctx, c.cosmosClient, existingCluster.ID, maestroBundleReference, maestroClient) + err = readAndPersistMaestroReadonlyBundleContent(ctx, existingCluster.ID, maestroBundleReference, maestroClient, managementClusterContentsDBClient) if err != nil { syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to read and persist HostedCluster: %w", err))) } diff --git a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go index e029306e6ab..1956338be87 100644 --- a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go +++ b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go @@ -73,6 +73,18 @@ func (e *errorInjectingMCCCRUD) Replace(ctx context.Context, obj *api.Management var _ database.ManagementClusterContentCRUD = &errorInjectingMCCCRUD{} +// hcpClusterCRUDWithInjectedMCC wraps HCPClusterCRUD to return a fixed ManagementClusterContentCRUD (for tests). +type hcpClusterCRUDWithInjectedMCC struct { + database.HCPClusterCRUD + mccCRUD database.ManagementClusterContentCRUD +} + +func (e *hcpClusterCRUDWithInjectedMCC) ManagementClusterContents(hcpClusterName string) database.ManagementClusterContentCRUD { + return e.mccCRUD +} + +var _ database.HCPClusterCRUD = &hcpClusterCRUDWithInjectedMCC{} + // errorInjectingDBClient wraps MockDBClient to return error-injecting CRUDs. type errorInjectingDBClient struct { *databasetesting.MockDBClient @@ -81,18 +93,17 @@ type errorInjectingDBClient struct { spcCRUD database.ServiceProviderClusterCRUD } -func (e *errorInjectingDBClient) ManagementClusterContents(subscriptionID, resourceGroupName, clusterName string) database.ManagementClusterContentCRUD { - if e.mccCRUD != nil { - return e.mccCRUD - } - return e.MockDBClient.ManagementClusterContents(subscriptionID, resourceGroupName, clusterName) -} - func (e *errorInjectingDBClient) HCPClusters(subscriptionID, resourceGroupName string) database.HCPClusterCRUD { + var base database.HCPClusterCRUD if e.clustersCRUD != nil { - return e.clustersCRUD + base = e.clustersCRUD + } else { + base = e.MockDBClient.HCPClusters(subscriptionID, resourceGroupName) + } + if e.mccCRUD != nil { + return &hcpClusterCRUDWithInjectedMCC{HCPClusterCRUD: base, mccCRUD: e.mccCRUD} } - return e.MockDBClient.HCPClusters(subscriptionID, resourceGroupName) + return base } func (e *errorInjectingDBClient) ServiceProviderClusters(subscriptionID, resourceGroupName, clusterName string) database.ServiceProviderClusterCRUD { @@ -248,9 +259,9 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.ManagementClusterContents("sub", "rg", "cluster") + mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") - err := readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, cluster.ID, ref, mockMaestro) + err := readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) require.NoError(t, err) // Content should have been created (name = bundle internal name) @@ -268,7 +279,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.ManagementClusterContents("sub", "rg", "cluster") + mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") // Pre-create existing content with different payload existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) existing := &api.ManagementClusterContent{ @@ -279,7 +290,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP _, err := mccCRUD.Create(ctx, existing, nil) require.NoError(t, err) - err = readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, cluster.ID, ref, mockMaestro) + err = readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) require.NoError(t, err) got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) @@ -304,7 +315,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.ManagementClusterContents("sub", "rg", "cluster") + mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) existingContent := &metav1.List{Items: []runtime.RawExtension{{Raw: []byte(`{}`)}}} existing := &api.ManagementClusterContent{ @@ -315,7 +326,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP _, err := mccCRUD.Create(ctx, existing, nil) require.NoError(t, err) - err = readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, cluster.ID, ref, mockMaestro) + err = readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) require.NoError(t, err) got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) @@ -333,7 +344,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil).Times(2) mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.ManagementClusterContents("sub", "rg", "cluster") + mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) // Pre-create content that matches exactly what the syncer would compute (same KubeContent and Degraded=False condition) // so that DeepEqual(existing, desired) is true and Replace is not called. @@ -348,7 +359,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP _, err = mccCRUD.Create(ctx, existing, nil) require.NoError(t, err) - err = readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, cluster.ID, ref, mockMaestro) + err = readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) require.NoError(t, err) // Document should still exist with same content (no Replace was needed) @@ -380,7 +391,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP }, } - err := readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, cluster.ID, ref, mockMaestro) + err := readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mockDB.mccCRUD) require.Error(t, err) assert.Contains(t, err.Error(), "failed to replace ManagementClusterContent") assert.True(t, database.IsResponseError(err, http.StatusPreconditionFailed), "expected 412 Precondition Failed") @@ -401,7 +412,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndP }, } - err := readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, cluster.ID, ref, mockMaestro) + err := readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mockDB.mccCRUD) require.Error(t, err) assert.Contains(t, err.Error(), "failed to get ManagementClusterContent") assert.Contains(t, err.Error(), "cosmos connection error") @@ -627,7 +638,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_SyncOnce err = syncer.SyncOnce(ctx, key) require.NoError(t, err) - mccCRUD := mockDBClient.ManagementClusterContents(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName) + mccCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).ManagementClusterContents(key.HCPClusterName) got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) require.NoError(t, err) require.NotNil(t, got) diff --git a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go index 9345d6cdce5..03e03deff34 100644 --- a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go +++ b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go @@ -30,11 +30,11 @@ import ( "github.com/Azure/ARO-HCP/internal/utils" ) -// readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer is a controller that reads the Maestro readonly bundles -// references stored in the ServiceProviderCluster resource, retrieves the Maestro readonly bundles using those +// readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer is a controller that reads the Maestro readonly bundles +// references stored in the ServiceProviderNodePool resource, retrieves the Maestro readonly bundles using those // references, extracts the content of the Maestro readonly bundles and persists them in Cosmos. // It is not responsible for creating the Maestro readonly bundles themselves. That is the responsibility of -// the createMaestroReadonlyBundlesSyncer controller. +// the createNodePoolScopedMaestroReadonlyBundlesSyncer controller. // As of now we support reading the content of the Maestro readonly bundle of the Hypershift's NodePools associated // to the Cluster. // This controller assumes that it has full ownership of the ManagementClusterContent resource. @@ -122,9 +122,11 @@ func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) SyncOn return utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) } + managementClusterContentsDBClient := c.cosmosClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName).ManagementClusterContents(key.HCPNodePoolName) + var syncErrors []error for _, maestroBundleReference := range existingServiceProviderNodePool.Status.MaestroReadonlyBundles { - err = readAndPersistMaestroReadonlyBundleContent(ctx, c.cosmosClient, existingNodePool.ID.Parent, maestroBundleReference, maestroClient) + err = readAndPersistMaestroReadonlyBundleContent(ctx, existingNodePool.ID, maestroBundleReference, maestroClient, managementClusterContentsDBClient) if err != nil { syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to read and persist NodePool: %w", err))) } diff --git a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go index 971332b14b2..ea740917c06 100644 --- a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go +++ b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go @@ -49,17 +49,9 @@ import ( // errorInjectingDBClientForNodePoolReadPersist wraps MockDBClient to return error-injecting CRUDs. type errorInjectingDBClientForNodePoolReadPersist struct { *databasetesting.MockDBClient - mccCRUD database.ManagementClusterContentCRUD spnpCRUD database.ServiceProviderNodePoolCRUD } -func (e *errorInjectingDBClientForNodePoolReadPersist) ManagementClusterContents(subscriptionID, resourceGroupName, clusterName string) database.ManagementClusterContentCRUD { - if e.mccCRUD != nil { - return e.mccCRUD - } - return e.MockDBClient.ManagementClusterContents(subscriptionID, resourceGroupName, clusterName) -} - func (e *errorInjectingDBClientForNodePoolReadPersist) ServiceProviderNodePools(subscriptionID, resourceGroupName, clusterName, nodePoolName string) database.ServiceProviderNodePoolCRUD { if e.spnpCRUD != nil { return e.spnpCRUD @@ -119,9 +111,8 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_calcula }, }, } - bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "test-nodepool") ref := &api.MaestroBundleReference{ - Name: bundleInternalName, + Name: api.MaestroBundleInternalNameReadonlyHypershiftNodePool, MaestroAPIMaestroBundleName: "bundle-name", } @@ -187,7 +178,7 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_calcula mockMaestro := maestro.NewMockClient(ctrl) tt.maestroGet(mockMaestro) - got, err := calculateManagementClusterContentFromMaestroBundle(context.Background(), nodepool.ID.Parent, ref, mockMaestro) + got, err := calculateManagementClusterContentFromMaestroBundle(context.Background(), nodepool.ID, ref, mockMaestro) if tt.wantErr { require.Error(t, err) assert.Contains(t, err.Error(), tt.errSub) @@ -213,7 +204,7 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_readAnd }, }, } - bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "test-nodepool") + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool ref := &api.MaestroBundleReference{ Name: bundleInternalName, MaestroAPIMaestroBundleName: "bundle-name", @@ -224,12 +215,13 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_readAnd require.NoError(t, err) validNPJSON := string(npJSONBytes) - // ManagementClusterContents is keyed by the cluster name (nodePool.ID.Parent.Name) + // Node-pool–scoped ManagementClusterContents are nested under the node pool ARM resource. const mccContainerSub = "sub" const mccContainerRG = "rg" const mccContainerCluster = "cluster" - existingMCCResourceIDStr := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/%s/managementClusterContents/%s", - mccContainerSub, mccContainerRG, mccContainerCluster, bundleInternalName) + const mccNodePoolName = "test-nodepool" + existingMCCResourceIDStr := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/%s/nodePools/%s/managementClusterContents/%s", + mccContainerSub, mccContainerRG, mccContainerCluster, mccNodePoolName, bundleInternalName) t.Run("creates new ManagementClusterContent when not found", func(t *testing.T) { ctrl := gomock.NewController(t) @@ -238,9 +230,9 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_readAnd mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.ManagementClusterContents(mccContainerSub, mccContainerRG, mccContainerCluster) + mccCRUD := mockDB.HCPClusters(mccContainerSub, mccContainerRG).NodePools(mccContainerCluster).ManagementClusterContents(mccNodePoolName) - err := readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, nodepool.ID.Parent, ref, mockMaestro) + err := readAndPersistMaestroReadonlyBundleContent(ctx, nodepool.ID, ref, mockMaestro, mccCRUD) require.NoError(t, err) got, err := mccCRUD.Get(ctx, string(bundleInternalName)) @@ -257,7 +249,7 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_readAnd mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.ManagementClusterContents(mccContainerSub, mccContainerRG, mccContainerCluster) + mccCRUD := mockDB.HCPClusters(mccContainerSub, mccContainerRG).NodePools(mccContainerCluster).ManagementClusterContents(mccNodePoolName) existingRID := api.Must(azcorearm.ParseResourceID(existingMCCResourceIDStr)) existing := &api.ManagementClusterContent{ CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, @@ -267,7 +259,7 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_readAnd _, err := mccCRUD.Create(ctx, existing, nil) require.NoError(t, err) - err = readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, nodepool.ID.Parent, ref, mockMaestro) + err = readAndPersistMaestroReadonlyBundleContent(ctx, nodepool.ID, ref, mockMaestro, mccCRUD) require.NoError(t, err) got, err := mccCRUD.Get(ctx, string(bundleInternalName)) @@ -292,7 +284,7 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_readAnd mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.ManagementClusterContents(mccContainerSub, mccContainerRG, mccContainerCluster) + mccCRUD := mockDB.HCPClusters(mccContainerSub, mccContainerRG).NodePools(mccContainerCluster).ManagementClusterContents(mccNodePoolName) existingRID := api.Must(azcorearm.ParseResourceID(existingMCCResourceIDStr)) existingContent := &metav1.List{Items: []runtime.RawExtension{{Raw: []byte(`{}`)}}} existing := &api.ManagementClusterContent{ @@ -303,7 +295,7 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_readAnd _, err := mccCRUD.Create(ctx, existing, nil) require.NoError(t, err) - err = readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, nodepool.ID.Parent, ref, mockMaestro) + err = readAndPersistMaestroReadonlyBundleContent(ctx, nodepool.ID, ref, mockMaestro, mccCRUD) require.NoError(t, err) got, err := mccCRUD.Get(ctx, string(bundleInternalName)) @@ -319,9 +311,9 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_readAnd mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil).Times(2) mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.ManagementClusterContents(mccContainerSub, mccContainerRG, mccContainerCluster) + mccCRUD := mockDB.HCPClusters(mccContainerSub, mccContainerRG).NodePools(mccContainerCluster).ManagementClusterContents(mccNodePoolName) existingRID := api.Must(azcorearm.ParseResourceID(existingMCCResourceIDStr)) - desired, err := calculateManagementClusterContentFromMaestroBundle(ctx, nodepool.ID.Parent, ref, mockMaestro) + desired, err := calculateManagementClusterContentFromMaestroBundle(ctx, nodepool.ID, ref, mockMaestro) require.NoError(t, err) existing := &api.ManagementClusterContent{ CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, @@ -331,7 +323,7 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_readAnd _, err = mccCRUD.Create(ctx, existing, nil) require.NoError(t, err) - err = readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, nodepool.ID.Parent, ref, mockMaestro) + err = readAndPersistMaestroReadonlyBundleContent(ctx, nodepool.ID, ref, mockMaestro, mccCRUD) require.NoError(t, err) got, err := mccCRUD.Get(ctx, string(bundleInternalName)) @@ -353,15 +345,15 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_readAnd Status: api.ManagementClusterContentStatus{KubeContent: &metav1.List{Items: []runtime.RawExtension{{Raw: []byte(validNPJSON)}}}}, } - mockDB := &errorInjectingDBClientForNodePoolReadPersist{ - MockDBClient: databasetesting.NewMockDBClient(), - mccCRUD: &errorInjectingMCCCRUDForNodePool{ - getResult: existingDoc, - replaceErr: databasetesting.NewPreconditionFailedError(), - }, + mockDB := databasetesting.NewMockDBClient() + baseMccCRUD := mockDB.HCPClusters(mccContainerSub, mccContainerRG).NodePools(mccContainerCluster).ManagementClusterContents(mccNodePoolName) + mccCRUD := &errorInjectingMCCCRUDForNodePool{ + ManagementClusterContentCRUD: baseMccCRUD, + getResult: existingDoc, + replaceErr: databasetesting.NewPreconditionFailedError(), } - err := readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, nodepool.ID.Parent, ref, mockMaestro) + err := readAndPersistMaestroReadonlyBundleContent(ctx, nodepool.ID, ref, mockMaestro, mccCRUD) require.Error(t, err) assert.Contains(t, err.Error(), "failed to replace ManagementClusterContent") assert.True(t, database.IsResponseError(err, http.StatusPreconditionFailed), "expected 412 Precondition Failed") @@ -373,14 +365,14 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_readAnd b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validNPJSON) mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - mockDB := &errorInjectingDBClientForNodePoolReadPersist{ - MockDBClient: databasetesting.NewMockDBClient(), - mccCRUD: &errorInjectingMCCCRUDForNodePool{ - getErr: fmt.Errorf("cosmos connection error"), - }, + mockDB := databasetesting.NewMockDBClient() + baseMccCRUD := mockDB.HCPClusters(mccContainerSub, mccContainerRG).NodePools(mccContainerCluster).ManagementClusterContents(mccNodePoolName) + mccCRUD := &errorInjectingMCCCRUDForNodePool{ + ManagementClusterContentCRUD: baseMccCRUD, + getErr: fmt.Errorf("cosmos connection error"), } - err := readAndPersistMaestroReadonlyBundleContent(ctx, mockDB, nodepool.ID.Parent, ref, mockMaestro) + err := readAndPersistMaestroReadonlyBundleContent(ctx, nodepool.ID, ref, mockMaestro, mccCRUD) require.Error(t, err) assert.Contains(t, err.Error(), "failed to get ManagementClusterContent") assert.Contains(t, err.Error(), "cosmos connection error") @@ -533,7 +525,7 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnc _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) require.NoError(t, err) - bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "test-nodepool") + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) spnp := &api.ServiceProviderNodePool{ CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, @@ -597,14 +589,13 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnc _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) require.NoError(t, err) - bundleInternalName := api.MaestroBundleInternalName(api.MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix + "test-nodepool") spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) spnp := &api.ServiceProviderNodePool{ CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, ResourceID: *spnpResourceID, Status: api.ServiceProviderNodePoolStatus{ MaestroReadonlyBundles: api.MaestroBundleReferenceList{ - {Name: bundleInternalName, MaestroAPIMaestroBundleName: "bundle-name"}, + {Name: api.MaestroBundleInternalNameReadonlyHypershiftNodePool, MaestroAPIMaestroBundleName: "bundle-name"}, }, }, } @@ -632,9 +623,8 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnc err = syncer.SyncOnce(ctx, key) require.NoError(t, err) - // ManagementClusterContents is keyed by cluster name (nodePool.ID.Parent.Name) - mccCRUD := mockDBClient.ManagementClusterContents(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName) - got, err := mccCRUD.Get(ctx, string(bundleInternalName)) + mccCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName).ManagementClusterContents(key.HCPNodePoolName) + got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftNodePool)) require.NoError(t, err) require.NotNil(t, got) require.NotNil(t, got.Status.KubeContent) diff --git a/backend/pkg/controllers/upgradecontrollers/control_plane_active_version_controller.go b/backend/pkg/controllers/upgradecontrollers/control_plane_active_version_controller.go index 72449152077..8417d4e8053 100644 --- a/backend/pkg/controllers/upgradecontrollers/control_plane_active_version_controller.go +++ b/backend/pkg/controllers/upgradecontrollers/control_plane_active_version_controller.go @@ -85,7 +85,7 @@ func (c *controlPlaneActiveVersionSyncer) SyncOnce(ctx context.Context, key cont return utils.TrackError(fmt.Errorf("failed to get Cluster: %w", err)) } - managementClusterContentClient := c.cosmosClient.ManagementClusterContents(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName) + managementClusterContentClient := c.cosmosClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).ManagementClusterContents(key.HCPClusterName) managementClusterContent, err := managementClusterContentClient.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) if database.IsResponseError(err, http.StatusNotFound) { return nil diff --git a/backend/pkg/controllers/upgradecontrollers/control_plane_active_version_controller_test.go b/backend/pkg/controllers/upgradecontrollers/control_plane_active_version_controller_test.go index 3064fcb7a64..3f247b4d6d3 100644 --- a/backend/pkg/controllers/upgradecontrollers/control_plane_active_version_controller_test.go +++ b/backend/pkg/controllers/upgradecontrollers/control_plane_active_version_controller_test.go @@ -309,6 +309,6 @@ func createManagementClusterContentWithKubeContentItems(t *testing.T, ctx contex }, }, } - _, err := mockDB.ManagementClusterContents(testSubscriptionID, testResourceGroupName, testClusterName).Create(ctx, managementClusterContent, nil) + _, err := mockDB.HCPClusters(testSubscriptionID, testResourceGroupName).ManagementClusterContents(testClusterName).Create(ctx, managementClusterContent, nil) require.NoError(t, err) } diff --git a/backend/pkg/informers/informers.go b/backend/pkg/informers/informers.go index 36a0046b16d..d98cd646bf8 100644 --- a/backend/pkg/informers/informers.go +++ b/backend/pkg/informers/informers.go @@ -340,13 +340,13 @@ func NewServiceProviderClusterInformerWithRelistDuration(lister database.GlobalL } // NewManagementClusterContentInformer creates an unstarted SharedIndexInformer for management cluster contents -// with a cluster index using the default relist duration. +// with cluster and node pool indexes using the default relist duration. func NewManagementClusterContentInformer(lister database.GlobalLister[api.ManagementClusterContent]) cache.SharedIndexInformer { return NewManagementClusterContentInformerWithRelistDuration(lister, ManagementClusterContentRelistDuration) } // NewManagementClusterContentInformerWithRelistDuration creates an unstarted SharedIndexInformer for management cluster contents -// with a cluster index and a configurable relist duration. +// with cluster and node pool indexes and a configurable relist duration. func NewManagementClusterContentInformerWithRelistDuration(lister database.GlobalLister[api.ManagementClusterContent], relistDuration time.Duration) cache.SharedIndexInformer { lw := &cache.ListWatch{ ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { @@ -381,7 +381,8 @@ func NewManagementClusterContentInformerWithRelistDuration(lister database.Globa cache.SharedIndexInformerOptions{ ResyncPeriod: 1 * time.Hour, // this is only a default. Shorter resyncs can be added when registering handlers. Indexers: cache.Indexers{ - listers.ByCluster: clusterResourceIDIndexFunc, + listers.ByCluster: clusterResourceIDIndexFunc, + listers.ByNodePool: nodePoolResourceIDIndexFunc, }, }, ) @@ -668,8 +669,8 @@ func activeOperationClusterIndexFunc(obj interface{}) ([]string, error) { return clusterResourceIDFromResourceID(op.ExternalID) } -// nodePoolResourceIDIndexFunc indexes service provider node pools by their parent node pool -// resource ID, derived from the embedded CosmosMetadata. +// nodePoolResourceIDIndexFunc indexes objects by the node pool resource ID of their nearest +// nodePool ancestor in the ARM path (Cosmos metadata resource ID). func nodePoolResourceIDIndexFunc(obj interface{}) ([]string, error) { switch castObj := obj.(type) { case arm.CosmosMetadataAccessor: diff --git a/backend/pkg/informers/types.go b/backend/pkg/informers/types.go index eca9492a230..449ade8c4ea 100644 --- a/backend/pkg/informers/types.go +++ b/backend/pkg/informers/types.go @@ -36,6 +36,8 @@ type BackendInformers interface { ServiceProviderClusters() (cache.SharedIndexInformer, listers.ServiceProviderClusterLister) ServiceProviderNodePools() (cache.SharedIndexInformer, listers.ServiceProviderNodePoolLister) Controllers() (cache.SharedIndexInformer, listers.ControllerLister) + // ManagementClusterContents is the single shared informer for all managementClusterContents documents belonging + // to different resource types. ManagementClusterContents() (cache.SharedIndexInformer, listers.ManagementClusterContentLister) BillingDocs() (cache.SharedIndexInformer, listers.BillingLister) diff --git a/backend/pkg/listers/management_cluster_content_lister.go b/backend/pkg/listers/management_cluster_content_lister.go index e2b8e115827..02a98227f0b 100644 --- a/backend/pkg/listers/management_cluster_content_lister.go +++ b/backend/pkg/listers/management_cluster_content_lister.go @@ -22,10 +22,11 @@ import ( "github.com/Azure/ARO-HCP/internal/api" ) -// ManagementClusterContentLister lists ManagementClusterContent from an informer's indexer. +// ManagementClusterContentLister lists ManagementClusterContent from the shared informer indexer. type ManagementClusterContentLister interface { List(ctx context.Context) ([]*api.ManagementClusterContent, error) ListForCluster(ctx context.Context, subscriptionID, resourceGroupName, clusterName string) ([]*api.ManagementClusterContent, error) + ListForNodePool(ctx context.Context, subscriptionName, resourceGroupName, clusterName, nodePoolName string) ([]*api.ManagementClusterContent, error) } // managementClusterContentLister implements ManagementClusterContentLister backed by a SharedIndexInformer. @@ -48,3 +49,8 @@ func (l *managementClusterContentLister) ListForCluster(ctx context.Context, sub key := api.ToClusterResourceIDString(subscriptionID, resourceGroupName, clusterName) return listFromIndex[api.ManagementClusterContent](l.indexer, ByCluster, key) } + +func (l *managementClusterContentLister) ListForNodePool(ctx context.Context, subscriptionName, resourceGroupName, clusterName, nodePoolName string) ([]*api.ManagementClusterContent, error) { + key := api.ToNodePoolResourceIDString(subscriptionName, resourceGroupName, clusterName, nodePoolName) + return listFromIndex[api.ManagementClusterContent](l.indexer, ByNodePool, key) +} diff --git a/backend/pkg/listertesting/db_listers.go b/backend/pkg/listertesting/db_listers.go index 41048b4558b..c91ecb16bd3 100644 --- a/backend/pkg/listertesting/db_listers.go +++ b/backend/pkg/listertesting/db_listers.go @@ -248,6 +248,46 @@ func (l *DBControllerLister) listWithPrefix(ctx context.Context, prefix string) return result, nil } +// DBManagementClusterContentLister implements listers.ManagementClusterContentLister backed by a database.DBClient. +type DBManagementClusterContentLister struct { + DBClient database.DBClient +} + +var _ listers.ManagementClusterContentLister = &DBManagementClusterContentLister{} + +func (l *DBManagementClusterContentLister) List(ctx context.Context) ([]*api.ManagementClusterContent, error) { + iter, err := l.DBClient.GlobalListers().ManagementClusterContents().List(ctx, nil) + if err != nil { + return nil, err + } + return collectFromIterator(ctx, iter) +} + +func (l *DBManagementClusterContentLister) ListForCluster(ctx context.Context, subscriptionID, resourceGroupName, clusterName string) ([]*api.ManagementClusterContent, error) { + prefix := api.ToClusterResourceIDString(subscriptionID, resourceGroupName, clusterName) + return l.listMCCWithPrefix(ctx, prefix) +} + +func (l *DBManagementClusterContentLister) ListForNodePool(ctx context.Context, subscriptionName, resourceGroupName, clusterName, nodePoolName string) ([]*api.ManagementClusterContent, error) { + prefix := api.ToNodePoolResourceIDString(subscriptionName, resourceGroupName, clusterName, nodePoolName) + return l.listMCCWithPrefix(ctx, prefix) +} + +func (l *DBManagementClusterContentLister) listMCCWithPrefix(ctx context.Context, prefix string) ([]*api.ManagementClusterContent, error) { + all, err := l.List(ctx) + if err != nil { + return nil, err + } + var result []*api.ManagementClusterContent + for _, mcc := range all { + rid := mcc.GetResourceID() + if rid != nil && strings.HasPrefix(strings.ToLower(rid.String()), strings.ToLower(prefix)) { + result = append(result, mcc) + } + } + return result, nil +} + // DBSubscriptionLister implements listers.SubscriptionLister backed by a database.DBClient. type DBSubscriptionLister struct { DBClient database.DBClient diff --git a/backend/pkg/listertesting/db_listers_test.go b/backend/pkg/listertesting/db_listers_test.go index a9f47f97aec..451a8545c94 100644 --- a/backend/pkg/listertesting/db_listers_test.go +++ b/backend/pkg/listertesting/db_listers_test.go @@ -355,6 +355,61 @@ func TestDBControllerLister(t *testing.T) { }) } +func TestDBManagementClusterContentLister(t *testing.T) { + ctx := context.Background() + + cluster1 := newTestCluster(testSubscriptionID, testResourceGroupName, testClusterName) + cluster2 := newTestCluster(testSubscriptionID, testResourceGroupName, testClusterName2) + np := newTestNodePool(testSubscriptionID, testResourceGroupName, testClusterName, testNodePoolName) + + mccCluster1 := newTestClusterScopedManagementClusterContent(testSubscriptionID, testResourceGroupName, testClusterName, "mcc-under-cluster") + mccCluster2 := newTestClusterScopedManagementClusterContent(testSubscriptionID, testResourceGroupName, testClusterName2, "mcc-under-cluster2") + mccNP := newTestNodePoolScopedManagementClusterContent(testSubscriptionID, testResourceGroupName, testClusterName, testNodePoolName, "mcc-under-np") + + mockDB, err := databasetesting.NewMockDBClientWithResources(ctx, []any{ + cluster1, cluster2, np, + mccCluster1, mccCluster2, mccNP, + }) + require.NoError(t, err) + + lister := &DBManagementClusterContentLister{DBClient: mockDB} + + t.Run("List returns all management cluster contents", func(t *testing.T) { + result, err := lister.List(ctx) + require.NoError(t, err) + assert.Len(t, result, 3) + }) + + t.Run("ListForCluster returns cluster-scoped and node-pool-scoped MCC for that cluster", func(t *testing.T) { + result, err := lister.ListForCluster(ctx, testSubscriptionID, testResourceGroupName, testClusterName) + require.NoError(t, err) + assert.Len(t, result, 2) + names := []string{result[0].GetResourceID().Name, result[1].GetResourceID().Name} + assert.Contains(t, names, "mcc-under-cluster") + assert.Contains(t, names, "mcc-under-np") + }) + + t.Run("ListForCluster returns only MCC for other cluster", func(t *testing.T) { + result, err := lister.ListForCluster(ctx, testSubscriptionID, testResourceGroupName, testClusterName2) + require.NoError(t, err) + require.Len(t, result, 1) + assert.Equal(t, "mcc-under-cluster2", result[0].GetResourceID().Name) + }) + + t.Run("ListForNodePool returns only node-pool-scoped MCC", func(t *testing.T) { + result, err := lister.ListForNodePool(ctx, testSubscriptionID, testResourceGroupName, testClusterName, testNodePoolName) + require.NoError(t, err) + require.Len(t, result, 1) + assert.Equal(t, "mcc-under-np", result[0].GetResourceID().Name) + }) + + t.Run("ListForNodePool returns empty for non-existent node pool", func(t *testing.T) { + result, err := lister.ListForNodePool(ctx, testSubscriptionID, testResourceGroupName, testClusterName, "non-existent") + require.NoError(t, err) + assert.Empty(t, result) + }) +} + func TestDBClusterListerWithEmptyDB(t *testing.T) { ctx := context.Background() mockDB := databasetesting.NewMockDBClient() diff --git a/backend/pkg/listertesting/slice_listers_test.go b/backend/pkg/listertesting/slice_listers_test.go index e7dcd9c4cb9..4052a273a9b 100644 --- a/backend/pkg/listertesting/slice_listers_test.go +++ b/backend/pkg/listertesting/slice_listers_test.go @@ -459,3 +459,30 @@ func newTestSubscription(subscriptionID string) *arm.Subscription { ResourceID: resourceID, } } + +func newTestClusterScopedManagementClusterContent(subscriptionID, resourceGroupName, clusterName, mccName string) *api.ManagementClusterContent { + resourceID := api.Must(azcorearm.ParseResourceID( + "/subscriptions/" + subscriptionID + + "/resourceGroups/" + resourceGroupName + + "/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/" + clusterName + + "/managementClusterContents/" + mccName, + )) + return &api.ManagementClusterContent{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: resourceID}, + ResourceID: *resourceID, + } +} + +func newTestNodePoolScopedManagementClusterContent(subscriptionID, resourceGroupName, clusterName, nodePoolName, mccName string) *api.ManagementClusterContent { + resourceID := api.Must(azcorearm.ParseResourceID( + "/subscriptions/" + subscriptionID + + "/resourceGroups/" + resourceGroupName + + "/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/" + clusterName + + "/nodePools/" + nodePoolName + + "/managementClusterContents/" + mccName, + )) + return &api.ManagementClusterContent{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: resourceID}, + ResourceID: *resourceID, + } +} diff --git a/backend/pkg/maestro/maestro_bundle.go b/backend/pkg/maestro/maestro_bundle.go index e30fa8c42fe..31855ee91f8 100644 --- a/backend/pkg/maestro/maestro_bundle.go +++ b/backend/pkg/maestro/maestro_bundle.go @@ -26,7 +26,7 @@ import ( "github.com/Azure/ARO-HCP/internal/utils" ) -// getOrCreateMaestroBundle gets (or creates if it does not exist) a Maestro Bundle for a given Maestro Bundle namespaced name. +// GetOrCreateMaestroBundle gets (or creates if it does not exist) a Maestro Bundle for a given Maestro Bundle namespaced name. func GetOrCreateMaestroBundle(ctx context.Context, maestroClient Client, maestroBundle *workv1.ManifestWork) (*workv1.ManifestWork, error) { logger := utils.LoggerFromContext(ctx) existingMaestroBundle, err := maestroClient.Get(ctx, maestroBundle.Name, metav1.GetOptions{}) diff --git a/internal/api/registry.go b/internal/api/registry.go index dca72a11760..15c501f94f1 100644 --- a/internal/api/registry.go +++ b/internal/api/registry.go @@ -81,20 +81,23 @@ const ( ) var ( - OperationStatusResourceType = azcorearm.NewResourceType(ProviderNamespace, OperationStatusResourceTypeName) - ClusterResourceType = azcorearm.NewResourceType(ProviderNamespace, ClusterResourceTypeName) - ServiceProviderClusterResourceType = azcorearm.NewResourceType(ProviderNamespace, ClusterResourceTypeName+"/"+ServiceProviderClusterResourceTypeName) - NodePoolResourceType = azcorearm.NewResourceType(ProviderNamespace, ClusterResourceTypeName+"/"+NodePoolResourceTypeName) - ServiceProviderNodePoolResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, NodePoolResourceTypeName, ServiceProviderNodePoolResourceTypeName)) - ExternalAuthResourceType = azcorearm.NewResourceType(ProviderNamespace, ClusterResourceTypeName+"/"+ExternalAuthResourceTypeName) - PreflightResourceType = azcorearm.NewResourceType(ProviderNamespace, "deployments/preflight") - VersionResourceType = azcorearm.NewResourceType(ProviderNamespace, "locations/"+VersionResourceTypeName) - ClusterControllerResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, ControllerResourceTypeName)) - NodePoolControllerResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, NodePoolResourceTypeName, ControllerResourceTypeName)) - ExternalAuthControllerResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, ExternalAuthResourceTypeName, ControllerResourceTypeName)) - RequestAdminCredentialActionType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, RequestAdminCredentialActionTypeName)) - RevokeAdminCredentialsActionType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, RevokeAdminCredentialsActionTypeName)) - ManagementClusterContentResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, ManagementClusterContentResourceTypeName)) + OperationStatusResourceType = azcorearm.NewResourceType(ProviderNamespace, OperationStatusResourceTypeName) + ClusterResourceType = azcorearm.NewResourceType(ProviderNamespace, ClusterResourceTypeName) + ServiceProviderClusterResourceType = azcorearm.NewResourceType(ProviderNamespace, ClusterResourceTypeName+"/"+ServiceProviderClusterResourceTypeName) + NodePoolResourceType = azcorearm.NewResourceType(ProviderNamespace, ClusterResourceTypeName+"/"+NodePoolResourceTypeName) + ServiceProviderNodePoolResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, NodePoolResourceTypeName, ServiceProviderNodePoolResourceTypeName)) + ExternalAuthResourceType = azcorearm.NewResourceType(ProviderNamespace, ClusterResourceTypeName+"/"+ExternalAuthResourceTypeName) + PreflightResourceType = azcorearm.NewResourceType(ProviderNamespace, "deployments/preflight") + VersionResourceType = azcorearm.NewResourceType(ProviderNamespace, "locations/"+VersionResourceTypeName) + ClusterControllerResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, ControllerResourceTypeName)) + NodePoolControllerResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, NodePoolResourceTypeName, ControllerResourceTypeName)) + ExternalAuthControllerResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, ExternalAuthResourceTypeName, ControllerResourceTypeName)) + RequestAdminCredentialActionType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, RequestAdminCredentialActionTypeName)) + RevokeAdminCredentialsActionType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, RevokeAdminCredentialsActionTypeName)) + // ClusterScopedManagementClusterContentResourceType is managementClusterContents nested directly under a Cluster + ClusterScopedManagementClusterContentResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, ManagementClusterContentResourceTypeName)) + // NodePoolScopedManagementClusterContentResourceType is managementClusterContents nested under a Node Pool + NodePoolScopedManagementClusterContentResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, NodePoolResourceTypeName, ManagementClusterContentResourceTypeName)) ) type VersionedResource interface { diff --git a/internal/api/types_management_cluster_content.go b/internal/api/types_management_cluster_content.go index b83784c3582..b1d028ac69c 100644 --- a/internal/api/types_management_cluster_content.go +++ b/internal/api/types_management_cluster_content.go @@ -23,7 +23,7 @@ import ( // ManagementClusterContent represents K8s resources in the Management Cluster // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type ManagementClusterContent struct { - // CosmosMetadata ResourceID is nested under the cluster so that association and cleanup work as expected + // CosmosMetadata is nested under the corresponding resource type so that association and cleanup work as expected CosmosMetadata `json:"cosmosMetadata"` // resourceID exists to match cosmosMetadata.resourceID until we're able to transition all types to use cosmosMetadata, diff --git a/internal/api/types_serviceprovider_nodepool.go b/internal/api/types_serviceprovider_nodepool.go index 1db008e65f9..64cbc941e74 100644 --- a/internal/api/types_serviceprovider_nodepool.go +++ b/internal/api/types_serviceprovider_nodepool.go @@ -118,11 +118,9 @@ type ServiceProviderNodePoolStatusVersion struct { } const ( - // MaestroBundleInternalNameReadonlyHypershiftNodePool is the prefix of the internal name of the Maestro Bundle - // that represents a NodePool's Hypershift's NodePool K8s resource. The full name of the maestro internal bundle - // is constructed by concatenating the prefix with the lowercased name of the NodePool. - // Example: "readonlyHypershiftNodePool-my-nodepool" - MaestroBundleInternalNameReadonlyHypershiftNodePoolPrefix = "readonlyHypershiftNodePool-" + // MaestroBundleInternalNameReadonlyHypershiftNodePool is the internal name of the Maestro Bundle that represents + // the NodePool's Hypershift's NodePool K8s resource. + MaestroBundleInternalNameReadonlyHypershiftNodePool MaestroBundleInternalName = "readonlyHypershiftNodePool" ) // HCPNodePoolActiveVersion represents a single version active in the nodepool. diff --git a/internal/database/crud_hcpcluster.go b/internal/database/crud_hcpcluster.go index 1047512b81f..03430cec658 100644 --- a/internal/database/crud_hcpcluster.go +++ b/internal/database/crud_hcpcluster.go @@ -108,6 +108,7 @@ func (d *operationCRUD) ListActiveOperations(options *DBClientListActiveOperatio type HCPClusterCRUD interface { ResourceCRUD[api.HCPOpenShiftCluster] ControllerContainer + ManagementClusterContentContainer ExternalAuth(hcpClusterID string) ExternalAuthsCRUD NodePools(hcpClusterID string) NodePoolsCRUD @@ -133,6 +134,7 @@ func NewHCPClusterCRUD(containerClient *azcosmos.ContainerClient, subscriptionID type NodePoolsCRUD interface { ResourceCRUD[api.HCPOpenShiftClusterNodePool] ControllerContainer + ManagementClusterContentContainer } type ExternalAuthsCRUD interface { @@ -193,6 +195,22 @@ func (h *hcpClusterCRUD) Controllers(hcpClusterName string) ResourceCRUD[api.Con return NewControllerCRUD(h.containerClient, parentResourceID, api.ClusterControllerResourceType) } +func (h *hcpClusterCRUD) ManagementClusterContents(hcpClusterName string) ManagementClusterContentCRUD { + parentResourceID := api.Must(azcorearm.ParseResourceID( + path.Join( + h.parentResourceID.String(), + "providers", + h.resourceType.Namespace, + h.resourceType.Type, + hcpClusterName))) + + return NewCosmosResourceCRUD[api.ManagementClusterContent, GenericDocument[api.ManagementClusterContent]]( + h.containerClient, + parentResourceID, + api.ClusterScopedManagementClusterContentResourceType, + ) +} + type externalAuthCRUD struct { *nestedCosmosResourceCRUD[api.HCPOpenShiftClusterExternalAuth, ExternalAuth] } @@ -223,6 +241,21 @@ func (h *nodePoolsCRUD) Controllers(nodePoolName string) ResourceCRUD[api.Contro return NewControllerCRUD(h.containerClient, parentResourceID, api.NodePoolControllerResourceType) } +func (h *nodePoolsCRUD) ManagementClusterContents(nodePoolName string) ManagementClusterContentCRUD { + parentResourceID := api.Must(azcorearm.ParseResourceID( + path.Join( + h.parentResourceID.String(), + h.resourceType.Types[len(h.resourceType.Types)-1], + nodePoolName, + ))) + + return NewCosmosResourceCRUD[api.ManagementClusterContent, GenericDocument[api.ManagementClusterContent]]( + h.containerClient, + parentResourceID, + api.NodePoolScopedManagementClusterContentResourceType, + ) +} + func NewControllerCRUD( containerClient *azcosmos.ContainerClient, parentResourceID *azcorearm.ResourceID, resourceType azcorearm.ResourceType) ResourceCRUD[api.Controller] { diff --git a/internal/database/crud_management_cluster_content.go b/internal/database/crud_management_cluster_content.go index 229f6893d09..ac8c3a80790 100644 --- a/internal/database/crud_management_cluster_content.go +++ b/internal/database/crud_management_cluster_content.go @@ -21,3 +21,7 @@ import ( type ManagementClusterContentCRUD interface { ResourceCRUD[api.ManagementClusterContent] } + +type ManagementClusterContentContainer interface { + ManagementClusterContents(resourceName string) ManagementClusterContentCRUD +} diff --git a/internal/database/database.go b/internal/database/database.go index e7b6c10943e..3773e051d68 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -119,8 +119,6 @@ type DBClient interface { GlobalListers() GlobalListers ServiceProviderNodePools(subscriptionID, resourceGroupName, clusterName, nodePoolName string) ServiceProviderNodePoolCRUD - - ManagementClusterContents(subscriptionID, resourceGroupName, clusterName string) ManagementClusterContentCRUD } var _ DBClient = &cosmosDBClient{} @@ -201,12 +199,6 @@ func (d *cosmosDBClient) ServiceProviderNodePools(subscriptionID, resourceGroupN d.resources, nodePoolResourceID, api.ServiceProviderNodePoolResourceType) } -func (d *cosmosDBClient) ManagementClusterContents(subscriptionID, resourceGroupName, clusterName string) ManagementClusterContentCRUD { - clusterResourceID := NewClusterResourceID(subscriptionID, resourceGroupName, clusterName) - return NewCosmosResourceCRUD[api.ManagementClusterContent, GenericDocument[api.ManagementClusterContent]]( - d.resources, clusterResourceID, api.ManagementClusterContentResourceType) -} - func (d *cosmosDBClient) UntypedCRUD(parentResourceID azcorearm.ResourceID) (UntypedResourceCRUD, error) { return NewUntypedCRUD(d.resources, parentResourceID), nil } diff --git a/internal/database/global_lister.go b/internal/database/global_lister.go index 936a6691843..f29601bff9c 100644 --- a/internal/database/global_lister.go +++ b/internal/database/global_lister.go @@ -43,6 +43,9 @@ type GlobalListers interface { ServiceProviderClusters() GlobalLister[api.ServiceProviderCluster] ServiceProviderNodePools() GlobalLister[api.ServiceProviderNodePool] Controllers() GlobalLister[api.Controller] + // ManagementClusterContents lists ManagementClusterContent documents across + // partitions for every Cosmos resource type where managementClusterContents + // is nested as a direct child resource. Those types are registered on the lister implementation. ManagementClusterContents() GlobalLister[api.ManagementClusterContent] Operations() GlobalLister[api.Operation] ActiveOperations() GlobalLister[api.Operation] @@ -118,9 +121,12 @@ func (g *cosmosGlobalListers) Controllers() GlobalLister[api.Controller] { } func (g *cosmosGlobalListers) ManagementClusterContents() GlobalLister[api.ManagementClusterContent] { - return &cosmosGlobalLister[api.ManagementClusterContent, GenericDocument[api.ManagementClusterContent]]{ + return &cosmosManagementClusterContentGlobalLister{ containerClient: g.resources, - resourceType: api.ManagementClusterContentResourceType, + managementClusterContentResourceTypes: []azcorearm.ResourceType{ + api.ClusterScopedManagementClusterContentResourceType, + api.NodePoolScopedManagementClusterContentResourceType, + }, } } @@ -247,3 +253,37 @@ func (l *cosmosBillingGlobalLister) List(ctx context.Context, options *DBClientL } return newQueryBillingIterator(pager), nil } + +// cosmosManagementClusterContentGlobalLister lists managementClusterContents whether nested under a +// cluster or under a node pool. +type cosmosManagementClusterContentGlobalLister struct { + containerClient *azcosmos.ContainerClient + managementClusterContentResourceTypes []azcorearm.ResourceType +} + +func (l *cosmosManagementClusterContentGlobalLister) List(ctx context.Context, options *DBClientListResourceDocsOptions) (DBClientIterator[api.ManagementClusterContent], error) { + var resourceTypeConditions []string + for _, resourceType := range l.managementClusterContentResourceTypes { + resourceTypeConditions = append(resourceTypeConditions, fmt.Sprintf("STRINGEQUALS(c.resourceType, %q, true)", resourceType.String())) + } + whereClause := strings.Join(resourceTypeConditions, " OR ") + query := fmt.Sprintf("SELECT * FROM c WHERE %s", whereClause) + + queryOptions := azcosmos.QueryOptions{ + PageSizeHint: -1, + } + if options != nil { + if options.PageSizeHint != nil { + queryOptions.PageSizeHint = max(*options.PageSizeHint, -1) + } + queryOptions.ContinuationToken = options.ContinuationToken + } + + partitionKey := azcosmos.NewPartitionKey() + pager := l.containerClient.NewQueryItemsPager(query, partitionKey, &queryOptions) + + if options != nil && ptr.Deref(options.PageSizeHint, -1) > 0 { + return newQueryResourcesSinglePageIterator[api.ManagementClusterContent, GenericDocument[api.ManagementClusterContent]](pager), nil + } + return newQueryResourcesIterator[api.ManagementClusterContent, GenericDocument[api.ManagementClusterContent]](pager), nil +} diff --git a/internal/databasetesting/mock_crud.go b/internal/databasetesting/mock_crud.go index f7cec9d5354..d004412f029 100644 --- a/internal/databasetesting/mock_crud.go +++ b/internal/databasetesting/mock_crud.go @@ -511,6 +511,18 @@ func (m *mockHCPClusterCRUD) Controllers(hcpClusterName string) database.Resourc return newMockResourceCRUD[api.Controller, database.GenericDocument[api.Controller]](m.client, parentResourceID, api.ClusterControllerResourceType) } +func (m *mockHCPClusterCRUD) ManagementClusterContents(hcpClusterName string) database.ManagementClusterContentCRUD { + parentResourceID := api.Must(azcorearm.ParseResourceID( + path.Join( + m.parentResourceID.String(), + "providers", + m.resourceType.Namespace, + m.resourceType.Type, + hcpClusterName))) + + return newMockManagementClusterContentCRUD(m.client, parentResourceID, api.ClusterScopedManagementClusterContentResourceType) +} + var _ database.HCPClusterCRUD = &mockHCPClusterCRUD{} // mockNodePoolsCRUD implements database.NodePoolsCRUD. @@ -529,6 +541,17 @@ func (m *mockNodePoolsCRUD) Controllers(nodePoolName string) database.ResourceCR return newMockResourceCRUD[api.Controller, database.GenericDocument[api.Controller]](m.client, parentResourceID, api.NodePoolControllerResourceType) } +func (m *mockNodePoolsCRUD) ManagementClusterContents(nodePoolName string) database.ManagementClusterContentCRUD { + parentResourceID := api.Must(azcorearm.ParseResourceID( + path.Join( + m.parentResourceID.String(), + m.resourceType.Types[len(m.resourceType.Types)-1], + nodePoolName, + ))) + + return newMockManagementClusterContentCRUD(m.client, parentResourceID, api.NodePoolScopedManagementClusterContentResourceType) +} + var _ database.NodePoolsCRUD = &mockNodePoolsCRUD{} // mockExternalAuthCRUD implements database.ExternalAuthsCRUD. @@ -689,10 +712,10 @@ type mockManagementClusterContentCRUD struct { *mockResourceCRUD[api.ManagementClusterContent, database.GenericDocument[api.ManagementClusterContent]] } -func newMockManagementClusterContentCRUD(client *MockDBClient, parentResourceID *azcorearm.ResourceID) *mockManagementClusterContentCRUD { +func newMockManagementClusterContentCRUD(client *MockDBClient, parentResourceID *azcorearm.ResourceID, resourceType azcorearm.ResourceType) *mockManagementClusterContentCRUD { return &mockManagementClusterContentCRUD{ mockResourceCRUD: newMockResourceCRUD[api.ManagementClusterContent, database.GenericDocument[api.ManagementClusterContent]]( - client, parentResourceID, api.ManagementClusterContentResourceType), + client, parentResourceID, resourceType), } } diff --git a/internal/databasetesting/mock_dbclient.go b/internal/databasetesting/mock_dbclient.go index de4b902bf1e..4389837b1ea 100644 --- a/internal/databasetesting/mock_dbclient.go +++ b/internal/databasetesting/mock_dbclient.go @@ -148,12 +148,6 @@ func (m *MockDBClient) ServiceProviderNodePools(subscriptionID, resourceGroupNam return newMockServiceProviderNodePoolCRUD(m, nodePoolResourceID) } -// ManagementClusterContents returns a CRUD interface for management cluster content resources. -func (m *MockDBClient) ManagementClusterContents(subscriptionID, resourceGroupName, clusterName string) database.ManagementClusterContentCRUD { - clusterResourceID := database.NewClusterResourceID(subscriptionID, resourceGroupName, clusterName) - return newMockManagementClusterContentCRUD(m, clusterResourceID) -} - // LoadFromDirectory loads cosmos-record context data from a directory. // It reads all JSON files that match the pattern for "load" directories. func (m *MockDBClient) LoadFromDirectory(dirPath string) error { diff --git a/internal/databasetesting/mock_global_lister.go b/internal/databasetesting/mock_global_lister.go index 30a7d83df68..1ead4b74450 100644 --- a/internal/databasetesting/mock_global_lister.go +++ b/internal/databasetesting/mock_global_lister.go @@ -84,9 +84,12 @@ func (g *mockGlobalListers) Controllers() database.GlobalLister[api.Controller] } func (g *mockGlobalListers) ManagementClusterContents() database.GlobalLister[api.ManagementClusterContent] { - return &mockTypedGlobalLister[api.ManagementClusterContent, database.GenericDocument[api.ManagementClusterContent]]{ - client: g.client, - resourceType: api.ManagementClusterContentResourceType, + return &mockManagementClusterContentGlobalLister{ + client: g.client, + resourceTypes: []azcorearm.ResourceType{ + api.ClusterScopedManagementClusterContentResourceType, + api.NodePoolScopedManagementClusterContentResourceType, + }, } } @@ -283,3 +286,49 @@ func (l *mockBillingGlobalLister) List(ctx context.Context, options *database.DB return newMockIterator(ids, items), nil } + +// mockManagementClusterContentGlobalLister lists management cluster content for cluster-scoped and node-pool-scoped documents. +type mockManagementClusterContentGlobalLister struct { + client *MockDBClient + resourceTypes []azcorearm.ResourceType +} + +func (l *mockManagementClusterContentGlobalLister) List(ctx context.Context, options *database.DBClientListResourceDocsOptions) (database.DBClientIterator[api.ManagementClusterContent], error) { + allDocs := l.client.GetAllDocuments() + + var ids []string + var items []*api.ManagementClusterContent + + for _, data := range allDocs { + var typedDoc database.TypedDocument + if err := json.Unmarshal(data, &typedDoc); err != nil { + continue + } + + resourceTypeMatches := false + for _, resourceType := range l.resourceTypes { + if strings.EqualFold(typedDoc.ResourceType, resourceType.String()) { + resourceTypeMatches = true + break + } + } + if !resourceTypeMatches { + continue + } + + var cosmosObj database.GenericDocument[api.ManagementClusterContent] + if err := json.Unmarshal(data, &cosmosObj); err != nil { + continue + } + + internalObj, err := database.CosmosGenericToInternal(&cosmosObj) + if err != nil { + continue + } + + ids = append(ids, typedDoc.ID) + items = append(items, internalObj) + } + + return newMockIterator(ids, items), nil +} diff --git a/internal/databasetesting/mock_init.go b/internal/databasetesting/mock_init.go index 88a5a277e55..670fe0ca485 100644 --- a/internal/databasetesting/mock_init.go +++ b/internal/databasetesting/mock_init.go @@ -32,6 +32,7 @@ import ( // - *api.ServiceProviderCluster // - *arm.Subscription // - *api.Controller +// - *api.ManagementClusterContent // // Returns an error if any resource cannot be created or if an unsupported type is encountered. func NewMockDBClientWithResources(ctx context.Context, resources []any) (*MockDBClient, error) { @@ -63,6 +64,8 @@ func (m *MockDBClient) addResource(ctx context.Context, resource any) error { return m.addSubscription(ctx, r) case *api.Controller: return m.addController(ctx, r) + case *api.ManagementClusterContent: + return m.addManagementClusterContent(ctx, r) default: return fmt.Errorf("unsupported resource type: %T", resource) } @@ -172,3 +175,32 @@ func (m *MockDBClient) addController(ctx context.Context, controller *api.Contro } return fmt.Errorf("unsupported parent resource type: %s", parentType) } + +func (m *MockDBClient) addManagementClusterContent(ctx context.Context, mcc *api.ManagementClusterContent) error { + resourceID := mcc.GetResourceID() + if resourceID == nil { + return fmt.Errorf("management cluster content is missing resource ID") + } + if resourceID.Parent == nil { + return fmt.Errorf("management cluster content is missing parent ID") + } + parentType := resourceID.Parent.ResourceType + switch { + case armhelpers.ResourceTypeEqual(parentType, api.ClusterResourceType): + clusterName := resourceID.Parent.Name + mccCRUD := m.HCPClusters(resourceID.SubscriptionID, resourceID.ResourceGroupName).ManagementClusterContents(clusterName) + _, err := mccCRUD.Create(ctx, mcc, nil) + return err + case armhelpers.ResourceTypeEqual(parentType, api.NodePoolResourceType): + if resourceID.Parent.Parent == nil { + return fmt.Errorf("node pool management cluster content is missing grandparent cluster ID") + } + clusterName := resourceID.Parent.Parent.Name + nodePoolName := resourceID.Parent.Name + mccCRUD := m.HCPClusters(resourceID.SubscriptionID, resourceID.ResourceGroupName).NodePools(clusterName).ManagementClusterContents(nodePoolName) + _, err := mccCRUD.Create(ctx, mcc, nil) + return err + default: + return fmt.Errorf("unsupported parent resource type for management cluster content: %s", parentType) + } +} From 6ba9ddd42de815a69a29da1adbc3f6c344493c05 Mon Sep 17 00:00:00 2001 From: Miguel Soriano Date: Wed, 1 Apr 2026 10:24:26 +0200 Subject: [PATCH 11/16] refactor: make maestro bundle name generator fully unexported in maestro controllers --- backend/pkg/app/backend.go | 4 ---- ...eate_cluster_scoped_maestro_readonly_bundles_controller.go | 3 +-- ...ate_nodepool_scoped_maestro_readonly_bundles_controller.go | 3 +-- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/backend/pkg/app/backend.go b/backend/pkg/app/backend.go index 9bfe68a748e..3841e4ba6fb 100644 --- a/backend/pkg/app/backend.go +++ b/backend/pkg/app/backend.go @@ -496,12 +496,9 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, backendInformers, ) - maestroAPIMaestroBundleNameGenerator := maestro.NewMaestroAPIMaestroBundleNameGenerator() - maestroCreateClusterScopedReadonlyBundlesController := controllers.NewCreateClusterScopedMaestroReadonlyBundlesController( activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient, backendInformers, b.options.MaestroSourceEnvironmentIdentifier, maestroClientBuilder, - maestroAPIMaestroBundleNameGenerator, ) maestroReadAndPersistClusterScopedReadonlyBundlesContentController := controllers.NewReadAndPersistClusterScopedMaestroReadonlyBundlesContentController( activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient, @@ -511,7 +508,6 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, maestroCreateNodePoolScopedReadonlyBundlesController := controllers.NewCreateNodePoolScopedMaestroReadonlyBundlesController( activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient, backendInformers, b.options.MaestroSourceEnvironmentIdentifier, maestroClientBuilder, - maestroAPIMaestroBundleNameGenerator, ) maestroReadAndPersistNodePoolScopedReadonlyBundlesContentController := controllers.NewReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentController( activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient, diff --git a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go index 2f84d4884bf..b8e67f6fe1b 100644 --- a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go @@ -76,7 +76,6 @@ func NewCreateClusterScopedMaestroReadonlyBundlesController( informers informers.BackendInformers, maestroSourceEnvironmentIdentifier string, maestroClientBuilder maestro.MaestroClientBuilder, - maestroAPIMaestroBundleNameGenerator maestro.MaestroAPIMaestroBundleNameGenerator, ) controllerutils.Controller { syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ @@ -86,7 +85,7 @@ func NewCreateClusterScopedMaestroReadonlyBundlesController( activeOperationLister: activeOperationLister, maestroSourceEnvironmentIdentifier: maestroSourceEnvironmentIdentifier, maestroClientBuilder: maestroClientBuilder, - maestroAPIMaestroBundleNameGenerator: maestroAPIMaestroBundleNameGenerator, + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } controller := controllerutils.NewClusterWatchingController( diff --git a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go index 4d8fcea199b..4465d11424e 100644 --- a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go @@ -76,7 +76,6 @@ func NewCreateNodePoolScopedMaestroReadonlyBundlesController( informers informers.BackendInformers, maestroSourceEnvironmentIdentifier string, maestroClientBuilder maestro.MaestroClientBuilder, - maestroAPIMaestroBundleNameGenerator maestro.MaestroAPIMaestroBundleNameGenerator, ) controllerutils.Controller { syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ @@ -86,7 +85,7 @@ func NewCreateNodePoolScopedMaestroReadonlyBundlesController( activeOperationLister: activeOperationLister, maestroSourceEnvironmentIdentifier: maestroSourceEnvironmentIdentifier, maestroClientBuilder: maestroClientBuilder, - maestroAPIMaestroBundleNameGenerator: maestroAPIMaestroBundleNameGenerator, + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } controller := controllerutils.NewNodePoolWatchingController( From d87619e31dfb6772312abaaa8fa566691f081b2e Mon Sep 17 00:00:00 2001 From: Miguel Soriano Date: Tue, 7 Apr 2026 10:44:08 +0200 Subject: [PATCH 12/16] refactor: move maestro readonly bundles resource specific tests to package level ones --- .../maestro_readonly_bundle_helpers_test.go | 368 +++++++++++++++++ ...eadonly_bundles_content_controller_test.go | 383 ------------------ ...eadonly_bundles_content_controller_test.go | 314 -------------- 3 files changed, 368 insertions(+), 697 deletions(-) diff --git a/backend/pkg/controllers/maestro_readonly_bundle_helpers_test.go b/backend/pkg/controllers/maestro_readonly_bundle_helpers_test.go index e4b3602373a..bd2d0742556 100644 --- a/backend/pkg/controllers/maestro_readonly_bundle_helpers_test.go +++ b/backend/pkg/controllers/maestro_readonly_bundle_helpers_test.go @@ -15,20 +15,34 @@ package controllers import ( + "context" + "encoding/json" + "fmt" + "net/http" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" workv1 "open-cluster-management.io/api/work/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos" + hsv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" + "github.com/Azure/ARO-HCP/backend/pkg/maestro" "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/api/arm" + "github.com/Azure/ARO-HCP/internal/database" + "github.com/Azure/ARO-HCP/internal/databasetesting" ) // buildTestMaestroBundleWithStatusFeedback builds a ManifestWork with exactly one resource status manifest @@ -227,3 +241,357 @@ func TestMaestroReadonlyBundleHelpers_getSingleResourceStatusFeedbackRawJSONFrom }) } } + +// errorInjectingMCCCRUD wraps ManagementClusterContentCRUD to allow error injection for testing. +type errorInjectingMCCCRUD struct { + database.ManagementClusterContentCRUD + getResult *api.ManagementClusterContent + getErr error + replaceErr error +} + +func (e *errorInjectingMCCCRUD) Get(ctx context.Context, resourceID string) (*api.ManagementClusterContent, error) { + if e.getErr != nil { + return nil, e.getErr + } + if e.getResult != nil { + return e.getResult, nil + } + return e.ManagementClusterContentCRUD.Get(ctx, resourceID) +} + +func (e *errorInjectingMCCCRUD) Replace(ctx context.Context, obj *api.ManagementClusterContent, opts *azcosmos.ItemOptions) (*api.ManagementClusterContent, error) { + if e.replaceErr != nil { + return nil, e.replaceErr + } + return e.ManagementClusterContentCRUD.Replace(ctx, obj, opts) +} + +var _ database.ManagementClusterContentCRUD = &errorInjectingMCCCRUD{} + +// hcpClusterCRUDWithInjectedMCC wraps HCPClusterCRUD to return a fixed ManagementClusterContentCRUD (for tests). +type hcpClusterCRUDWithInjectedMCC struct { + database.HCPClusterCRUD + mccCRUD database.ManagementClusterContentCRUD +} + +func (e *hcpClusterCRUDWithInjectedMCC) ManagementClusterContents(hcpClusterName string) database.ManagementClusterContentCRUD { + return e.mccCRUD +} + +var _ database.HCPClusterCRUD = &hcpClusterCRUDWithInjectedMCC{} + +// errorInjectingDBClient wraps MockDBClient to return error-injecting CRUDs. +type errorInjectingDBClient struct { + *databasetesting.MockDBClient + mccCRUD database.ManagementClusterContentCRUD + clustersCRUD database.HCPClusterCRUD + spcCRUD database.ServiceProviderClusterCRUD +} + +func (e *errorInjectingDBClient) HCPClusters(subscriptionID, resourceGroupName string) database.HCPClusterCRUD { + var base database.HCPClusterCRUD + if e.clustersCRUD != nil { + base = e.clustersCRUD + } else { + base = e.MockDBClient.HCPClusters(subscriptionID, resourceGroupName) + } + if e.mccCRUD != nil { + return &hcpClusterCRUDWithInjectedMCC{HCPClusterCRUD: base, mccCRUD: e.mccCRUD} + } + return base +} + +func (e *errorInjectingDBClient) ServiceProviderClusters(subscriptionID, resourceGroupName, clusterName string) database.ServiceProviderClusterCRUD { + if e.spcCRUD != nil { + return e.spcCRUD + } + return e.MockDBClient.ServiceProviderClusters(subscriptionID, resourceGroupName, clusterName) +} + +var _ database.DBClient = &errorInjectingDBClient{} + +// errorInjectingSPCCRUD wraps ServiceProviderClusterCRUD to allow error injection. +type errorInjectingSPCCRUD struct { + database.ServiceProviderClusterCRUD + getErr error +} + +func (e *errorInjectingSPCCRUD) Get(ctx context.Context, resourceID string) (*api.ServiceProviderCluster, error) { + if e.getErr != nil { + return nil, e.getErr + } + return e.ServiceProviderClusterCRUD.Get(ctx, resourceID) +} + +var _ database.ServiceProviderClusterCRUD = &errorInjectingSPCCRUD{} + +func TestMaestroReadonlyBundleHelpers_calculateManagementClusterContentFromMaestroBundle(t *testing.T) { + clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) + cluster := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterResourceID}}, + } + ref := &api.MaestroBundleReference{ + Name: api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster, + MaestroAPIMaestroBundleName: "bundle-name", + } + + hc := hsv1beta1.HostedCluster{TypeMeta: metav1.TypeMeta{APIVersion: "hypershift.openshift.io/v1beta1", Kind: "HostedCluster"}, ObjectMeta: metav1.ObjectMeta{Name: "hc1", Namespace: "ns1"}} + hcJSONBytes, err := json.Marshal(hc) + require.NoError(t, err) + validHCJSON := string(hcJSONBytes) + + tests := []struct { + name string + maestroGet func(*maestro.MockClient) + wantDegraded bool + wantKubeContent bool + wantErr bool + errSub string + }{ + { + name: "bundle not found - desired with degraded condition", + maestroGet: func(m *maestro.MockClient) { + m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "bundle-name")) + }, + wantDegraded: true, + wantKubeContent: false, + }, + { + name: "maestro api maestro bundleget error - returns error", + maestroGet: func(m *maestro.MockClient) { + m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(nil, fmt.Errorf("connection error")) + }, + wantErr: true, + errSub: "failed to get Maestro Bundle", + }, + { + name: "bundle has invalid status feedback - desired with degraded", + maestroGet: func(m *maestro.MockClient) { + // Bundle with no status feedback values + b := &workv1.ManifestWork{ + Status: workv1.ManifestWorkStatus{ + ResourceStatus: workv1.ManifestResourceStatus{ + Manifests: []workv1.ManifestCondition{ + {ResourceMeta: workv1.ManifestResourceMeta{}, StatusFeedbacks: workv1.StatusFeedbackResult{Values: []workv1.FeedbackValue{}}, Conditions: []metav1.Condition{}}, + }, + }, + }, + } + m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + }, + wantDegraded: true, + wantKubeContent: false, + }, + { + name: "success - desired with kube content", + maestroGet: func(m *maestro.MockClient) { + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) + m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + }, + wantDegraded: false, + wantKubeContent: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + tt.maestroGet(mockMaestro) + + got, err := calculateManagementClusterContentFromMaestroBundle(context.Background(), cluster.ID, ref, mockMaestro) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errSub) + } else { + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, tt.wantKubeContent, got.Status.KubeContent != nil && len(got.Status.KubeContent.Items) > 0) + hasDegradedTrue := controllerutils.IsConditionTrue(got.Status.Conditions, "Degraded") + assert.Equal(t, tt.wantDegraded, hasDegradedTrue) + } + }) + } +} + +func TestMaestroReadonlyBundleHelpers_readAndPersistMaestroReadonlyBundleContent(t *testing.T) { + ctx := context.Background() + clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) + cluster := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterResourceID}}, + } + ref := &api.MaestroBundleReference{ + Name: api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster, + MaestroAPIMaestroBundleName: "bundle-name", + } + hc := hsv1beta1.HostedCluster{TypeMeta: metav1.TypeMeta{APIVersion: "hypershift.openshift.io/v1beta1", Kind: "HostedCluster"}, ObjectMeta: metav1.ObjectMeta{Name: "hc1", Namespace: "ns1"}} + hcJSONBytes, err := json.Marshal(hc) + require.NoError(t, err) + validHCJSON := string(hcJSONBytes) + + t.Run("creates new ManagementClusterContent when not found", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + + mockDB := databasetesting.NewMockDBClient() + mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") + + err := readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) + require.NoError(t, err) + + // Content should have been created (name = bundle internal name) + got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.Status.KubeContent) + require.Len(t, got.Status.KubeContent.Items, 1) + }) + + t.Run("replaces existing when content changed", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + + mockDB := databasetesting.NewMockDBClient() + mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") + // Pre-create existing content with different payload + existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) + existing := &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, + ResourceID: *existingRID, + Status: api.ManagementClusterContentStatus{KubeContent: &metav1.List{Items: []runtime.RawExtension{}}}, + } + _, err := mccCRUD.Create(ctx, existing, nil) + require.NoError(t, err) + + err = readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) + require.NoError(t, err) + + got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) + require.NoError(t, err) + require.NotNil(t, got.Status.KubeContent) + require.Len(t, got.Status.KubeContent.Items, 1) + }) + + t.Run("keeps existing kube content when desired has no content (degraded)", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + // Return bundle that has no valid status feedback so desired has no KubeContent + b := &workv1.ManifestWork{ + Status: workv1.ManifestWorkStatus{ + ResourceStatus: workv1.ManifestResourceStatus{ + Manifests: []workv1.ManifestCondition{ + {ResourceMeta: workv1.ManifestResourceMeta{}, StatusFeedbacks: workv1.StatusFeedbackResult{Values: []workv1.FeedbackValue{}}, Conditions: []metav1.Condition{}}, + }, + }, + }, + } + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + + mockDB := databasetesting.NewMockDBClient() + mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") + existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) + existingContent := &metav1.List{Items: []runtime.RawExtension{{Raw: []byte(`{}`)}}} + existing := &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, + ResourceID: *existingRID, + Status: api.ManagementClusterContentStatus{KubeContent: existingContent}, + } + _, err := mccCRUD.Create(ctx, existing, nil) + require.NoError(t, err) + + err = readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) + require.NoError(t, err) + + got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) + require.NoError(t, err) + // Should have kept existing content + require.NotNil(t, got.Status.KubeContent) + assert.Equal(t, existingContent.Items[0].Raw, got.Status.KubeContent.Items[0].Raw) + }) + + t.Run("no replace occurs when content has not changed", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) + // Get is called once when building desired for pre-create, and once inside readAndPersistMaestroReadonlyBundleContent. + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil).Times(2) + + mockDB := databasetesting.NewMockDBClient() + mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") + existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) + // Pre-create content that matches exactly what the syncer would compute (same KubeContent and Degraded=False condition) + // so that DeepEqual(existing, desired) is true and Replace is not called. + desired, err := calculateManagementClusterContentFromMaestroBundle(ctx, cluster.ID, ref, mockMaestro) + require.NoError(t, err) + require.NotNil(t, desired) + existing := &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, + ResourceID: *existingRID, + Status: desired.Status, + } + _, err = mccCRUD.Create(ctx, existing, nil) + require.NoError(t, err) + + err = readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) + require.NoError(t, err) + + // Document should still exist with same content (no Replace was needed) + got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) + require.NoError(t, err) + require.NotNil(t, got.Status.KubeContent) + require.Len(t, got.Status.KubeContent.Items, 1) + }) + + t.Run("error occurs when object has been modified in Cosmos since we retrieved it", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + + existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) + existingDoc := &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, + ResourceID: *existingRID, + Status: api.ManagementClusterContentStatus{KubeContent: &metav1.List{Items: []runtime.RawExtension{{Raw: []byte(validHCJSON)}}}}, + } + + // Use error-injecting wrapper to simulate 412 Precondition Failed on Replace + mockDB := &errorInjectingDBClient{ + MockDBClient: databasetesting.NewMockDBClient(), + mccCRUD: &errorInjectingMCCCRUD{ + getResult: existingDoc, + replaceErr: databasetesting.NewPreconditionFailedError(), + }, + } + + err := readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mockDB.mccCRUD) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to replace ManagementClusterContent") + assert.True(t, database.IsResponseError(err, http.StatusPreconditionFailed), "expected 412 Precondition Failed") + }) + + t.Run("error occurs when managementClusterContentsDBClient.Get fails", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + + getErr := fmt.Errorf("cosmos connection error") + // Use error-injecting wrapper to simulate Get error + mockDB := &errorInjectingDBClient{ + MockDBClient: databasetesting.NewMockDBClient(), + mccCRUD: &errorInjectingMCCCRUD{ + getErr: getErr, + }, + } + + err := readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mockDB.mccCRUD) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to get ManagementClusterContent") + assert.Contains(t, err.Error(), "cosmos connection error") + }) +} diff --git a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go index 1956338be87..42962c9cf55 100644 --- a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go +++ b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go @@ -18,407 +18,24 @@ import ( "context" "encoding/json" "fmt" - "net/http" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" - workv1 "open-cluster-management.io/api/work/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" - "github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos" - - hsv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" "github.com/Azure/ARO-HCP/backend/pkg/maestro" "github.com/Azure/ARO-HCP/internal/api" "github.com/Azure/ARO-HCP/internal/api/arm" - "github.com/Azure/ARO-HCP/internal/database" "github.com/Azure/ARO-HCP/internal/databasetesting" "github.com/Azure/ARO-HCP/internal/ocm" ) -// errorInjectingMCCCRUD wraps ManagementClusterContentCRUD to allow error injection for testing. -type errorInjectingMCCCRUD struct { - database.ManagementClusterContentCRUD - getResult *api.ManagementClusterContent - getErr error - replaceErr error -} - -func (e *errorInjectingMCCCRUD) Get(ctx context.Context, resourceID string) (*api.ManagementClusterContent, error) { - if e.getErr != nil { - return nil, e.getErr - } - if e.getResult != nil { - return e.getResult, nil - } - return e.ManagementClusterContentCRUD.Get(ctx, resourceID) -} - -func (e *errorInjectingMCCCRUD) Replace(ctx context.Context, obj *api.ManagementClusterContent, opts *azcosmos.ItemOptions) (*api.ManagementClusterContent, error) { - if e.replaceErr != nil { - return nil, e.replaceErr - } - return e.ManagementClusterContentCRUD.Replace(ctx, obj, opts) -} - -var _ database.ManagementClusterContentCRUD = &errorInjectingMCCCRUD{} - -// hcpClusterCRUDWithInjectedMCC wraps HCPClusterCRUD to return a fixed ManagementClusterContentCRUD (for tests). -type hcpClusterCRUDWithInjectedMCC struct { - database.HCPClusterCRUD - mccCRUD database.ManagementClusterContentCRUD -} - -func (e *hcpClusterCRUDWithInjectedMCC) ManagementClusterContents(hcpClusterName string) database.ManagementClusterContentCRUD { - return e.mccCRUD -} - -var _ database.HCPClusterCRUD = &hcpClusterCRUDWithInjectedMCC{} - -// errorInjectingDBClient wraps MockDBClient to return error-injecting CRUDs. -type errorInjectingDBClient struct { - *databasetesting.MockDBClient - mccCRUD database.ManagementClusterContentCRUD - clustersCRUD database.HCPClusterCRUD - spcCRUD database.ServiceProviderClusterCRUD -} - -func (e *errorInjectingDBClient) HCPClusters(subscriptionID, resourceGroupName string) database.HCPClusterCRUD { - var base database.HCPClusterCRUD - if e.clustersCRUD != nil { - base = e.clustersCRUD - } else { - base = e.MockDBClient.HCPClusters(subscriptionID, resourceGroupName) - } - if e.mccCRUD != nil { - return &hcpClusterCRUDWithInjectedMCC{HCPClusterCRUD: base, mccCRUD: e.mccCRUD} - } - return base -} - -func (e *errorInjectingDBClient) ServiceProviderClusters(subscriptionID, resourceGroupName, clusterName string) database.ServiceProviderClusterCRUD { - if e.spcCRUD != nil { - return e.spcCRUD - } - return e.MockDBClient.ServiceProviderClusters(subscriptionID, resourceGroupName, clusterName) -} - -var _ database.DBClient = &errorInjectingDBClient{} - -// errorInjectingHCPClusterCRUD wraps HCPClusterCRUD to allow error injection. -type errorInjectingHCPClusterCRUD struct { - database.HCPClusterCRUD - getResult *api.HCPOpenShiftCluster - getErr error -} - -func (e *errorInjectingHCPClusterCRUD) Get(ctx context.Context, resourceID string) (*api.HCPOpenShiftCluster, error) { - if e.getErr != nil { - return nil, e.getErr - } - if e.getResult != nil { - return e.getResult, nil - } - return e.HCPClusterCRUD.Get(ctx, resourceID) -} - -var _ database.HCPClusterCRUD = &errorInjectingHCPClusterCRUD{} - -// errorInjectingSPCCRUD wraps ServiceProviderClusterCRUD to allow error injection. -type errorInjectingSPCCRUD struct { - database.ServiceProviderClusterCRUD - getErr error -} - -func (e *errorInjectingSPCCRUD) Get(ctx context.Context, resourceID string) (*api.ServiceProviderCluster, error) { - if e.getErr != nil { - return nil, e.getErr - } - return e.ServiceProviderClusterCRUD.Get(ctx, resourceID) -} - -var _ database.ServiceProviderClusterCRUD = &errorInjectingSPCCRUD{} - -func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_calculateManagementClusterContentFromMaestroBundle(t *testing.T) { - clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) - cluster := &api.HCPOpenShiftCluster{ - TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterResourceID}}, - } - ref := &api.MaestroBundleReference{ - Name: api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster, - MaestroAPIMaestroBundleName: "bundle-name", - } - - hc := hsv1beta1.HostedCluster{TypeMeta: metav1.TypeMeta{APIVersion: "hypershift.openshift.io/v1beta1", Kind: "HostedCluster"}, ObjectMeta: metav1.ObjectMeta{Name: "hc1", Namespace: "ns1"}} - hcJSONBytes, err := json.Marshal(hc) - require.NoError(t, err) - validHCJSON := string(hcJSONBytes) - - tests := []struct { - name string - maestroGet func(*maestro.MockClient) - wantDegraded bool - wantKubeContent bool - wantErr bool - errSub string - }{ - { - name: "bundle not found - desired with degraded condition", - maestroGet: func(m *maestro.MockClient) { - m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "bundle-name")) - }, - wantDegraded: true, - wantKubeContent: false, - }, - { - name: "maestro api maestro bundleget error - returns error", - maestroGet: func(m *maestro.MockClient) { - m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(nil, fmt.Errorf("connection error")) - }, - wantErr: true, - errSub: "failed to get Maestro Bundle", - }, - { - name: "bundle has invalid status feedback - desired with degraded", - maestroGet: func(m *maestro.MockClient) { - // Bundle with no status feedback values - b := &workv1.ManifestWork{ - Status: workv1.ManifestWorkStatus{ - ResourceStatus: workv1.ManifestResourceStatus{ - Manifests: []workv1.ManifestCondition{ - {ResourceMeta: workv1.ManifestResourceMeta{}, StatusFeedbacks: workv1.StatusFeedbackResult{Values: []workv1.FeedbackValue{}}, Conditions: []metav1.Condition{}}, - }, - }, - }, - } - m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - }, - wantDegraded: true, - wantKubeContent: false, - }, - { - name: "success - desired with kube content", - maestroGet: func(m *maestro.MockClient) { - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) - m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - }, - wantDegraded: false, - wantKubeContent: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - tt.maestroGet(mockMaestro) - - got, err := calculateManagementClusterContentFromMaestroBundle(context.Background(), cluster.ID, ref, mockMaestro) - if tt.wantErr { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.errSub) - } else { - require.NoError(t, err) - require.NotNil(t, got) - assert.Equal(t, tt.wantKubeContent, got.Status.KubeContent != nil && len(got.Status.KubeContent.Items) > 0) - hasDegradedTrue := controllerutils.IsConditionTrue(got.Status.Conditions, "Degraded") - assert.Equal(t, tt.wantDegraded, hasDegradedTrue) - } - }) - } -} - -func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndPersistMaestroBundleContent(t *testing.T) { - ctx := context.Background() - clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) - cluster := &api.HCPOpenShiftCluster{ - TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterResourceID}}, - } - ref := &api.MaestroBundleReference{ - Name: api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster, - MaestroAPIMaestroBundleName: "bundle-name", - } - hc := hsv1beta1.HostedCluster{TypeMeta: metav1.TypeMeta{APIVersion: "hypershift.openshift.io/v1beta1", Kind: "HostedCluster"}, ObjectMeta: metav1.ObjectMeta{Name: "hc1", Namespace: "ns1"}} - hcJSONBytes, err := json.Marshal(hc) - require.NoError(t, err) - validHCJSON := string(hcJSONBytes) - - t.Run("creates new ManagementClusterContent when not found", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - - mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") - - err := readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) - require.NoError(t, err) - - // Content should have been created (name = bundle internal name) - got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) - require.NoError(t, err) - require.NotNil(t, got) - require.NotNil(t, got.Status.KubeContent) - require.Len(t, got.Status.KubeContent.Items, 1) - }) - - t.Run("replaces existing when content changed", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - - mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") - // Pre-create existing content with different payload - existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) - existing := &api.ManagementClusterContent{ - CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, - ResourceID: *existingRID, - Status: api.ManagementClusterContentStatus{KubeContent: &metav1.List{Items: []runtime.RawExtension{}}}, - } - _, err := mccCRUD.Create(ctx, existing, nil) - require.NoError(t, err) - - err = readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) - require.NoError(t, err) - - got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) - require.NoError(t, err) - require.NotNil(t, got.Status.KubeContent) - require.Len(t, got.Status.KubeContent.Items, 1) - }) - - t.Run("keeps existing kube content when desired has no content (degraded)", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - // Return bundle that has no valid status feedback so desired has no KubeContent - b := &workv1.ManifestWork{ - Status: workv1.ManifestWorkStatus{ - ResourceStatus: workv1.ManifestResourceStatus{ - Manifests: []workv1.ManifestCondition{ - {ResourceMeta: workv1.ManifestResourceMeta{}, StatusFeedbacks: workv1.StatusFeedbackResult{Values: []workv1.FeedbackValue{}}, Conditions: []metav1.Condition{}}, - }, - }, - }, - } - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - - mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") - existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) - existingContent := &metav1.List{Items: []runtime.RawExtension{{Raw: []byte(`{}`)}}} - existing := &api.ManagementClusterContent{ - CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, - ResourceID: *existingRID, - Status: api.ManagementClusterContentStatus{KubeContent: existingContent}, - } - _, err := mccCRUD.Create(ctx, existing, nil) - require.NoError(t, err) - - err = readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) - require.NoError(t, err) - - got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) - require.NoError(t, err) - // Should have kept existing content - require.NotNil(t, got.Status.KubeContent) - assert.Equal(t, existingContent.Items[0].Raw, got.Status.KubeContent.Items[0].Raw) - }) - - t.Run("no replace occurs when content has not changed", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) - // Get is called once when building desired for pre-create, and once inside readAndPersistMaestroReadonlyBundleContent. - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil).Times(2) - - mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") - existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) - // Pre-create content that matches exactly what the syncer would compute (same KubeContent and Degraded=False condition) - // so that DeepEqual(existing, desired) is true and Replace is not called. - desired, err := calculateManagementClusterContentFromMaestroBundle(ctx, cluster.ID, ref, mockMaestro) - require.NoError(t, err) - require.NotNil(t, desired) - existing := &api.ManagementClusterContent{ - CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, - ResourceID: *existingRID, - Status: desired.Status, - } - _, err = mccCRUD.Create(ctx, existing, nil) - require.NoError(t, err) - - err = readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) - require.NoError(t, err) - - // Document should still exist with same content (no Replace was needed) - got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) - require.NoError(t, err) - require.NotNil(t, got.Status.KubeContent) - require.Len(t, got.Status.KubeContent.Items, 1) - }) - - t.Run("error occurs when object has been modified in Cosmos since we retrieved it", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - - existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) - existingDoc := &api.ManagementClusterContent{ - CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, - ResourceID: *existingRID, - Status: api.ManagementClusterContentStatus{KubeContent: &metav1.List{Items: []runtime.RawExtension{{Raw: []byte(validHCJSON)}}}}, - } - - // Use error-injecting wrapper to simulate 412 Precondition Failed on Replace - mockDB := &errorInjectingDBClient{ - MockDBClient: databasetesting.NewMockDBClient(), - mccCRUD: &errorInjectingMCCCRUD{ - getResult: existingDoc, - replaceErr: databasetesting.NewPreconditionFailedError(), - }, - } - - err := readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mockDB.mccCRUD) - require.Error(t, err) - assert.Contains(t, err.Error(), "failed to replace ManagementClusterContent") - assert.True(t, database.IsResponseError(err, http.StatusPreconditionFailed), "expected 412 Precondition Failed") - }) - - t.Run("error occurs when managementClusterContentsDBClient.Get fails", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - - getErr := fmt.Errorf("cosmos connection error") - // Use error-injecting wrapper to simulate Get error - mockDB := &errorInjectingDBClient{ - MockDBClient: databasetesting.NewMockDBClient(), - mccCRUD: &errorInjectingMCCCRUD{ - getErr: getErr, - }, - } - - err := readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mockDB.mccCRUD) - require.Error(t, err) - assert.Contains(t, err.Error(), "failed to get ManagementClusterContent") - assert.Contains(t, err.Error(), "cosmos connection error") - }) -} - func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_SyncOnce_ClusterNotFound(t *testing.T) { mockDBClient := databasetesting.NewMockDBClient() syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{ diff --git a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go index ea740917c06..51d7b9ce4d9 100644 --- a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go +++ b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go @@ -18,24 +18,15 @@ import ( "context" "encoding/json" "fmt" - "net/http" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" - workv1 "open-cluster-management.io/api/work/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" - "github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos" - - hsv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" "github.com/Azure/ARO-HCP/backend/pkg/maestro" @@ -74,311 +65,6 @@ func (e *errorInjectingSPNPCRUD) Get(ctx context.Context, resourceID string) (*a return e.ServiceProviderNodePoolCRUD.Get(ctx, resourceID) } -// errorInjectingMCCCRUDForNodePool wraps ManagementClusterContentCRUD to allow error injection. -type errorInjectingMCCCRUDForNodePool struct { - database.ManagementClusterContentCRUD - getResult *api.ManagementClusterContent - getErr error - replaceErr error -} - -func (e *errorInjectingMCCCRUDForNodePool) Get(ctx context.Context, resourceID string) (*api.ManagementClusterContent, error) { - if e.getErr != nil { - return nil, e.getErr - } - if e.getResult != nil { - return e.getResult, nil - } - return e.ManagementClusterContentCRUD.Get(ctx, resourceID) -} - -func (e *errorInjectingMCCCRUDForNodePool) Replace(ctx context.Context, obj *api.ManagementClusterContent, opts *azcosmos.ItemOptions) (*api.ManagementClusterContent, error) { - if e.replaceErr != nil { - return nil, e.replaceErr - } - return e.ManagementClusterContentCRUD.Replace(ctx, obj, opts) -} - -var _ database.ManagementClusterContentCRUD = &errorInjectingMCCCRUDForNodePool{} - -func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_calculateManagementClusterContentFromMaestroBundle(t *testing.T) { - nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/nodePools/test-nodepool")) - nodepool := &api.HCPOpenShiftClusterNodePool{ - TrackedResource: arm.TrackedResource{ - Resource: arm.Resource{ - ID: nodepoolResourceID, - Name: "test-nodepool", - }, - }, - } - ref := &api.MaestroBundleReference{ - Name: api.MaestroBundleInternalNameReadonlyHypershiftNodePool, - MaestroAPIMaestroBundleName: "bundle-name", - } - - np := hsv1beta1.NodePool{TypeMeta: metav1.TypeMeta{APIVersion: "hypershift.openshift.io/v1beta1", Kind: "NodePool"}, ObjectMeta: metav1.ObjectMeta{Name: "np1", Namespace: "ns1"}} - npJSONBytes, err := json.Marshal(np) - require.NoError(t, err) - validNPJSON := string(npJSONBytes) - - tests := []struct { - name string - maestroGet func(*maestro.MockClient) - wantDegraded bool - wantKubeContent bool - wantErr bool - errSub string - }{ - { - name: "bundle not found - desired with degraded condition", - maestroGet: func(m *maestro.MockClient) { - m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "bundle-name")) - }, - wantDegraded: true, - wantKubeContent: false, - }, - { - name: "maestro bundle get error - returns error", - maestroGet: func(m *maestro.MockClient) { - m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(nil, fmt.Errorf("connection error")) - }, - wantErr: true, - errSub: "failed to get Maestro Bundle", - }, - { - name: "bundle has invalid status feedback - desired with degraded", - maestroGet: func(m *maestro.MockClient) { - b := &workv1.ManifestWork{ - Status: workv1.ManifestWorkStatus{ - ResourceStatus: workv1.ManifestResourceStatus{ - Manifests: []workv1.ManifestCondition{ - {ResourceMeta: workv1.ManifestResourceMeta{}, StatusFeedbacks: workv1.StatusFeedbackResult{Values: []workv1.FeedbackValue{}}, Conditions: []metav1.Condition{}}, - }, - }, - }, - } - m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - }, - wantDegraded: true, - wantKubeContent: false, - }, - { - name: "success - desired with kube content", - maestroGet: func(m *maestro.MockClient) { - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validNPJSON) - m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - }, - wantDegraded: false, - wantKubeContent: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - tt.maestroGet(mockMaestro) - - got, err := calculateManagementClusterContentFromMaestroBundle(context.Background(), nodepool.ID, ref, mockMaestro) - if tt.wantErr { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.errSub) - } else { - require.NoError(t, err) - require.NotNil(t, got) - assert.Equal(t, tt.wantKubeContent, got.Status.KubeContent != nil && len(got.Status.KubeContent.Items) > 0) - hasDegradedTrue := controllerutils.IsConditionTrue(got.Status.Conditions, "Degraded") - assert.Equal(t, tt.wantDegraded, hasDegradedTrue) - } - }) - } -} - -func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_readAndPersistMaestroBundleContent(t *testing.T) { - ctx := context.Background() - nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/nodePools/test-nodepool")) - nodepool := &api.HCPOpenShiftClusterNodePool{ - TrackedResource: arm.TrackedResource{ - Resource: arm.Resource{ - ID: nodepoolResourceID, - Name: "test-nodepool", - }, - }, - } - bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool - ref := &api.MaestroBundleReference{ - Name: bundleInternalName, - MaestroAPIMaestroBundleName: "bundle-name", - } - - np := hsv1beta1.NodePool{TypeMeta: metav1.TypeMeta{APIVersion: "hypershift.openshift.io/v1beta1", Kind: "NodePool"}, ObjectMeta: metav1.ObjectMeta{Name: "np1", Namespace: "ns1"}} - npJSONBytes, err := json.Marshal(np) - require.NoError(t, err) - validNPJSON := string(npJSONBytes) - - // Node-pool–scoped ManagementClusterContents are nested under the node pool ARM resource. - const mccContainerSub = "sub" - const mccContainerRG = "rg" - const mccContainerCluster = "cluster" - const mccNodePoolName = "test-nodepool" - existingMCCResourceIDStr := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/%s/nodePools/%s/managementClusterContents/%s", - mccContainerSub, mccContainerRG, mccContainerCluster, mccNodePoolName, bundleInternalName) - - t.Run("creates new ManagementClusterContent when not found", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validNPJSON) - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - - mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.HCPClusters(mccContainerSub, mccContainerRG).NodePools(mccContainerCluster).ManagementClusterContents(mccNodePoolName) - - err := readAndPersistMaestroReadonlyBundleContent(ctx, nodepool.ID, ref, mockMaestro, mccCRUD) - require.NoError(t, err) - - got, err := mccCRUD.Get(ctx, string(bundleInternalName)) - require.NoError(t, err) - require.NotNil(t, got) - require.NotNil(t, got.Status.KubeContent) - require.Len(t, got.Status.KubeContent.Items, 1) - }) - - t.Run("replaces existing when content changed", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validNPJSON) - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - - mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.HCPClusters(mccContainerSub, mccContainerRG).NodePools(mccContainerCluster).ManagementClusterContents(mccNodePoolName) - existingRID := api.Must(azcorearm.ParseResourceID(existingMCCResourceIDStr)) - existing := &api.ManagementClusterContent{ - CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, - ResourceID: *existingRID, - Status: api.ManagementClusterContentStatus{KubeContent: &metav1.List{Items: []runtime.RawExtension{}}}, - } - _, err := mccCRUD.Create(ctx, existing, nil) - require.NoError(t, err) - - err = readAndPersistMaestroReadonlyBundleContent(ctx, nodepool.ID, ref, mockMaestro, mccCRUD) - require.NoError(t, err) - - got, err := mccCRUD.Get(ctx, string(bundleInternalName)) - require.NoError(t, err) - require.NotNil(t, got.Status.KubeContent) - require.Len(t, got.Status.KubeContent.Items, 1) - }) - - t.Run("keeps existing kube content when desired has no content (degraded)", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - // Bundle with no valid status feedback -> desired has no KubeContent - b := &workv1.ManifestWork{ - Status: workv1.ManifestWorkStatus{ - ResourceStatus: workv1.ManifestResourceStatus{ - Manifests: []workv1.ManifestCondition{ - {ResourceMeta: workv1.ManifestResourceMeta{}, StatusFeedbacks: workv1.StatusFeedbackResult{Values: []workv1.FeedbackValue{}}, Conditions: []metav1.Condition{}}, - }, - }, - }, - } - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - - mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.HCPClusters(mccContainerSub, mccContainerRG).NodePools(mccContainerCluster).ManagementClusterContents(mccNodePoolName) - existingRID := api.Must(azcorearm.ParseResourceID(existingMCCResourceIDStr)) - existingContent := &metav1.List{Items: []runtime.RawExtension{{Raw: []byte(`{}`)}}} - existing := &api.ManagementClusterContent{ - CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, - ResourceID: *existingRID, - Status: api.ManagementClusterContentStatus{KubeContent: existingContent}, - } - _, err := mccCRUD.Create(ctx, existing, nil) - require.NoError(t, err) - - err = readAndPersistMaestroReadonlyBundleContent(ctx, nodepool.ID, ref, mockMaestro, mccCRUD) - require.NoError(t, err) - - got, err := mccCRUD.Get(ctx, string(bundleInternalName)) - require.NoError(t, err) - require.NotNil(t, got.Status.KubeContent) - assert.Equal(t, existingContent.Items[0].Raw, got.Status.KubeContent.Items[0].Raw) - }) - - t.Run("no replace occurs when content has not changed", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validNPJSON) - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil).Times(2) - - mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.HCPClusters(mccContainerSub, mccContainerRG).NodePools(mccContainerCluster).ManagementClusterContents(mccNodePoolName) - existingRID := api.Must(azcorearm.ParseResourceID(existingMCCResourceIDStr)) - desired, err := calculateManagementClusterContentFromMaestroBundle(ctx, nodepool.ID, ref, mockMaestro) - require.NoError(t, err) - existing := &api.ManagementClusterContent{ - CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, - ResourceID: *existingRID, - Status: desired.Status, - } - _, err = mccCRUD.Create(ctx, existing, nil) - require.NoError(t, err) - - err = readAndPersistMaestroReadonlyBundleContent(ctx, nodepool.ID, ref, mockMaestro, mccCRUD) - require.NoError(t, err) - - got, err := mccCRUD.Get(ctx, string(bundleInternalName)) - require.NoError(t, err) - require.NotNil(t, got.Status.KubeContent) - require.Len(t, got.Status.KubeContent.Items, 1) - }) - - t.Run("error occurs when object has been modified in Cosmos since we retrieved it", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validNPJSON) - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - - existingRID := api.Must(azcorearm.ParseResourceID(existingMCCResourceIDStr)) - existingDoc := &api.ManagementClusterContent{ - CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, - ResourceID: *existingRID, - Status: api.ManagementClusterContentStatus{KubeContent: &metav1.List{Items: []runtime.RawExtension{{Raw: []byte(validNPJSON)}}}}, - } - - mockDB := databasetesting.NewMockDBClient() - baseMccCRUD := mockDB.HCPClusters(mccContainerSub, mccContainerRG).NodePools(mccContainerCluster).ManagementClusterContents(mccNodePoolName) - mccCRUD := &errorInjectingMCCCRUDForNodePool{ - ManagementClusterContentCRUD: baseMccCRUD, - getResult: existingDoc, - replaceErr: databasetesting.NewPreconditionFailedError(), - } - - err := readAndPersistMaestroReadonlyBundleContent(ctx, nodepool.ID, ref, mockMaestro, mccCRUD) - require.Error(t, err) - assert.Contains(t, err.Error(), "failed to replace ManagementClusterContent") - assert.True(t, database.IsResponseError(err, http.StatusPreconditionFailed), "expected 412 Precondition Failed") - }) - - t.Run("error occurs when managementClusterContentsDBClient.Get fails", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validNPJSON) - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - - mockDB := databasetesting.NewMockDBClient() - baseMccCRUD := mockDB.HCPClusters(mccContainerSub, mccContainerRG).NodePools(mccContainerCluster).ManagementClusterContents(mccNodePoolName) - mccCRUD := &errorInjectingMCCCRUDForNodePool{ - ManagementClusterContentCRUD: baseMccCRUD, - getErr: fmt.Errorf("cosmos connection error"), - } - - err := readAndPersistMaestroReadonlyBundleContent(ctx, nodepool.ID, ref, mockMaestro, mccCRUD) - require.Error(t, err) - assert.Contains(t, err.Error(), "failed to get ManagementClusterContent") - assert.Contains(t, err.Error(), "cosmos connection error") - }) -} - func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnce_NodePoolNotFound(t *testing.T) { mockDBClient := databasetesting.NewMockDBClient() syncer := &readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer{ From e3949cd0f9c3832a5576491e27af33886ee9aaea Mon Sep 17 00:00:00 2001 From: Miguel Soriano Date: Tue, 7 Apr 2026 10:47:20 +0200 Subject: [PATCH 13/16] fix: preserve lastTransitionTime in ManagementClusterContents conditions The existing ManagementClusterContent in Cosmos might include conditions that already exist beforehand and that have been calculated in the new desired content. This commit ensures that LastTransitionTime of those conditions in the case where the status of them hasn't changed is preserved. --- .../maestro_readonly_bundle_helpers.go | 25 ++++++++++ .../maestro_readonly_bundle_helpers_test.go | 46 +++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/backend/pkg/controllers/maestro_readonly_bundle_helpers.go b/backend/pkg/controllers/maestro_readonly_bundle_helpers.go index 4fbc59f8276..a92340be031 100644 --- a/backend/pkg/controllers/maestro_readonly_bundle_helpers.go +++ b/backend/pkg/controllers/maestro_readonly_bundle_helpers.go @@ -297,6 +297,31 @@ func readAndPersistMaestroReadonlyBundleContent( desired.Status.KubeContent = existing.Status.KubeContent } + // The existing ManagementClusterContent in Cosmos might include conditions that already exist beforehand and that + // have been calculated in the new desired content. To preserve the LastTransitionTime of those conditions in the case + // where the status of them hasn't changed, what we do is: + // 1. Deep copy of the existing status from Cosmos, which includes the conditions + // 2. Iterate over the newly calculated desired conditions, and for each condition: + // 2.1. Check if the condition already exists in the existing status + // 2.2. If it does, update the condition in the existing status with the new values using SetCondition. This + // will update the LastTransitionTime to the current time if there's been a change or keep the existing + // LastTransitionTime if the condition hasn't changed its status. Then, use the newly updated condition as + // the desired one. + // 2.3. If it does not, then keep the condition as is + // 3. Assign the merged conditions to the desired status. + tmpExistingStatus := existing.Status.DeepCopy() + mergedConditions := make([]api.Condition, 0, len(desired.Status.Conditions)) + for _, desiredCondition := range desired.Status.Conditions { + if controllerutils.GetCondition(tmpExistingStatus.Conditions, desiredCondition.Type) != nil { + controllerutils.SetCondition(&tmpExistingStatus.Conditions, desiredCondition) + merged := controllerutils.GetCondition(tmpExistingStatus.Conditions, desiredCondition.Type) + mergedConditions = append(mergedConditions, *merged) + continue + } + mergedConditions = append(mergedConditions, desiredCondition) + } + desired.Status.Conditions = mergedConditions + if equality.Semantic.DeepEqual(existing, desired) { return nil } diff --git a/backend/pkg/controllers/maestro_readonly_bundle_helpers_test.go b/backend/pkg/controllers/maestro_readonly_bundle_helpers_test.go index bd2d0742556..baee76842da 100644 --- a/backend/pkg/controllers/maestro_readonly_bundle_helpers_test.go +++ b/backend/pkg/controllers/maestro_readonly_bundle_helpers_test.go @@ -20,6 +20,7 @@ import ( "fmt" "net/http" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -594,4 +595,49 @@ func TestMaestroReadonlyBundleHelpers_readAndPersistMaestroReadonlyBundleContent assert.Contains(t, err.Error(), "failed to get ManagementClusterContent") assert.Contains(t, err.Error(), "cosmos connection error") }) + + t.Run("preserves Degraded LastTransitionTime from Cosmos when status unchanged", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil).Times(1) + + mockDB := databasetesting.NewMockDBClient() + mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") + existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) + + u := &unstructured.Unstructured{} + require.NoError(t, json.Unmarshal([]byte(validHCJSON), u)) + historicLTT := time.Date(2020, 6, 15, 12, 0, 0, 0, time.UTC) + existing := &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, + ResourceID: *existingRID, + Status: api.ManagementClusterContentStatus{ + KubeContent: &metav1.List{ + Items: []runtime.RawExtension{{Object: u}}, + }, + Conditions: []api.Condition{ + { + Type: "Degraded", + Status: api.ConditionFalse, + Reason: "NoErrors", + Message: "As expected.", + LastTransitionTime: historicLTT, + }, + }, + }, + } + _, err := mccCRUD.Create(ctx, existing, nil) + require.NoError(t, err) + + err = readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) + require.NoError(t, err) + + got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) + require.NoError(t, err) + degraded := controllerutils.GetCondition(got.Status.Conditions, "Degraded") + require.NotNil(t, degraded) + assert.True(t, degraded.LastTransitionTime.Equal(historicLTT), "expected LastTransitionTime from Cosmos to be preserved, got %v", degraded.LastTransitionTime) + }) + } From 4ec8a5aa891ca1f6a45f0bf1fbfbc22041c5496e Mon Sep 17 00:00:00 2001 From: Miguel Soriano Date: Tue, 7 Apr 2026 17:04:10 +0200 Subject: [PATCH 14/16] feat: scope ManagementClusterContent informer events to immediate parent --- .../cluster_watching_controller.go | 7 +- .../generic_watching_controller.go | 60 +++- .../generic_watching_controller_test.go | 284 ++++++++++++++++++ .../nodepool_watching_controller.go | 8 +- 4 files changed, 349 insertions(+), 10 deletions(-) create mode 100644 backend/pkg/controllers/controllerutils/generic_watching_controller_test.go diff --git a/backend/pkg/controllers/controllerutils/cluster_watching_controller.go b/backend/pkg/controllers/controllerutils/cluster_watching_controller.go index faeea603ef9..19d8c2a9119 100644 --- a/backend/pkg/controllers/controllerutils/cluster_watching_controller.go +++ b/backend/pkg/controllers/controllerutils/cluster_watching_controller.go @@ -64,8 +64,13 @@ func NewClusterWatchingController( if informers != nil { clusterInformer, _ := informers.Clusters() serviceProviderInformer, _ := informers.ServiceProviderClusters() + err := clusterController.QueueForInformers(resyncDuration, clusterInformer, serviceProviderInformer) + if err != nil { + panic(err) // coding error + } managementClusterContentInformer, _ := informers.ManagementClusterContents() - err := clusterController.QueueForInformers(resyncDuration, clusterInformer, serviceProviderInformer, managementClusterContentInformer) + // Limit the max depth of ManagementClusterContent to 1 to only consider the cluster-scoped ManagementClusterContents + err = clusterController.QueueForInformersWithMaxDepth(resyncDuration, 1, managementClusterContentInformer) if err != nil { panic(err) // coding error } diff --git a/backend/pkg/controllers/controllerutils/generic_watching_controller.go b/backend/pkg/controllers/controllerutils/generic_watching_controller.go index d55a98ccfdb..fa8a290133c 100644 --- a/backend/pkg/controllers/controllerutils/generic_watching_controller.go +++ b/backend/pkg/controllers/controllerutils/generic_watching_controller.go @@ -143,13 +143,25 @@ func (c *genericWatchingController[T]) processNextWorkItem(ctx context.Context) return true } +// QueueForInformers is equivalent to calling QueueForInformersWithMaxDepth with maxDepth of -1. +// See QueueForInformersWithMaxDepth for more details. func (c *genericWatchingController[T]) QueueForInformers(resyncDuration time.Duration, notifiers ...Notifier) error { + return c.QueueForInformersWithMaxDepth(resyncDuration, -1, notifiers...) +} + +// QueueForInformersWithMaxDepth adds event handlers to the notifiers for the controller with a given max depth. +// maxDepth is the maximum number of parent hops to traverse when searching for a resourceID whose type is c.resourceType. Each +// walk to Parent consumes one level. +// maxDepth 0 means only the resourceID itself is considered. +// maxDepth -1 (or any negative value) means no limit. The parent walk continues until a match or nil parent is reached. +// It is exposed so that individual controllers can add other items to requeue based on easily. +func (c *genericWatchingController[T]) QueueForInformersWithMaxDepth(resyncDuration time.Duration, maxDepth int, notifiers ...Notifier) error { errs := []error{} for _, notifier := range notifiers { _, err := notifier.AddEventHandlerWithOptions( cache.ResourceEventHandlerFuncs{ - AddFunc: c.EnqueueCosmosAdd, - UpdateFunc: c.EnqueueCosmosUpdate, + AddFunc: c.enqueueCosmosAddFunc(maxDepth), + UpdateFunc: c.enqueueCosmosUpdateFunc(maxDepth), }, cache.HandlerOptions{ ResyncPeriod: ptr.To(resyncDuration), @@ -159,14 +171,34 @@ func (c *genericWatchingController[T]) QueueForInformers(resyncDuration time.Dur return errors.Join(errs...) } -// EnqueueResourceIDAdd traverses to find a resourceID that is an hcpcluster and adds it if found. +// enqueueResourceIDAdd is equivalent to calling EnqueueResourceIDAddWithMaxDepth with a maxDepth of -1. +// See EnqueueResourceIDAddWithMaxDepth for more details. // It is exposed so that individual controllers can add other items to requeue based on easily. func (c *genericWatchingController[T]) EnqueueResourceIDAdd(resourceID *azcorearm.ResourceID, changed bool) { + c.EnqueueResourceIDAddWithMaxDepth(resourceID, changed, -1) +} + +// enqueueResourceIDAddWithMaxDepth traverses resourceID and its parents according to maxDepth until it +// finds a resourceID that is of the resource type of c.resourceType and adds it if found. Each walk to Parent consumes one level. +// maxDepth is the maximum number of parent hops to traverse when searching for a resourceID of type c.resourceType. +// maxDepth 0 means only the resourceID itself is considered. +// maxDepth -1 (or any negative value) means no limit. The parent walk continues until a match or nil parent is reached. +// It is exposed so that individual controllers can add other items to requeue based on easily. +// When there's a match of resourceType: when changed is true, the resourceID is added to the queue immediately. Otherwise, the resourceID is +// added to the queue only if the cooldown checker allows it. +func (c *genericWatchingController[T]) EnqueueResourceIDAddWithMaxDepth(resourceID *azcorearm.ResourceID, changed bool, maxDepth int) { if resourceID == nil { return } if !armhelpers.ResourceTypeEqual(resourceID.ResourceType, c.resourceType) { - c.EnqueueResourceIDAdd(resourceID.Parent, changed) + if maxDepth == 0 { + return + } + nextDepth := maxDepth + if maxDepth > 0 { + nextDepth = maxDepth - 1 + } + c.EnqueueResourceIDAddWithMaxDepth(resourceID.Parent, changed, nextDepth) return } @@ -191,11 +223,23 @@ func (c *genericWatchingController[T]) EnqueueResourceIDAdd(resourceID *azcorear c.queue.Add(key) } -func (c *genericWatchingController[T]) EnqueueCosmosAdd(newObj any) { - c.EnqueueResourceIDAdd(newObj.(arm.CosmosPersistable).GetCosmosData().GetResourceID(), true) +func (c *genericWatchingController[T]) enqueueCosmosAddFunc(maxDepth int) func(any) { + return func(newObj any) { + c.enqueueCosmosAddWithMaxDepth(newObj, maxDepth) + } +} + +func (c *genericWatchingController[T]) enqueueCosmosAddWithMaxDepth(newObj any, maxDepth int) { + c.EnqueueResourceIDAddWithMaxDepth(newObj.(arm.CosmosPersistable).GetCosmosData().GetResourceID(), true, maxDepth) +} + +func (c *genericWatchingController[T]) enqueueCosmosUpdateFunc(maxDepth int) func(any, any) { + return func(oldObj, newObj any) { + c.enqueueCosmosUpdateWithMaxDepth(oldObj, newObj, maxDepth) + } } -func (c *genericWatchingController[T]) EnqueueCosmosUpdate(oldObj, newObj any) { +func (c *genericWatchingController[T]) enqueueCosmosUpdateWithMaxDepth(oldObj, newObj any, maxDepth int) { changed := oldObj.(arm.CosmosPersistable).GetCosmosData().GetEtag() != newObj.(arm.CosmosPersistable).GetCosmosData().GetEtag() - c.EnqueueResourceIDAdd(newObj.(arm.CosmosPersistable).GetCosmosData().GetResourceID(), changed) + c.EnqueueResourceIDAddWithMaxDepth(newObj.(arm.CosmosPersistable).GetCosmosData().GetResourceID(), changed, maxDepth) } diff --git a/backend/pkg/controllers/controllerutils/generic_watching_controller_test.go b/backend/pkg/controllers/controllerutils/generic_watching_controller_test.go new file mode 100644 index 00000000000..e12b7d6ead3 --- /dev/null +++ b/backend/pkg/controllers/controllerutils/generic_watching_controller_test.go @@ -0,0 +1,284 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllerutils + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "k8s.io/client-go/tools/cache" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + + "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/api/arm" +) + +const ( + testClusterARMID = "/subscriptions/sub1/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/c1" + testNodePoolARMID = testClusterARMID + "/nodePools/np1" +) + +type stringSyncer struct { + cooldown CooldownChecker +} + +func (s *stringSyncer) MakeKey(rid *azcorearm.ResourceID) string { + if rid == nil { + return "" + } + return rid.String() +} + +func (s *stringSyncer) SyncOnce(context.Context, string) error { return nil } + +func (s *stringSyncer) CooldownChecker() CooldownChecker { + if s.cooldown == nil { + return alwaysAllowCooldown{} + } + return s.cooldown +} + +type alwaysAllowCooldown struct{} + +func (alwaysAllowCooldown) CanSync(context.Context, any) bool { return true } + +type neverAllowCooldown struct{} + +func (neverAllowCooldown) CanSync(context.Context, any) bool { return false } + +func newTestWatchingController() (*genericWatchingController[string], *azcorearm.ResourceID, *azcorearm.ResourceID) { + clusterID := api.Must(azcorearm.ParseResourceID(testClusterARMID)) + npID := api.Must(azcorearm.ParseResourceID(testNodePoolARMID)) + syncer := &stringSyncer{} + c := newGenericWatchingController("test", clusterID.ResourceType, syncer) + return c, clusterID, npID +} + +func popAllQueue(c *genericWatchingController[string]) []string { + var keys []string + for c.queue.Len() > 0 { + k, shutdown := c.queue.Get() + if shutdown { + panic("queue shut down unexpectedly") + } + c.queue.Done(k) + c.queue.Forget(k) + keys = append(keys, k) + } + return keys +} + +func TestEnqueueResourceIDAddWithMaxDepth(t *testing.T) { + clusterID := api.Must(azcorearm.ParseResourceID(testClusterARMID)) + npID := api.Must(azcorearm.ParseResourceID(testNodePoolARMID)) + + tests := []struct { + name string + resource *azcorearm.ResourceID + changed bool + maxDepth int + wantKeys []string + }{ + { + name: "nil resource", + changed: true, + maxDepth: -1, + wantKeys: nil, + }, + { + name: "direct cluster match changed", + resource: clusterID, + changed: true, + maxDepth: 0, + wantKeys: []string{clusterID.String()}, + }, + { + name: "node pool maxDepth 0 does not walk parent", + resource: npID, + changed: true, + maxDepth: 0, + wantKeys: nil, + }, + { + name: "node pool maxDepth 1 enqueues cluster", + resource: npID, + changed: true, + maxDepth: 1, + wantKeys: []string{clusterID.String()}, + }, + { + name: "node pool negative maxDepth enqueues cluster", + resource: npID, + changed: true, + maxDepth: -1, + wantKeys: []string{clusterID.String()}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, _, _ := newTestWatchingController() + c.EnqueueResourceIDAddWithMaxDepth(tt.resource, tt.changed, tt.maxDepth) + got := popAllQueue(c) + require.Equal(t, tt.wantKeys, got) + }) + } +} + +func TestEnqueueResourceIDAddWithMaxDepth_changedAndCooldown(t *testing.T) { + clusterID := api.Must(azcorearm.ParseResourceID(testClusterARMID)) + syncer := &stringSyncer{cooldown: neverAllowCooldown{}} + c := newGenericWatchingController("cooldown", clusterID.ResourceType, syncer) + + tests := []struct { + name string + changed bool + wantNumElems int + }{ + {name: "unchanged skipped when CanSync false", changed: false, wantNumElems: 0}, + {name: "changed enqueues despite cooldown", changed: true, wantNumElems: 1}, + } + + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + c.EnqueueResourceIDAddWithMaxDepth(clusterID, testCase.changed, -1) + got := popAllQueue(c) + require.Len(t, got, testCase.wantNumElems) + if testCase.wantNumElems == 1 { + require.Equal(t, clusterID.String(), got[0]) + } + }) + } +} + +func TestEnqueueCosmosWithMaxDepth(t *testing.T) { + _, clusterID, npID := newTestWatchingController() + + tests := []struct { + name string + run func(*genericWatchingController[string]) + want []string + }{ + { + name: "add from node pool metadata", + run: func(c *genericWatchingController[string]) { + c.enqueueCosmosAddWithMaxDepth(&arm.CosmosMetadata{ResourceID: npID}, 1) + }, + want: []string{clusterID.String()}, + }, + { + name: "update same etag uses unchanged path", + run: func(c *genericWatchingController[string]) { + etag := azcore.ETag("e1") + oldObj := &arm.CosmosMetadata{ResourceID: npID, CosmosETag: etag} + newObj := &arm.CosmosMetadata{ResourceID: npID, CosmosETag: etag} + c.enqueueCosmosUpdateWithMaxDepth(oldObj, newObj, 1) + }, + want: []string{clusterID.String()}, + }, + { + name: "update different etag", + run: func(c *genericWatchingController[string]) { + oldObj := &arm.CosmosMetadata{ResourceID: npID, CosmosETag: azcore.ETag("a")} + newObj := &arm.CosmosMetadata{ResourceID: npID, CosmosETag: azcore.ETag("b")} + c.enqueueCosmosUpdateWithMaxDepth(oldObj, newObj, 1) + }, + want: []string{clusterID.String()}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, _, _ := newTestWatchingController() + tt.run(c) + require.Equal(t, tt.want, popAllQueue(c)) + }) + } +} + +type capturingNotifier struct { + addFunc func(any) + updateFunc func(any, any) +} + +func (n *capturingNotifier) AddEventHandlerWithOptions(handler cache.ResourceEventHandler, opts cache.HandlerOptions) (cache.ResourceEventHandlerRegistration, error) { + f, ok := handler.(cache.ResourceEventHandlerFuncs) + if !ok { + return nil, fmt.Errorf("expected ResourceEventHandlerFuncs, got %T", handler) + } + n.addFunc = f.AddFunc + n.updateFunc = f.UpdateFunc + return nil, nil +} + +func TestQueueForInformersWithMaxDepth(t *testing.T) { + _, clusterID, npID := newTestWatchingController() + + tests := []struct { + name string + run func(t *testing.T, c *genericWatchingController[string], n *capturingNotifier) + }{ + { + name: "Add handler respects maxDepth", + run: func(t *testing.T, c *genericWatchingController[string], n *capturingNotifier) { + require.NotNil(t, n.addFunc) + n.addFunc(&arm.CosmosMetadata{ResourceID: npID}) + require.Equal(t, []string{clusterID.String()}, popAllQueue(c)) + }, + }, + { + name: "Update handler respects maxDepth", + run: func(t *testing.T, c *genericWatchingController[string], n *capturingNotifier) { + require.NotNil(t, n.updateFunc) + oldObj := &arm.CosmosMetadata{ResourceID: npID, CosmosETag: azcore.ETag("a")} + newObj := &arm.CosmosMetadata{ResourceID: npID, CosmosETag: azcore.ETag("b")} + n.updateFunc(oldObj, newObj) + require.Equal(t, []string{clusterID.String()}, popAllQueue(c)) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, _, _ := newTestWatchingController() + n := &capturingNotifier{} + require.NoError(t, c.QueueForInformersWithMaxDepth(time.Minute, 1, n)) + tt.run(t, c, n) + }) + } + + t.Run("joins notifier errors", func(t *testing.T) { + c, _, _ := newTestWatchingController() + bad := &errNotifier{err: errors.New("register failed")} + err := c.QueueForInformersWithMaxDepth(time.Minute, -1, bad) + require.Error(t, err) + require.ErrorIs(t, err, bad.err) + }) +} + +type errNotifier struct { + err error +} + +func (e *errNotifier) AddEventHandlerWithOptions(handler cache.ResourceEventHandler, opts cache.HandlerOptions) (cache.ResourceEventHandlerRegistration, error) { + return nil, e.err +} diff --git a/backend/pkg/controllers/controllerutils/nodepool_watching_controller.go b/backend/pkg/controllers/controllerutils/nodepool_watching_controller.go index 139426d9f2e..e4bd5bd78b2 100644 --- a/backend/pkg/controllers/controllerutils/nodepool_watching_controller.go +++ b/backend/pkg/controllers/controllerutils/nodepool_watching_controller.go @@ -63,8 +63,14 @@ func NewNodePoolWatchingController( if informers != nil { nodePoolInformer, _ := informers.NodePools() serviceProviderNodePoolInformer, _ := informers.ServiceProviderNodePools() + err := nodePoolController.QueueForInformers(resyncDuration, nodePoolInformer, serviceProviderNodePoolInformer) + if err != nil { + panic(err) // coding error + } + managementClusterContentInformer, _ := informers.ManagementClusterContents() - err := nodePoolController.QueueForInformers(resyncDuration, nodePoolInformer, serviceProviderNodePoolInformer, managementClusterContentInformer) + // Limit the max depth of ManagementClusterContent to 1 to only consider the nodepool-scoped ManagementClusterContents + err = nodePoolController.QueueForInformersWithMaxDepth(resyncDuration, 1, managementClusterContentInformer) if err != nil { panic(err) // coding error } From c1582611821ccf15376e682b51cf1604774e6288 Mon Sep 17 00:00:00 2001 From: Miguel Soriano Date: Fri, 17 Apr 2026 13:19:35 +0200 Subject: [PATCH 15/16] feat: update maestro delete orphan bundle to skip cosmos resources that don't have CS ID We do not want to make the whole sync fail when there are some cosmos resources that do not have the ClusterServiceID attribute set yet. This could be a common occurrence when there's enough amount of creation activity of resources. It should be safe to skip those because if a maestro bundle in the maestro API exists it means that there should be a corresponding Cosmos resource with a maestro bundle reference. If for some reason during the orphan calculation inbetween the first read of cosmos resources and the read in the Maestro api there's a new bundle in maestro, the second read of cosmos resources will catch that and prevent accidental deletion. --- ...ned_maestro_readonly_bundles_controller.go | 67 +++++++++++++------ ...aestro_readonly_bundles_controller_test.go | 37 ++++++++++ 2 files changed, 83 insertions(+), 21 deletions(-) diff --git a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go index 4676d810e33..fc2ea05c54e 100644 --- a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go @@ -218,13 +218,22 @@ func (c *deleteOrphanedMaestroReadonlyBundles) buildMaestroClientsByProvisionSha // mapServiceProviderClustersByProvisionShard groups ServiceProviderClusters by Cluster Service provision shard ID. // Every resolved shard must exist in maestroClientsByShard (registered provision shards). +// ServiceProviderClusters whose parent cluster has no ClusterServiceID yet (pre–Cluster Service registration) are skipped +// so the syncer doesn't fail if there are some resources that still don't have it set func (c *deleteOrphanedMaestroReadonlyBundles) mapServiceProviderClustersByProvisionShard(ctx context.Context, spcs []*api.ServiceProviderCluster, maestroClientsByShard map[string]*shardMaestroClient) (map[string][]*api.ServiceProviderCluster, error) { res := make(map[string][]*api.ServiceProviderCluster) for _, spc := range spcs { - shardID, err := c.clusterProvisionShardIDForServiceProviderCluster(ctx, spc) + shardID, skip, err := c.clusterProvisionShardIDForServiceProviderCluster(ctx, spc) if err != nil { return nil, err } + if skip { + // It should be safe to skip those because if a maestro bundle in the maestro API exists it means that there should be a corresponding Cosmos + // resource with a maestro bundle reference. If for some reason during the orphan calculation inbetween the first read of cosmos + // resources and the read in the Maestro api there's a new bundle in maestro, the second read of cosmos resources will catch that + // and prevent accidental deletion. + continue + } if _, ok := maestroClientsByShard[shardID]; !ok { return nil, utils.TrackError(fmt.Errorf("provision shard %s for ServiceProviderCluster %s is not present in provision shards map", shardID, spc.ResourceID.String())) } @@ -235,13 +244,22 @@ func (c *deleteOrphanedMaestroReadonlyBundles) mapServiceProviderClustersByProvi // mapServiceProviderNodePoolsByProvisionShard groups ServiceProviderNodePools by Cluster Service provision shard ID // of their owning cluster. Every resolved shard must exist in maestroClientsByShard (registered provision shards). +// ServiceProviderNodePools whose parent cluster has no ClusterServiceID yet (pre–Cluster Service registration) are skipped +// so the syncer doesn't fail if there are some resources that still don't have it set. func (c *deleteOrphanedMaestroReadonlyBundles) mapServiceProviderNodePoolsByProvisionShard(ctx context.Context, spnps []*api.ServiceProviderNodePool, maestroClientsByShard map[string]*shardMaestroClient) (map[string][]*api.ServiceProviderNodePool, error) { res := make(map[string][]*api.ServiceProviderNodePool) for _, spnp := range spnps { - shardID, err := c.clusterProvisionShardIDForServiceProviderNodePool(ctx, spnp) + shardID, skip, err := c.clusterProvisionShardIDForServiceProviderNodePool(ctx, spnp) if err != nil { return nil, err } + if skip { + // It should be safe to skip those because if a maestro bundle in the maestro API exists it means that there should be a corresponding Cosmos + // resource with a maestro bundle reference. If for some reason during the orphan calculation inbetween the first read of cosmos + // resources and the read in the Maestro api there's a new bundle in maestro, the second read of cosmos resources will catch that + // and prevent accidental deletion. + continue + } if _, ok := maestroClientsByShard[shardID]; !ok { return nil, utils.TrackError(fmt.Errorf("provision shard %s for ServiceProviderNodePool %s is not present in provision shards map", shardID, spnp.ResourceID.String())) } @@ -426,46 +444,53 @@ func (c *deleteOrphanedMaestroReadonlyBundles) conditionallyDeleteOrphanReadonly } // clusterProvisionShardIDForServiceProviderCluster returns the Cluster Service provision shard ID for the cluster that owns the SPC. -func (c *deleteOrphanedMaestroReadonlyBundles) clusterProvisionShardIDForServiceProviderCluster(ctx context.Context, spc *api.ServiceProviderCluster) (string, error) { +// skip is true when the parent cluster has no ClusterServiceID yet. +func (c *deleteOrphanedMaestroReadonlyBundles) clusterProvisionShardIDForServiceProviderCluster(ctx context.Context, spc *api.ServiceProviderCluster) (shardID string, skip bool, err error) { clusterResourceID := spc.ResourceID.Parent if clusterResourceID == nil { - return "", utils.TrackError(fmt.Errorf("ServiceProviderCluster %s has no parent resource ID", spc.ResourceID.String())) + return "", false, utils.TrackError(fmt.Errorf("ServiceProviderCluster %s has no parent resource ID", spc.ResourceID.String())) } cluster, err := c.cosmosClient.HCPClusters(clusterResourceID.SubscriptionID, clusterResourceID.ResourceGroupName).Get(ctx, clusterResourceID.Name) if err != nil { - return "", utils.TrackError(fmt.Errorf("failed to get Cluster: %w", err)) - } - // TODO We get the provision shard ID from CS but at some point we should have - // the information in Cosmos and this should be changed to use that instead. - // TODO should we take into account that at some point in the future we will implement migration between management - // clusters, where a cluster could have bundles allocated to different provision shards at the same time? For now - // we assume that the cluster is associated to a single provision shard at a time. - clusterCSShard, err := c.clusterServiceClient.GetClusterProvisionShard(ctx, cluster.ServiceProviderProperties.ClusterServiceID) - if err != nil { - return "", utils.TrackError(fmt.Errorf("failed to get Cluster Provision Shard: %w", err)) + return "", false, utils.TrackError(fmt.Errorf("failed to get Cluster: %w", err)) } - return clusterCSShard.ID(), nil + return c.provisionShardIDFromCluster(ctx, cluster) } // clusterProvisionShardIDForServiceProviderNodePool returns the Cluster Service provision shard ID for the cluster that owns the node pool. -func (c *deleteOrphanedMaestroReadonlyBundles) clusterProvisionShardIDForServiceProviderNodePool(ctx context.Context, spnp *api.ServiceProviderNodePool) (string, error) { +// skip is true when the parent cluster has no ClusterServiceID yet. +func (c *deleteOrphanedMaestroReadonlyBundles) clusterProvisionShardIDForServiceProviderNodePool(ctx context.Context, spnp *api.ServiceProviderNodePool) (shardID string, skip bool, err error) { nodePoolResourceID := spnp.ResourceID.Parent if nodePoolResourceID == nil { - return "", utils.TrackError(fmt.Errorf("ServiceProviderNodePool %s has no parent resource ID", spnp.ResourceID.String())) + return "", false, utils.TrackError(fmt.Errorf("ServiceProviderNodePool %s has no parent resource ID", spnp.ResourceID.String())) } clusterResourceID := nodePoolResourceID.Parent if clusterResourceID == nil { - return "", utils.TrackError(fmt.Errorf("ServiceProviderNodePool %s has no grandparent cluster resource ID", spnp.ResourceID.String())) + return "", false, utils.TrackError(fmt.Errorf("ServiceProviderNodePool %s has no grandparent cluster resource ID", spnp.ResourceID.String())) } cluster, err := c.cosmosClient.HCPClusters(clusterResourceID.SubscriptionID, clusterResourceID.ResourceGroupName).Get(ctx, clusterResourceID.Name) if err != nil { - return "", utils.TrackError(fmt.Errorf("failed to get Cluster: %w", err)) + return "", false, utils.TrackError(fmt.Errorf("failed to get Cluster: %w", err)) + } + return c.provisionShardIDFromCluster(ctx, cluster) +} + +// provisionShardIDFromCluster resolves the provision shard for a Cosmos cluster document. skip is true when ClusterServiceID +// is unset so the cluster is not yet registered with Cluster Service (same gate as create-*-scoped Maestro bundle controllers). +func (c *deleteOrphanedMaestroReadonlyBundles) provisionShardIDFromCluster(ctx context.Context, cluster *api.HCPOpenShiftCluster) (shardID string, skip bool, err error) { + if len(cluster.ServiceProviderProperties.ClusterServiceID.String()) == 0 { + return "", true, nil } + // TODO We get the provision shard ID from CS but at some point we should have + // the information in Cosmos and this should be changed to use that instead. + // TODO should we take into account that at some point in the future we will implement migration between management + // clusters, where a cluster could have bundles allocated to different provision shards at the same time? For now + // we assume that the cluster is associated to a single provision shard at a time. clusterCSShard, err := c.clusterServiceClient.GetClusterProvisionShard(ctx, cluster.ServiceProviderProperties.ClusterServiceID) if err != nil { - return "", utils.TrackError(fmt.Errorf("failed to get Cluster Provision Shard: %w", err)) + return "", false, utils.TrackError(fmt.Errorf("failed to get Cluster Provision Shard: %w", err)) } - return clusterCSShard.ID(), nil + return clusterCSShard.ID(), false, nil } // buildClusterScopedMaestroAPIMaestroBundleNamesByShard maps provision shard ID to the set of Maestro API bundle names referenced by diff --git a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller_test.go b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller_test.go index f6938067a31..d94d92ec828 100644 --- a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller_test.go +++ b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller_test.go @@ -947,6 +947,43 @@ func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderClustersByProvis } } +func TestDeleteOrphanedMaestroReadonlyBundles_provisionShardIDFromCluster(t *testing.T) { + ctx := context.Background() + + t.Run("skip when ClusterServiceID is empty", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + c := &deleteOrphanedMaestroReadonlyBundles{clusterServiceClient: mockCS} + cluster := &api.HCPOpenShiftCluster{ + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.InternalID{}, + }, + } + shardID, skip, err := c.provisionShardIDFromCluster(ctx, cluster) + require.NoError(t, err) + assert.True(t, skip) + assert.Empty(t, shardID) + }) + + t.Run("returns shard ID when ClusterServiceID is set", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + c := &deleteOrphanedMaestroReadonlyBundles{clusterServiceClient: mockCS} + csID := api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid")) + cluster := &api.HCPOpenShiftCluster{ + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: csID, + }, + } + provisionShard := buildTestProvisionShard("consumer") + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), csID).Return(provisionShard, nil) + shardID, skip, err := c.provisionShardIDFromCluster(ctx, cluster) + require.NoError(t, err) + assert.False(t, skip) + assert.Equal(t, provisionShard.ID(), shardID) + }) +} + func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderNodePoolsByProvisionShard(t *testing.T) { ctx := context.Background() // mapServiceProviderNodePoolsByProvisionShard does not use the Maestro client; tests only need shard IDs in the map. From f6ed2d92c3031e65888f81d9423e89db816cc046 Mon Sep 17 00:00:00 2001 From: Miguel Soriano Date: Fri, 17 Apr 2026 13:51:23 +0200 Subject: [PATCH 16/16] feat: skip processing when no CS ID is set for create and read nodepool readonly maestro bundles When we move CS creation of resources to the backend the CS ID might not be set yet once the controllers run so we skip their processing. --- ...ped_maestro_readonly_bundles_controller.go | 4 ++ ...aestro_readonly_bundles_controller_test.go | 63 +++++++++++++++++++ ...tro_readonly_bundles_content_controller.go | 4 ++ ...eadonly_bundles_content_controller_test.go | 56 +++++++++++++++++ 4 files changed, 127 insertions(+) diff --git a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go index 4465d11424e..0c82e1a0b80 100644 --- a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go @@ -107,6 +107,10 @@ func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) SyncOnce(ctx context. if err != nil { return utils.TrackError(fmt.Errorf("failed to get NodePool: %w", err)) } + if len(existingNodePool.ServiceProviderProperties.ClusterServiceID.String()) == 0 { + // TODO remove this once we have the information all in cosmos. + return nil + } existingServiceProviderNodePool, err := database.GetOrCreateServiceProviderNodePool(ctx, c.cosmosClient, key.GetResourceID()) if err != nil { diff --git a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go index 305114ebc07..db6b10b37c7 100644 --- a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go +++ b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go @@ -496,6 +496,69 @@ func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_NodePoolNotFo assert.NoError(t, err) } +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_EmptyClusterServiceID(t *testing.T) { + ctrl := gomock.NewController(t) + ctx := context.Background() + + mockDBClient := databasetesting.NewMockDBClient() + mockClusterService := ocm.NewMockClusterServiceClientSpec(ctrl) + + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + clusterServiceClient: mockClusterService, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.InternalID{}, + }, + } + nodepoolsCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{}, + }, + } + spnpCRUD := mockDBClient.ServiceProviderNodePools(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName, key.HCPNodePoolName) + _, err = spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + + // Cluster service ID not yet populated: skip sync (no OCM / Maestro calls), even though a bundle still needs syncing. + err = syncer.SyncOnce(ctx, key) + assert.NoError(t, err) + + // SPNP should be unchanged (sync did not run). + got, err := spnpCRUD.Get(ctx, api.ServiceProviderNodePoolResourceName) + require.NoError(t, err) + ref, err := got.Status.MaestroReadonlyBundles.Get(bundleInternalName) + require.NoError(t, err) + assert.Nil(t, ref, "Maestro bundle ref should not be created when ClusterServiceID is empty") +} + func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_GetServiceProviderNodePoolError(t *testing.T) { ctx := context.Background() diff --git a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go index 03e03deff34..a1d49e6694f 100644 --- a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go +++ b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go @@ -90,6 +90,10 @@ func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) SyncOn if err != nil { return utils.TrackError(fmt.Errorf("failed to get NodePool: %w", err)) } + if len(existingNodePool.ServiceProviderProperties.ClusterServiceID.String()) == 0 { + // TODO remove this once we have the information all in cosmos. + return nil + } existingServiceProviderNodePool, err := database.GetOrCreateServiceProviderNodePool(ctx, c.cosmosClient, key.GetResourceID()) if err != nil { diff --git a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go index 51d7b9ce4d9..0b542a9fe76 100644 --- a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go +++ b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go @@ -84,6 +84,62 @@ func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnc assert.NoError(t, err) } +func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnce_EmptyClusterServiceID(t *testing.T) { + ctrl := gomock.NewController(t) + ctx := context.Background() + + mockDBClient := databasetesting.NewMockDBClient() + mockClusterService := ocm.NewMockClusterServiceClientSpec(ctrl) + + syncer := &readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + clusterServiceClient: mockClusterService, + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.InternalID{}, + }, + } + nodepoolsCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + {Name: bundleInternalName, MaestroAPIMaestroBundleName: "bundle-name"}, + }, + }, + } + spnpCRUD := mockDBClient.ServiceProviderNodePools(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName, key.HCPNodePoolName) + _, err = spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + + // Cluster service ID not yet populated: skip sync (no OCM / Maestro calls). + err = syncer.SyncOnce(ctx, key) + assert.NoError(t, err) +} + func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnce_GetServiceProviderNodePoolError(t *testing.T) { ctx := context.Background()