diff --git a/backend/pkg/app/backend.go b/backend/pkg/app/backend.go index fd56c2cc880..3841e4ba6fb 100644 --- a/backend/pkg/app/backend.go +++ b/backend/pkg/app/backend.go @@ -495,14 +495,25 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, activeOperationLister, backendInformers, ) - maestroCreateReadonlyBundlesController := controllers.NewCreateClusterScopedMaestroReadonlyBundlesController( + + maestroCreateClusterScopedReadonlyBundlesController := controllers.NewCreateClusterScopedMaestroReadonlyBundlesController( activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient, backendInformers, b.options.MaestroSourceEnvironmentIdentifier, maestroClientBuilder, ) - maestroReadAndPersistReadonlyBundlesContentController := controllers.NewReadAndPersistClusterScopedMaestroReadonlyBundlesContentController( + maestroReadAndPersistClusterScopedReadonlyBundlesContentController := controllers.NewReadAndPersistClusterScopedMaestroReadonlyBundlesContentController( + activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient, + backendInformers, b.options.MaestroSourceEnvironmentIdentifier, maestroClientBuilder, + ) + + maestroCreateNodePoolScopedReadonlyBundlesController := controllers.NewCreateNodePoolScopedMaestroReadonlyBundlesController( activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient, backendInformers, b.options.MaestroSourceEnvironmentIdentifier, maestroClientBuilder, ) + maestroReadAndPersistNodePoolScopedReadonlyBundlesContentController := controllers.NewReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentController( + activeOperationLister, b.options.CosmosDBClient, b.options.ClustersServiceClient, + backendInformers, b.options.MaestroSourceEnvironmentIdentifier, maestroClientBuilder, + ) + maestroDeleteOrphanedReadonlyBundlesController := controllers.NewDeleteOrphanedMaestroReadonlyBundlesController( b.options.CosmosDBClient, b.options.ClustersServiceClient, @@ -609,8 +620,10 @@ func (b *Backend) runBackendControllersUnderLeaderElection(ctx context.Context, go azureClusterResourceGroupExistenceValidationController.Run(ctx, 20) go azureClusterManagedIdentitiesExistenceValidationController.Run(ctx, 20) go nodePoolVersionController.Run(ctx, 20) - go maestroCreateReadonlyBundlesController.Run(ctx, 20) - go maestroReadAndPersistReadonlyBundlesContentController.Run(ctx, 20) + go maestroCreateClusterScopedReadonlyBundlesController.Run(ctx, 20) + go maestroReadAndPersistClusterScopedReadonlyBundlesContentController.Run(ctx, 20) + go maestroCreateNodePoolScopedReadonlyBundlesController.Run(ctx, 20) + go maestroReadAndPersistNodePoolScopedReadonlyBundlesContentController.Run(ctx, 20) go maestroDeleteOrphanedReadonlyBundlesController.Run(ctx, 20) go triggerNodePoolUpgradeController.Run(ctx, 20) go nodePoolPropertiesSyncController.Run(ctx, 20) diff --git a/backend/pkg/controllers/controllerutils/cluster_watching_controller.go b/backend/pkg/controllers/controllerutils/cluster_watching_controller.go index faeea603ef9..19d8c2a9119 100644 --- a/backend/pkg/controllers/controllerutils/cluster_watching_controller.go +++ b/backend/pkg/controllers/controllerutils/cluster_watching_controller.go @@ -64,8 +64,13 @@ func NewClusterWatchingController( if informers != nil { clusterInformer, _ := informers.Clusters() serviceProviderInformer, _ := informers.ServiceProviderClusters() + err := clusterController.QueueForInformers(resyncDuration, clusterInformer, serviceProviderInformer) + if err != nil { + panic(err) // coding error + } managementClusterContentInformer, _ := informers.ManagementClusterContents() - err := clusterController.QueueForInformers(resyncDuration, clusterInformer, serviceProviderInformer, managementClusterContentInformer) + // Limit the max depth of ManagementClusterContent to 1 to only consider the cluster-scoped ManagementClusterContents + err = clusterController.QueueForInformersWithMaxDepth(resyncDuration, 1, managementClusterContentInformer) if err != nil { panic(err) // coding error } diff --git a/backend/pkg/controllers/controllerutils/generic_watching_controller.go b/backend/pkg/controllers/controllerutils/generic_watching_controller.go index d55a98ccfdb..fa8a290133c 100644 --- a/backend/pkg/controllers/controllerutils/generic_watching_controller.go +++ b/backend/pkg/controllers/controllerutils/generic_watching_controller.go @@ -143,13 +143,25 @@ func (c *genericWatchingController[T]) processNextWorkItem(ctx context.Context) return true } +// QueueForInformers is equivalent to calling QueueForInformersWithMaxDepth with maxDepth of -1. +// See QueueForInformersWithMaxDepth for more details. func (c *genericWatchingController[T]) QueueForInformers(resyncDuration time.Duration, notifiers ...Notifier) error { + return c.QueueForInformersWithMaxDepth(resyncDuration, -1, notifiers...) +} + +// QueueForInformersWithMaxDepth adds event handlers to the notifiers for the controller with a given max depth. +// maxDepth is the maximum number of parent hops to traverse when searching for a resourceID whose type is c.resourceType. Each +// walk to Parent consumes one level. +// maxDepth 0 means only the resourceID itself is considered. +// maxDepth -1 (or any negative value) means no limit. The parent walk continues until a match or nil parent is reached. +// It is exposed so that individual controllers can add other items to requeue based on easily. +func (c *genericWatchingController[T]) QueueForInformersWithMaxDepth(resyncDuration time.Duration, maxDepth int, notifiers ...Notifier) error { errs := []error{} for _, notifier := range notifiers { _, err := notifier.AddEventHandlerWithOptions( cache.ResourceEventHandlerFuncs{ - AddFunc: c.EnqueueCosmosAdd, - UpdateFunc: c.EnqueueCosmosUpdate, + AddFunc: c.enqueueCosmosAddFunc(maxDepth), + UpdateFunc: c.enqueueCosmosUpdateFunc(maxDepth), }, cache.HandlerOptions{ ResyncPeriod: ptr.To(resyncDuration), @@ -159,14 +171,34 @@ func (c *genericWatchingController[T]) QueueForInformers(resyncDuration time.Dur return errors.Join(errs...) } -// EnqueueResourceIDAdd traverses to find a resourceID that is an hcpcluster and adds it if found. +// enqueueResourceIDAdd is equivalent to calling EnqueueResourceIDAddWithMaxDepth with a maxDepth of -1. +// See EnqueueResourceIDAddWithMaxDepth for more details. // It is exposed so that individual controllers can add other items to requeue based on easily. func (c *genericWatchingController[T]) EnqueueResourceIDAdd(resourceID *azcorearm.ResourceID, changed bool) { + c.EnqueueResourceIDAddWithMaxDepth(resourceID, changed, -1) +} + +// enqueueResourceIDAddWithMaxDepth traverses resourceID and its parents according to maxDepth until it +// finds a resourceID that is of the resource type of c.resourceType and adds it if found. Each walk to Parent consumes one level. +// maxDepth is the maximum number of parent hops to traverse when searching for a resourceID of type c.resourceType. +// maxDepth 0 means only the resourceID itself is considered. +// maxDepth -1 (or any negative value) means no limit. The parent walk continues until a match or nil parent is reached. +// It is exposed so that individual controllers can add other items to requeue based on easily. +// When there's a match of resourceType: when changed is true, the resourceID is added to the queue immediately. Otherwise, the resourceID is +// added to the queue only if the cooldown checker allows it. +func (c *genericWatchingController[T]) EnqueueResourceIDAddWithMaxDepth(resourceID *azcorearm.ResourceID, changed bool, maxDepth int) { if resourceID == nil { return } if !armhelpers.ResourceTypeEqual(resourceID.ResourceType, c.resourceType) { - c.EnqueueResourceIDAdd(resourceID.Parent, changed) + if maxDepth == 0 { + return + } + nextDepth := maxDepth + if maxDepth > 0 { + nextDepth = maxDepth - 1 + } + c.EnqueueResourceIDAddWithMaxDepth(resourceID.Parent, changed, nextDepth) return } @@ -191,11 +223,23 @@ func (c *genericWatchingController[T]) EnqueueResourceIDAdd(resourceID *azcorear c.queue.Add(key) } -func (c *genericWatchingController[T]) EnqueueCosmosAdd(newObj any) { - c.EnqueueResourceIDAdd(newObj.(arm.CosmosPersistable).GetCosmosData().GetResourceID(), true) +func (c *genericWatchingController[T]) enqueueCosmosAddFunc(maxDepth int) func(any) { + return func(newObj any) { + c.enqueueCosmosAddWithMaxDepth(newObj, maxDepth) + } +} + +func (c *genericWatchingController[T]) enqueueCosmosAddWithMaxDepth(newObj any, maxDepth int) { + c.EnqueueResourceIDAddWithMaxDepth(newObj.(arm.CosmosPersistable).GetCosmosData().GetResourceID(), true, maxDepth) +} + +func (c *genericWatchingController[T]) enqueueCosmosUpdateFunc(maxDepth int) func(any, any) { + return func(oldObj, newObj any) { + c.enqueueCosmosUpdateWithMaxDepth(oldObj, newObj, maxDepth) + } } -func (c *genericWatchingController[T]) EnqueueCosmosUpdate(oldObj, newObj any) { +func (c *genericWatchingController[T]) enqueueCosmosUpdateWithMaxDepth(oldObj, newObj any, maxDepth int) { changed := oldObj.(arm.CosmosPersistable).GetCosmosData().GetEtag() != newObj.(arm.CosmosPersistable).GetCosmosData().GetEtag() - c.EnqueueResourceIDAdd(newObj.(arm.CosmosPersistable).GetCosmosData().GetResourceID(), changed) + c.EnqueueResourceIDAddWithMaxDepth(newObj.(arm.CosmosPersistable).GetCosmosData().GetResourceID(), changed, maxDepth) } diff --git a/backend/pkg/controllers/controllerutils/generic_watching_controller_test.go b/backend/pkg/controllers/controllerutils/generic_watching_controller_test.go new file mode 100644 index 00000000000..e12b7d6ead3 --- /dev/null +++ b/backend/pkg/controllers/controllerutils/generic_watching_controller_test.go @@ -0,0 +1,284 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllerutils + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "k8s.io/client-go/tools/cache" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + + "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/api/arm" +) + +const ( + testClusterARMID = "/subscriptions/sub1/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/c1" + testNodePoolARMID = testClusterARMID + "/nodePools/np1" +) + +type stringSyncer struct { + cooldown CooldownChecker +} + +func (s *stringSyncer) MakeKey(rid *azcorearm.ResourceID) string { + if rid == nil { + return "" + } + return rid.String() +} + +func (s *stringSyncer) SyncOnce(context.Context, string) error { return nil } + +func (s *stringSyncer) CooldownChecker() CooldownChecker { + if s.cooldown == nil { + return alwaysAllowCooldown{} + } + return s.cooldown +} + +type alwaysAllowCooldown struct{} + +func (alwaysAllowCooldown) CanSync(context.Context, any) bool { return true } + +type neverAllowCooldown struct{} + +func (neverAllowCooldown) CanSync(context.Context, any) bool { return false } + +func newTestWatchingController() (*genericWatchingController[string], *azcorearm.ResourceID, *azcorearm.ResourceID) { + clusterID := api.Must(azcorearm.ParseResourceID(testClusterARMID)) + npID := api.Must(azcorearm.ParseResourceID(testNodePoolARMID)) + syncer := &stringSyncer{} + c := newGenericWatchingController("test", clusterID.ResourceType, syncer) + return c, clusterID, npID +} + +func popAllQueue(c *genericWatchingController[string]) []string { + var keys []string + for c.queue.Len() > 0 { + k, shutdown := c.queue.Get() + if shutdown { + panic("queue shut down unexpectedly") + } + c.queue.Done(k) + c.queue.Forget(k) + keys = append(keys, k) + } + return keys +} + +func TestEnqueueResourceIDAddWithMaxDepth(t *testing.T) { + clusterID := api.Must(azcorearm.ParseResourceID(testClusterARMID)) + npID := api.Must(azcorearm.ParseResourceID(testNodePoolARMID)) + + tests := []struct { + name string + resource *azcorearm.ResourceID + changed bool + maxDepth int + wantKeys []string + }{ + { + name: "nil resource", + changed: true, + maxDepth: -1, + wantKeys: nil, + }, + { + name: "direct cluster match changed", + resource: clusterID, + changed: true, + maxDepth: 0, + wantKeys: []string{clusterID.String()}, + }, + { + name: "node pool maxDepth 0 does not walk parent", + resource: npID, + changed: true, + maxDepth: 0, + wantKeys: nil, + }, + { + name: "node pool maxDepth 1 enqueues cluster", + resource: npID, + changed: true, + maxDepth: 1, + wantKeys: []string{clusterID.String()}, + }, + { + name: "node pool negative maxDepth enqueues cluster", + resource: npID, + changed: true, + maxDepth: -1, + wantKeys: []string{clusterID.String()}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, _, _ := newTestWatchingController() + c.EnqueueResourceIDAddWithMaxDepth(tt.resource, tt.changed, tt.maxDepth) + got := popAllQueue(c) + require.Equal(t, tt.wantKeys, got) + }) + } +} + +func TestEnqueueResourceIDAddWithMaxDepth_changedAndCooldown(t *testing.T) { + clusterID := api.Must(azcorearm.ParseResourceID(testClusterARMID)) + syncer := &stringSyncer{cooldown: neverAllowCooldown{}} + c := newGenericWatchingController("cooldown", clusterID.ResourceType, syncer) + + tests := []struct { + name string + changed bool + wantNumElems int + }{ + {name: "unchanged skipped when CanSync false", changed: false, wantNumElems: 0}, + {name: "changed enqueues despite cooldown", changed: true, wantNumElems: 1}, + } + + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + c.EnqueueResourceIDAddWithMaxDepth(clusterID, testCase.changed, -1) + got := popAllQueue(c) + require.Len(t, got, testCase.wantNumElems) + if testCase.wantNumElems == 1 { + require.Equal(t, clusterID.String(), got[0]) + } + }) + } +} + +func TestEnqueueCosmosWithMaxDepth(t *testing.T) { + _, clusterID, npID := newTestWatchingController() + + tests := []struct { + name string + run func(*genericWatchingController[string]) + want []string + }{ + { + name: "add from node pool metadata", + run: func(c *genericWatchingController[string]) { + c.enqueueCosmosAddWithMaxDepth(&arm.CosmosMetadata{ResourceID: npID}, 1) + }, + want: []string{clusterID.String()}, + }, + { + name: "update same etag uses unchanged path", + run: func(c *genericWatchingController[string]) { + etag := azcore.ETag("e1") + oldObj := &arm.CosmosMetadata{ResourceID: npID, CosmosETag: etag} + newObj := &arm.CosmosMetadata{ResourceID: npID, CosmosETag: etag} + c.enqueueCosmosUpdateWithMaxDepth(oldObj, newObj, 1) + }, + want: []string{clusterID.String()}, + }, + { + name: "update different etag", + run: func(c *genericWatchingController[string]) { + oldObj := &arm.CosmosMetadata{ResourceID: npID, CosmosETag: azcore.ETag("a")} + newObj := &arm.CosmosMetadata{ResourceID: npID, CosmosETag: azcore.ETag("b")} + c.enqueueCosmosUpdateWithMaxDepth(oldObj, newObj, 1) + }, + want: []string{clusterID.String()}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, _, _ := newTestWatchingController() + tt.run(c) + require.Equal(t, tt.want, popAllQueue(c)) + }) + } +} + +type capturingNotifier struct { + addFunc func(any) + updateFunc func(any, any) +} + +func (n *capturingNotifier) AddEventHandlerWithOptions(handler cache.ResourceEventHandler, opts cache.HandlerOptions) (cache.ResourceEventHandlerRegistration, error) { + f, ok := handler.(cache.ResourceEventHandlerFuncs) + if !ok { + return nil, fmt.Errorf("expected ResourceEventHandlerFuncs, got %T", handler) + } + n.addFunc = f.AddFunc + n.updateFunc = f.UpdateFunc + return nil, nil +} + +func TestQueueForInformersWithMaxDepth(t *testing.T) { + _, clusterID, npID := newTestWatchingController() + + tests := []struct { + name string + run func(t *testing.T, c *genericWatchingController[string], n *capturingNotifier) + }{ + { + name: "Add handler respects maxDepth", + run: func(t *testing.T, c *genericWatchingController[string], n *capturingNotifier) { + require.NotNil(t, n.addFunc) + n.addFunc(&arm.CosmosMetadata{ResourceID: npID}) + require.Equal(t, []string{clusterID.String()}, popAllQueue(c)) + }, + }, + { + name: "Update handler respects maxDepth", + run: func(t *testing.T, c *genericWatchingController[string], n *capturingNotifier) { + require.NotNil(t, n.updateFunc) + oldObj := &arm.CosmosMetadata{ResourceID: npID, CosmosETag: azcore.ETag("a")} + newObj := &arm.CosmosMetadata{ResourceID: npID, CosmosETag: azcore.ETag("b")} + n.updateFunc(oldObj, newObj) + require.Equal(t, []string{clusterID.String()}, popAllQueue(c)) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, _, _ := newTestWatchingController() + n := &capturingNotifier{} + require.NoError(t, c.QueueForInformersWithMaxDepth(time.Minute, 1, n)) + tt.run(t, c, n) + }) + } + + t.Run("joins notifier errors", func(t *testing.T) { + c, _, _ := newTestWatchingController() + bad := &errNotifier{err: errors.New("register failed")} + err := c.QueueForInformersWithMaxDepth(time.Minute, -1, bad) + require.Error(t, err) + require.ErrorIs(t, err, bad.err) + }) +} + +type errNotifier struct { + err error +} + +func (e *errNotifier) AddEventHandlerWithOptions(handler cache.ResourceEventHandler, opts cache.HandlerOptions) (cache.ResourceEventHandlerRegistration, error) { + return nil, e.err +} diff --git a/backend/pkg/controllers/controllerutils/management_cluster_content_controller.go b/backend/pkg/controllers/controllerutils/management_cluster_content_controller.go new file mode 100644 index 00000000000..dbaf7a09955 --- /dev/null +++ b/backend/pkg/controllers/controllerutils/management_cluster_content_controller.go @@ -0,0 +1,42 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllerutils + +import ( + "fmt" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + + "github.com/Azure/ARO-HCP/internal/api" +) + +// NewInitialManagementClusterContent returns a new ManagementClusterContent with +// the given full managementClusterContents ARM resource ID. +// The returned value can be used to consistently initialize a new ManagementClusterContent +func NewInitialManagementClusterContent(managementClusterContentResourceID *azcorearm.ResourceID) *api.ManagementClusterContent { + return &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ + ResourceID: managementClusterContentResourceID, + }, + ResourceID: *managementClusterContentResourceID, + } +} + +// ManagementClusterContentResourceIDFromParentResourceID returns the resource ID for the +// ManagementClusterContent nested under parentResourceID with the given +// maestro bundle internal name. +func ManagementClusterContentResourceIDFromParentResourceID(parentResourceID *azcorearm.ResourceID, maestroBundleInternalName api.MaestroBundleInternalName) *azcorearm.ResourceID { + return api.Must(azcorearm.ParseResourceID(fmt.Sprintf("%s/%s/%s", parentResourceID.String(), api.ManagementClusterContentResourceTypeName, maestroBundleInternalName))) +} diff --git a/backend/pkg/controllers/controllerutils/management_cluster_content_controller_test.go b/backend/pkg/controllers/controllerutils/management_cluster_content_controller_test.go new file mode 100644 index 00000000000..c3020fa6dd6 --- /dev/null +++ b/backend/pkg/controllers/controllerutils/management_cluster_content_controller_test.go @@ -0,0 +1,46 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllerutils + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + + "github.com/Azure/ARO-HCP/internal/api" +) + +func TestManagementClusterContentResourceIDFromClusterResourceID(t *testing.T) { + clusterRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/mycluster")) + + got := ManagementClusterContentResourceIDFromParentResourceID(clusterRID, api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster) + require.NotNil(t, got) + assert.Equal(t, got.ResourceType.Type, api.ClusterScopedManagementClusterContentResourceType.Type) + // Name is the last segment of the resource ID (the management cluster content name) + assert.Equal(t, got.Name, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) +} + +func TestManagementClusterContentResourceIDFromNodePoolResourceID(t *testing.T) { + nodePoolRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/mycluster/nodePools/mynodepool")) + + got := ManagementClusterContentResourceIDFromParentResourceID(nodePoolRID, api.MaestroBundleInternalNameReadonlyHypershiftNodePool) + require.NotNil(t, got) + assert.Equal(t, got.ResourceType.Type, api.NodePoolScopedManagementClusterContentResourceType.Type) + // Name is the last segment of the resource ID (the management cluster content name) + assert.Equal(t, got.Name, string(api.MaestroBundleInternalNameReadonlyHypershiftNodePool)) +} diff --git a/backend/pkg/controllers/controllerutils/nodepool_watching_controller.go b/backend/pkg/controllers/controllerutils/nodepool_watching_controller.go index b99054fef42..e4bd5bd78b2 100644 --- a/backend/pkg/controllers/controllerutils/nodepool_watching_controller.go +++ b/backend/pkg/controllers/controllerutils/nodepool_watching_controller.go @@ -67,6 +67,13 @@ func NewNodePoolWatchingController( if err != nil { panic(err) // coding error } + + managementClusterContentInformer, _ := informers.ManagementClusterContents() + // Limit the max depth of ManagementClusterContent to 1 to only consider the nodepool-scoped ManagementClusterContents + err = nodePoolController.QueueForInformersWithMaxDepth(resyncDuration, 1, managementClusterContentInformer) + if err != nil { + panic(err) // coding error + } } return nodePoolController diff --git a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go index 3a424c1bd8d..b8e67f6fe1b 100644 --- a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller.go @@ -20,12 +20,9 @@ import ( "net/http" "time" - "github.com/google/uuid" workv1 "open-cluster-management.io/api/work/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" @@ -42,8 +39,6 @@ import ( ) const ( - // readonlyBundleManagedByK8sLabelKey is the key of the K8s label that is used to identify the controller that manages the readonly Maestro bundle. - readonlyBundleManagedByK8sLabelKey = "aro-hcp.azure.com/readonly-bundle-managed-by" // readonlyBundleManagedByK8sLabelValueClusterScoped is the K8s label associated to the readonlyBundleManagedByK8sLabelKey // key that indicates that the readonly Maestro bundle is managed by the create // cluster scoped maestro readonly bundles controller. @@ -69,10 +64,7 @@ type createClusterScopedMaestroReadonlyBundlesSyncer struct { maestroClientBuilder maestro.MaestroClientBuilder - // uuidV4Generator is used to generate a new UUIDv4. It must be provided. - // We define it as a dependency to enable deterministic testing in some - // scenarios. - uuidV4Generator func() (uuid.UUID, error) + maestroAPIMaestroBundleNameGenerator maestro.MaestroAPIMaestroBundleNameGenerator } var _ controllerutils.ClusterSyncer = (*createClusterScopedMaestroReadonlyBundlesSyncer)(nil) @@ -87,13 +79,13 @@ func NewCreateClusterScopedMaestroReadonlyBundlesController( ) controllerutils.Controller { syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - cooldownChecker: controllerutils.DefaultActiveOperationPrioritizingCooldown(activeOperationLister), - cosmosClient: cosmosClient, - clusterServiceClient: clusterServiceClient, - activeOperationLister: activeOperationLister, - maestroSourceEnvironmentIdentifier: maestroSourceEnvironmentIdentifier, - maestroClientBuilder: maestroClientBuilder, - uuidV4Generator: uuid.NewRandom, + cooldownChecker: controllerutils.DefaultActiveOperationPrioritizingCooldown(activeOperationLister), + cosmosClient: cosmosClient, + clusterServiceClient: clusterServiceClient, + activeOperationLister: activeOperationLister, + maestroSourceEnvironmentIdentifier: maestroSourceEnvironmentIdentifier, + maestroClientBuilder: maestroClientBuilder, + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } controller := controllerutils.NewClusterWatchingController( @@ -173,7 +165,7 @@ func (c *createClusterScopedMaestroReadonlyBundlesSyncer) SyncOnce(ctx context.C // This is important to avoid leaking resources when the sync is done. ctx, cancel := context.WithCancel(ctx) defer cancel() - maestroClient, err := c.createMaestroClientFromProvisionShard(ctx, clusterProvisionShard) + maestroClient, err := createMaestroClientFromCSProvisionShard(ctx, c.maestroSourceEnvironmentIdentifier, c.maestroClientBuilder, clusterProvisionShard) if err != nil { return utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) } @@ -230,7 +222,7 @@ func (c *createClusterScopedMaestroReadonlyBundlesSyncer) syncMaestroBundle( // and it makes it resistant to crashes/reboots. if existingMaestroBundleRef == nil { var err error - existingMaestroBundleRef, err = c.buildInitialMaestroBundleReference(maestroBundleInternalName) + existingMaestroBundleRef, err = buildInitialMaestroBundleReference(maestroBundleInternalName, c.maestroAPIMaestroBundleNameGenerator) if err != nil { return lastPersistedSPC, utils.TrackError(fmt.Errorf("failed to build initial Maestro Bundle reference: %w", err)) } @@ -266,7 +258,7 @@ func (c *createClusterScopedMaestroReadonlyBundlesSyncer) syncMaestroBundle( return lastPersistedSPC, utils.TrackError(fmt.Errorf("unrecognized Maestro Bundle internal name: %s", maestroBundleInternalName)) } - resultMaestroBundle, err := c.getOrCreateMaestroBundle(ctx, maestroClient, desiredMaestroBundle) + resultMaestroBundle, err := maestro.GetOrCreateMaestroBundle(ctx, maestroClient, desiredMaestroBundle) if err != nil { return lastPersistedSPC, utils.TrackError(fmt.Errorf("failed to get or create Maestro Bundle: %w", err)) } @@ -336,111 +328,7 @@ func (c *createClusterScopedMaestroReadonlyBundlesSyncer) buildInitialReadonlyMa Namespace: hostedCluster.Namespace, } - return c.buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName, maestroBundleResourceIdentifier, hostedCluster) -} - -// buildInitialReadonlyMaestroBundle builds an initial readonly Maestro Bundle for a given resource specified in obj. -// objResourceIdentifier is the resource identifier of the resource specified in obj. -// maestroBundleNamespacedName is the namespaced name of the Maestro Bundle. -// Used to create the readonly Maestro bundle associated to the resource specified in obj. -func (c *createClusterScopedMaestroReadonlyBundlesSyncer) buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName types.NamespacedName, objResourceIdentifier workv1.ResourceIdentifier, obj runtime.Object) *workv1.ManifestWork { - maestroBundleObjMeta := metav1.ObjectMeta{ - Name: maestroBundleNamespacedName.Name, - Namespace: maestroBundleNamespacedName.Namespace, - ResourceVersion: "0", // TODO is this needed when creating a maestro bundle? - Labels: map[string]string{ - // We define it as a K8s label because Maestro supports server-side filtering based on K8s labels. - // We can define it as a K8s label because for this specific use case we can comply with - // K8s labels length and charset restrictions https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set. - readonlyBundleManagedByK8sLabelKey: readonlyBundleManagedByK8sLabelValueClusterScoped, - }, - } - - // We build the Maestro Bundle that will contain the resource specified in obj. - // Aside from putting the resource (manifest) previously built above, we - // also define a FeedbackRule that will allow us to retrieve the whole content - // from the management cluster - maestroBundle := &workv1.ManifestWork{ - ObjectMeta: maestroBundleObjMeta, - Spec: workv1.ManifestWorkSpec{ - Workload: workv1.ManifestsTemplate{ - Manifests: []workv1.Manifest{ - { - RawExtension: runtime.RawExtension{ - // We put the resource (manifest) specified in obj. - // In Maestro only the desired `spec` as defined in the bundle can be retrieved - // from here when querying the Maestro Bundle. - // To retrieve another section other than the desired spec Maestro - // requires defining FeedbackRule(s) in the Maestro bundle. - // For maestro readonly resources, not even the desired spec can be retrieved from here. For - // those type of resources it needs to be retrieved via status feedback rule(s) too. - // For owned resources, here the desired spec can be retrieved but that - // is not necessarily the actual spec in the management cluster side. If that is - // desired it is again necessary to get the spec via FeedbackRule(s). - Object: obj, - }, - }, - }, - }, - ManifestConfigs: []workv1.ManifestConfigOption{ - // We also need to define the ManifestConfig associated to the resource(manifest) - // that is being put within the Maestro Bundle. - { - // ResourceIdentifier needs to be specified and it is the information - // associated to the manifest that is being put within the Maestro Bundle. - ResourceIdentifier: objResourceIdentifier, - // We need to set the UpdateStrategy to read only. This - // creates a "readonly maestro bundle". - UpdateStrategy: &workv1.UpdateStrategy{ - Type: workv1.UpdateStrategyTypeReadOnly, - }, - // We define a feedbackrule based on JSONPath. We alias the name - // of this JSONPath as "resource" and its real JSONPath is "@" which - // signals the whole object is retrieved. This includes both spec - // and status. - FeedbackRules: []workv1.FeedbackRule{ - { - Type: workv1.JSONPathsType, - JsonPaths: []workv1.JsonPath{ - { - Name: "resource", - Path: "@", - }, - }, - }, - }, - }, - }, - }, - } - - return maestroBundle -} - -// buildInitialMaestroBundleReference builds an initial Maestro Bundle reference for a given maestro bundle internal name. -func (c *createClusterScopedMaestroReadonlyBundlesSyncer) buildInitialMaestroBundleReference(internalName api.MaestroBundleInternalName) (*api.MaestroBundleReference, error) { - maestroAPIMaestroBundleName, err := c.generateNewMaestroAPIMaestroBundleName() - if err != nil { - return nil, utils.TrackError(fmt.Errorf("failed to generate Maestro API Maestro Bundle name: %w", err)) - } - hostedClusterMWMaestroBundleReference := &api.MaestroBundleReference{ - Name: internalName, - MaestroAPIMaestroBundleName: maestroAPIMaestroBundleName, - MaestroAPIMaestroBundleID: "", - } - - return hostedClusterMWMaestroBundleReference, nil -} - -// generateNewMaestroAPIMaestroBundleName generates a new Maestro API Maestro Bundle name. -// Used to generate a new Maestro API Maestro Bundle name for a new Maestro Bundle reference. -// The generated name is a UUIDv4. -func (c *createClusterScopedMaestroReadonlyBundlesSyncer) generateNewMaestroAPIMaestroBundleName() (string, error) { - newUUIDForMaestroAPIMaestroBundleName, err := c.uuidV4Generator() - if err != nil { - return "", utils.TrackError(fmt.Errorf("failed to generate UUIDv4 for Maestro API Maestro Bundle name: %w", err)) - } - return newUUIDForMaestroAPIMaestroBundleName.String(), nil + return buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName, maestroBundleResourceIdentifier, hostedCluster, readonlyBundleManagedByK8sLabelValueClusterScoped) } // getHostedClusterNamespace gets the namespace for the hosted cluster based on the environment name and the cluster service OCM Cluster ID. @@ -452,56 +340,6 @@ func (c *createClusterScopedMaestroReadonlyBundlesSyncer) getHostedClusterNamesp return fmt.Sprintf("ocm-%s-%s", envName, csClusterID) } -// getOrCreateMaestroBundle gets (or creates if it does not exist) a Maestro Bundle for a given Maestro Bundle namespaced name. -func (c *createClusterScopedMaestroReadonlyBundlesSyncer) getOrCreateMaestroBundle(ctx context.Context, maestroClient maestro.Client, maestroBundle *workv1.ManifestWork) (*workv1.ManifestWork, error) { - logger := utils.LoggerFromContext(ctx) - existingMaestroBundle, err := maestroClient.Get(ctx, maestroBundle.Name, metav1.GetOptions{}) - if err == nil { - logger.Info(fmt.Sprintf("retrieved maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) - return existingMaestroBundle, nil - } - if !k8serrors.IsNotFound(err) { - logger.Error(err, "failed to get Maestro Bundle and it is not already exists error") - return nil, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle: %w", err)) - } - - logger.Info(fmt.Sprintf("attempting to create maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) - existingMaestroBundle, err = maestroClient.Create(ctx, maestroBundle, metav1.CreateOptions{}) - if err == nil { - logger.Info(fmt.Sprintf("created maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) - return existingMaestroBundle, nil - } - if !k8serrors.IsAlreadyExists(err) { - logger.Error(err, "failed to create Maestro Bundle and it is not already exists error") - return nil, utils.TrackError(fmt.Errorf("failed to create Maestro Bundle: %w", err)) - } - logger.Error(err, "failed to create Maestro Bundle because it returned already exists error. Attempting to get it again") - existingMaestroBundle, err = maestroClient.Get(ctx, maestroBundle.Name, metav1.GetOptions{}) - return existingMaestroBundle, err -} - func (c *createClusterScopedMaestroReadonlyBundlesSyncer) CooldownChecker() controllerutils.CooldownChecker { return c.cooldownChecker } - -// createMaestroClientFromProvisionShard creates a Maestro client for the given provision shard. -// The client is scoped to the Maestro Consumer associated to the provision shard, as well -// as to the the Maestro Source ID associated to the provision shard which is calculated from the provision shard ID and the -// environment specified in c.maestroSourceEnvironmentIdentifier. -func (c *createClusterScopedMaestroReadonlyBundlesSyncer) createMaestroClientFromProvisionShard( - ctx context.Context, provisionShard *arohcpv1alpha1.ProvisionShard, -) (maestro.Client, error) { - provisionShardMaestroConsumerName := provisionShard.MaestroConfig().ConsumerName() - provisionShardMaestroRESTAPIEndpoint := provisionShard.MaestroConfig().RestApiConfig().Url() - provisionShardMaestroGRPCAPIEndpoint := provisionShard.MaestroConfig().GrpcApiConfig().Url() - // This allows us to be able to have visibility on the Maestro Bundles owned by the same source ID for a given - // provision shard and environment. This should have the same source ID as what CS has in each corresponding environment - // because otherwise we would not have visibility on the Maestro Bundles owned - // TODO do we want to use the same source ID that CS uses or do we want intentionally a different one? This has consequences - // on the visibility of the Maestro Bundles, including processing of events sent by Maestro. - maestroSourceID := maestro.GenerateMaestroSourceID(c.maestroSourceEnvironmentIdentifier, provisionShard.ID()) - - maestroClient, err := c.maestroClientBuilder.NewClient(ctx, provisionShardMaestroRESTAPIEndpoint, provisionShardMaestroGRPCAPIEndpoint, provisionShardMaestroConsumerName, maestroSourceID) - - return maestroClient, err -} diff --git a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller_test.go b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller_test.go index 932279ae082..8f2a70fb855 100644 --- a/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller_test.go +++ b/backend/pkg/controllers/create_cluster_scoped_maestro_readonly_bundles_controller_test.go @@ -76,8 +76,8 @@ var _ database.ServiceProviderClusterCRUD = &errorInjectingSPCCRUDForCreate{} func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_buildClusterEmptyHostedCluster(t *testing.T) { syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - maestroSourceEnvironmentIdentifier: "testenv", - uuidV4Generator: uuid.NewRandom, + maestroSourceEnvironmentIdentifier: "testenv", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } csClusterID := "11111111111111111111111111111111" @@ -97,8 +97,8 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_getHostedClusterNamespa expected := fmt.Sprintf("ocm-%s-%s", envName, csClusterID) syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - maestroSourceEnvironmentIdentifier: envName, - uuidV4Generator: uuid.NewRandom, + maestroSourceEnvironmentIdentifier: envName, + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } result := syncer.getHostedClusterNamespace(envName, csClusterID) @@ -106,11 +106,9 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_getHostedClusterNamespa } func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_buildInitialMaestroBundleReference(t *testing.T) { - syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - uuidV4Generator: uuid.NewRandom, - } + generator := maestro.NewMaestroAPIMaestroBundleNameGenerator() - ref, err := syncer.buildInitialMaestroBundleReference(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster) + ref, err := buildInitialMaestroBundleReference(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster, generator) require.NoError(t, err) assert.NotNil(t, ref) @@ -125,8 +123,8 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_buildInitialMaestroBund func TestBuildInitialReadonlyMaestroBundleForHostedCluster(t *testing.T) { syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - maestroSourceEnvironmentIdentifier: "testenv", - uuidV4Generator: uuid.NewRandom, + maestroSourceEnvironmentIdentifier: "testenv", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster")) @@ -166,108 +164,6 @@ func TestBuildInitialReadonlyMaestroBundleForHostedCluster(t *testing.T) { assert.Equal(t, expectedHostedCluster, bundle.Spec.Workload.Manifests[0].Object) } -func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_getOrCreateMaestroBundle(t *testing.T) { - desiredBundle := &workv1.ManifestWork{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-maestro-api-maestro-bundle-name", - Namespace: "test-maestro-consumer", - }, - Spec: workv1.ManifestWorkSpec{ - ManifestConfigs: []workv1.ManifestConfigOption{ - { - ResourceIdentifier: workv1.ResourceIdentifier{ - Name: "hostedcluster-name", - Namespace: "ocm-testenv-11111111111111111111111111111111", - }, - }, - }, - }, - } - - tests := []struct { - name string - setupMock func(*maestro.MockClient, *workv1.ManifestWork) - wantBundle *workv1.ManifestWork - wantErr bool - errSubstr string - }{ - { - name: "returns existing bundle if it already exists", - setupMock: func(m *maestro.MockClient, want *workv1.ManifestWork) { - m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(want, nil) - }, - wantBundle: &workv1.ManifestWork{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-maestro-api-maestro-bundle-name", Namespace: "test-maestro-consumer", UID: "existing-uid", - }, - }, - }, - { - name: "creates new bundle if it does not exist", - setupMock: func(m *maestro.MockClient, want *workv1.ManifestWork) { - m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "not-found")) - m.EXPECT().Create(gomock.Any(), desiredBundle, gomock.Any()).Return(want, nil) - }, - wantBundle: &workv1.ManifestWork{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-maestro-api-maestro-bundle-name", Namespace: "test-maestro-consumer", UID: "new-uid", - }, - }, - }, - { - name: "returns existing bundle when internal call to create returns AlreadyExists and then the following get succeeds", - setupMock: func(m *maestro.MockClient, want *workv1.ManifestWork) { - m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "not-found")) - m.EXPECT().Create(gomock.Any(), desiredBundle, gomock.Any()).Return(nil, k8serrors.NewAlreadyExists(schema.GroupResource{}, "test-maestro-api-maestro-bundle-name")) - m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(want, nil) - }, - wantBundle: &workv1.ManifestWork{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-maestro-api-maestro-bundle-name", Namespace: "test-maestro-consumer", UID: "existing-uid", - }, - }, - }, - { - name: "returns error if it fails to get the bundle", - setupMock: func(m *maestro.MockClient, _ *workv1.ManifestWork) { - m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(nil, fmt.Errorf("connection error")) - }, - wantErr: true, - errSubstr: "failed to get Maestro Bundle", - }, - { - name: "returns error if it fails to create the bundle", - setupMock: func(m *maestro.MockClient, _ *workv1.ManifestWork) { - m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "test-maestro-api-maestro-bundle-name")) - m.EXPECT().Create(gomock.Any(), desiredBundle, gomock.Any()).Return(nil, fmt.Errorf("maestro API error")) - }, - wantErr: true, - errSubstr: "failed to create Maestro Bundle: maestro API error", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - tt.setupMock(mockMaestro, tt.wantBundle) - syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - uuidV4Generator: uuid.NewRandom, - } - - result, err := syncer.getOrCreateMaestroBundle(context.Background(), mockMaestro, desiredBundle) - - if tt.wantErr { - require.Error(t, err) - assert.Nil(t, result) - assert.Contains(t, err.Error(), tt.errSubstr) - } else { - require.NoError(t, err) - assert.Equal(t, tt.wantBundle, result) - } - }) - } -} func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_syncMaestroBundle(t *testing.T) { // syncMaestroBundleTestDeterministicUUID is the fixed UUID used when testing "no bundle reference initially" so returned values are deterministic. syncMaestroBundleTestDeterministicUUID := uuid.MustParse("aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee") @@ -503,10 +399,9 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_syncMaestroBundle(t *te mockMaestro := maestro.NewMockClient(ctrl) tt.maestroClientSetupMock(mockMaestro) - deterministicUUIDGenerator := func() (uuid.UUID, error) { return syncMaestroBundleTestDeterministicUUID, nil } syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - maestroSourceEnvironmentIdentifier: "test-env", - uuidV4Generator: deterministicUUIDGenerator, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewAlwaysSameNameMaestroAPIMaestroBundleNameGenerator(syncMaestroBundleTestDeterministicUUID.String()), } ctx := context.Background() cluster := &api.HCPOpenShiftCluster{ @@ -552,35 +447,7 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_syncMaestroBundle(t *te } } -func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_generateNewMaestroAPIMaestroBundleName(t *testing.T) { - syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - uuidV4Generator: uuid.NewRandom, - } - - // Test successful generation - name1, err := syncer.generateNewMaestroAPIMaestroBundleName() - require.NoError(t, err) - assert.NotEmpty(t, name1) - - // Verify it's a valid UUID - _, err = uuid.Parse(name1) - assert.NoError(t, err, "Generated name should be a valid UUID") - - // Test that multiple calls generate different UUIDs - name2, err := syncer.generateNewMaestroAPIMaestroBundleName() - require.NoError(t, err) - assert.NotEqual(t, name1, name2, "Multiple calls should generate different UUIDs") - - // Verify second name is also a valid UUID - _, err = uuid.Parse(name2) - assert.NoError(t, err, "Second generated name should also be a valid UUID") -} - func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_buildInitialReadonlyMaestroBundle(t *testing.T) { - syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - uuidV4Generator: uuid.NewRandom, - } - maestroBundleNamespacedName := types.NamespacedName{ Name: "custom-bundle", Namespace: "custom-namespace", @@ -605,7 +472,7 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_buildInitialReadonlyMae Namespace: configMap.Namespace, } - bundle := syncer.buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName, resourceIdentifier, configMap) + bundle := buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName, resourceIdentifier, configMap, readonlyBundleManagedByK8sLabelValueClusterScoped) // Verify bundle metadata assert.NotNil(t, bundle) @@ -644,10 +511,10 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_buildInitialReadonlyMae func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_SyncOnce_ClusterNotFound(t *testing.T) { mockDBClient := databasetesting.NewMockDBClient() syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - cooldownChecker: &alwaysSyncCooldownChecker{}, - cosmosClient: mockDBClient, - maestroSourceEnvironmentIdentifier: "test-env", - uuidV4Generator: uuid.NewRandom, + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } key := controllerutils.HCPClusterKey{ @@ -697,10 +564,10 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_SyncOnce_GetServiceProv } syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - cooldownChecker: &alwaysSyncCooldownChecker{}, - cosmosClient: mockDBClient, - maestroSourceEnvironmentIdentifier: "test-env", - uuidV4Generator: uuid.NewRandom, + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } err = syncer.SyncOnce(ctx, key) @@ -712,10 +579,10 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_SyncOnce_AllBundlesAlre ctx := context.Background() mockDBClient := databasetesting.NewMockDBClient() syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - cooldownChecker: &alwaysSyncCooldownChecker{}, - cosmosClient: mockDBClient, - maestroSourceEnvironmentIdentifier: "test-env", - uuidV4Generator: uuid.NewRandom, + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } key := controllerutils.HCPClusterKey{ @@ -779,12 +646,12 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_SyncOnce_SyncLoopExecut mockMaestroClient := maestro.NewMockClient(ctrl) syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - cooldownChecker: &alwaysSyncCooldownChecker{}, - cosmosClient: mockDBClient, - clusterServiceClient: mockClusterService, - maestroClientBuilder: mockMaestroBuilder, - maestroSourceEnvironmentIdentifier: "test-env", - uuidV4Generator: uuid.NewRandom, + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + clusterServiceClient: mockClusterService, + maestroClientBuilder: mockMaestroBuilder, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } key := controllerutils.HCPClusterKey{ @@ -894,12 +761,12 @@ func TestCreateClusterScopedMaestroReadonlyBundlesSyncer_SyncOnce_ProcessesParti mockMaestroClient := maestro.NewMockClient(ctrl) syncer := &createClusterScopedMaestroReadonlyBundlesSyncer{ - cooldownChecker: &alwaysSyncCooldownChecker{}, - cosmosClient: mockDBClient, - clusterServiceClient: mockClusterService, - maestroClientBuilder: mockMaestroBuilder, - maestroSourceEnvironmentIdentifier: "test-env", - uuidV4Generator: uuid.NewRandom, + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + clusterServiceClient: mockClusterService, + maestroClientBuilder: mockMaestroBuilder, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), } key := controllerutils.HCPClusterKey{ diff --git a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go new file mode 100644 index 00000000000..0c82e1a0b80 --- /dev/null +++ b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller.go @@ -0,0 +1,363 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package controllers + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + workv1 "open-cluster-management.io/api/work/v1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" + hsv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + + "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" + "github.com/Azure/ARO-HCP/backend/pkg/informers" + "github.com/Azure/ARO-HCP/backend/pkg/listers" + "github.com/Azure/ARO-HCP/backend/pkg/maestro" + "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/database" + "github.com/Azure/ARO-HCP/internal/ocm" + "github.com/Azure/ARO-HCP/internal/utils" +) + +const ( + // readonlyBundleManagedByK8sLabelValueNodePoolScoped is the K8s label associated to the readonlyBundleManagedByK8sLabelKey + // key that indicates that the readonly Maestro bundle is managed by the create + // nodepool scoped maestro readonly bundles controller. + readonlyBundleManagedByK8sLabelValueNodePoolScoped = "create-nodepool-scoped-maestro-readonly-bundles-controller" +) + +// createNodePoolScopedMaestroReadonlyBundlesSyncer is a controller that creates Maestro readonly bundles for the node pools. +// It is responsible for creating the Maestro readonly bundles and storing a reference to them in Cosmos. It does +// not persist the content of the Maestro bundles themselves. That is the responsibility of the +// readAndPersistMaestroReadonlyBundlesContentSyncer controller. +// As of now we support the creation of a Maestro readonly bundle for the Hypershift's NodePool CRs associated to +// the Cluster. +type createNodePoolScopedMaestroReadonlyBundlesSyncer struct { + cooldownChecker controllerutils.CooldownChecker + + activeOperationLister listers.ActiveOperationLister + + cosmosClient database.DBClient + + clusterServiceClient ocm.ClusterServiceClientSpec + + maestroSourceEnvironmentIdentifier string + + maestroClientBuilder maestro.MaestroClientBuilder + + maestroAPIMaestroBundleNameGenerator maestro.MaestroAPIMaestroBundleNameGenerator +} + +var _ controllerutils.NodePoolSyncer = (*createNodePoolScopedMaestroReadonlyBundlesSyncer)(nil) + +func NewCreateNodePoolScopedMaestroReadonlyBundlesController( + activeOperationLister listers.ActiveOperationLister, + cosmosClient database.DBClient, + clusterServiceClient ocm.ClusterServiceClientSpec, + informers informers.BackendInformers, + maestroSourceEnvironmentIdentifier string, + maestroClientBuilder maestro.MaestroClientBuilder, +) controllerutils.Controller { + + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + cooldownChecker: controllerutils.DefaultActiveOperationPrioritizingCooldown(activeOperationLister), + cosmosClient: cosmosClient, + clusterServiceClient: clusterServiceClient, + activeOperationLister: activeOperationLister, + maestroSourceEnvironmentIdentifier: maestroSourceEnvironmentIdentifier, + maestroClientBuilder: maestroClientBuilder, + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + controller := controllerutils.NewNodePoolWatchingController( + "CreateNodePoolScopedMaestroReadonlyBundles", + cosmosClient, + informers, + 1*time.Minute, + syncer, + ) + + return controller +} + +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) SyncOnce(ctx context.Context, key controllerutils.HCPNodePoolKey) error { + existingNodePool, err := c.cosmosClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName).Get(ctx, key.HCPNodePoolName) + if database.IsResponseError(err, http.StatusNotFound) { + return nil // nodepool doesn't exist, no work to do + } + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get NodePool: %w", err)) + } + if len(existingNodePool.ServiceProviderProperties.ClusterServiceID.String()) == 0 { + // TODO remove this once we have the information all in cosmos. + return nil + } + + existingServiceProviderNodePool, err := database.GetOrCreateServiceProviderNodePool(ctx, c.cosmosClient, key.GetResourceID()) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get or create ServiceProviderNodePool: %w", err)) + } + + // The list of Maestro Bundle internal names that are recognized by the controller. + // Any Maestro Bundle internal name that is not in this list will not be synced by the + // controller and reported as an error. + recognizedMaestroBundles := []api.MaestroBundleInternalName{ + api.MaestroBundleInternalNameReadonlyHypershiftNodePool, + } + + var maestroBundlesToSync []api.MaestroBundleInternalName + // We first check if there's any recognized Maestro Bundle reference that needs to be synced. + for _, maestroBundleInternalName := range recognizedMaestroBundles { + currentMaestroBundleReference, err := existingServiceProviderNodePool.Status.MaestroReadonlyBundles.Get(maestroBundleInternalName) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get Maestro Bundle reference: %w", err)) + } + + if currentMaestroBundleReference == nil { + maestroBundlesToSync = append(maestroBundlesToSync, maestroBundleInternalName) + continue + } + if len(currentMaestroBundleReference.MaestroAPIMaestroBundleName) == 0 { + maestroBundlesToSync = append(maestroBundlesToSync, maestroBundleInternalName) + continue + } + if len(currentMaestroBundleReference.MaestroAPIMaestroBundleID) == 0 { + maestroBundlesToSync = append(maestroBundlesToSync, maestroBundleInternalName) + continue + } + } + if len(maestroBundlesToSync) == 0 { + return nil + } + + serviceProviderNodePoolsDBClient := c.cosmosClient.ServiceProviderNodePools( + key.SubscriptionID, + key.ResourceGroupName, + key.HCPClusterName, + key.HCPNodePoolName, + ) + + // We get the provision shard (management cluster) the CS cluster is allocated to. + // As of now in CS the shard allocation occurs synchronously during aro-hcp cluster creation call in CS API so + // we are guaranteed to have a shard allocated for the cluster. If this changes in the future + // we would need to change the logic in controllers to check that the retrieved cluster has a + // shard allocated. + + csClusterID := existingNodePool.ServiceProviderProperties.ClusterServiceID.ClusterID() + csClusterHREF := ocm.GenerateAROHCPClusterHREF(csClusterID) + csClusterInternalID := api.Must(api.NewInternalID(csClusterHREF)) + clusterProvisionShard, err := c.clusterServiceClient.GetClusterProvisionShard(ctx, csClusterInternalID) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get Cluster Provision Shard from Cluster Service: %w", err)) + } + + // We create a new context with a cancel function so we can cancel the Maestro client when the sync is done. + // This is important to avoid leaking resources when the sync is done. + ctx, cancel := context.WithCancel(ctx) + defer cancel() + maestroClient, err := createMaestroClientFromCSProvisionShard(ctx, c.maestroSourceEnvironmentIdentifier, c.maestroClientBuilder, clusterProvisionShard) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) + } + + csCluster, err := c.clusterServiceClient.GetCluster(ctx, csClusterInternalID) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get Cluster from Cluster Service: %w", err)) + } + csClusterDomainPrefix := csCluster.DomainPrefix() + + // We sync the Maestro Bundles that need to be synced. + // We pass the latest existingServiceProviderNodePool into each iteration and use the returned + // updated SPNP for the next, so that multiple bundles see persisted updates from previous iterations. + // We always apply updatedSPNP (even on error) so in-memory state stays in sync with Cosmos + // when syncMaestroBundle persisted a partial change before failing. + var syncErrors []error + for _, maestroBundleInternalName := range maestroBundlesToSync { + updatedSPNP, syncErr := c.syncMaestroBundle( + ctx, maestroBundleInternalName, existingServiceProviderNodePool, existingNodePool, maestroClient, + serviceProviderNodePoolsDBClient, clusterProvisionShard, csClusterDomainPrefix, + ) + existingServiceProviderNodePool = updatedSPNP + if syncErr != nil { + syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to sync Maestro Bundle %q: %w", maestroBundleInternalName, syncErr))) + } + } + + return utils.TrackError(errors.Join(syncErrors...)) +} + +// syncMaestroBundle ensures the given Maestro bundle exists in Maestro, as well as a reference to it in ServiceProviderNodePool. +// It returns the updated ServiceProviderNodePool (after any Replace calls) so the caller can pass it into the next sync. +// On error, the first return value is always the lastest persisted ServiceProviderNodePool, so the +// caller can keep in-memory state in sync and subsequent bundle syncs in the same run never see stale data. +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) syncMaestroBundle( + ctx context.Context, + maestroBundleInternalName api.MaestroBundleInternalName, + existingServiceProviderNodePool *api.ServiceProviderNodePool, + existingNodePool *api.HCPOpenShiftClusterNodePool, + maestroClient maestro.Client, + serviceProviderNodePoolsDBClient database.ServiceProviderNodePoolCRUD, + clusterProvisionShard *arohcpv1alpha1.ProvisionShard, + csClusterDomainPrefix string, +) (*api.ServiceProviderNodePool, error) { + lastPersistedSPNP := existingServiceProviderNodePool + + existingMaestroBundleRef, err := existingServiceProviderNodePool.Status.MaestroReadonlyBundles.Get(maestroBundleInternalName) + if err != nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle reference: %w", err)) + } + // If the Maestro Bundle reference does not exist, we create a new Maestro Bundle + // reference for the Maestro API Maestro Bundle name. When this occurs we also immediately + // store the content in Cosmos. This ensures that we have the name reserved for it + // and it makes it resistant to crashes/reboots. + if existingMaestroBundleRef == nil { + var err error + existingMaestroBundleRef, err = buildInitialMaestroBundleReference(maestroBundleInternalName, c.maestroAPIMaestroBundleNameGenerator) + if err != nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to build initial Maestro Bundle reference: %w", err)) + } + err = existingServiceProviderNodePool.Status.MaestroReadonlyBundles.Set(existingMaestroBundleRef) + if err != nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to set internal Maestro Bundle reference: %w", err)) + } + existingServiceProviderNodePool, err = serviceProviderNodePoolsDBClient.Replace(ctx, existingServiceProviderNodePool, nil) + if err != nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to replace ServiceProviderNodePool in database: %w", err)) + } + lastPersistedSPNP = existingServiceProviderNodePool + existingMaestroBundleRef, err = existingServiceProviderNodePool.Status.MaestroReadonlyBundles.Get(maestroBundleInternalName) + if err != nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle reference: %w", err)) + } + if existingMaestroBundleRef == nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("maestro Bundle reference %q not found in ServiceProviderNodePool", maestroBundleInternalName)) + } + } + + // We ensure that the Maestro Bundle exists using the Maestro API + maestroBundleNamespacedName := types.NamespacedName{ + Name: existingMaestroBundleRef.MaestroAPIMaestroBundleName, + Namespace: clusterProvisionShard.MaestroConfig().ConsumerName(), + } + + var desiredMaestroBundle *workv1.ManifestWork + switch maestroBundleInternalName { + case api.MaestroBundleInternalNameReadonlyHypershiftNodePool: + desiredMaestroBundle = c.buildInitialReadonlyMaestroBundleForNodePool(existingNodePool, csClusterDomainPrefix, maestroBundleNamespacedName) + default: + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("unrecognized Maestro Bundle internal name: %s", maestroBundleInternalName)) + } + + resultMaestroBundle, err := maestro.GetOrCreateMaestroBundle(ctx, maestroClient, desiredMaestroBundle) + if err != nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to get or create Maestro Bundle: %w", err)) + } + + // If the Maestro API MaestroBundle ID is not set we store the returned Maestro Bundle ID in the corresponding Maestro Bundle reference of the ServiceProviderNodePool in Cosmos. + if len(existingMaestroBundleRef.MaestroAPIMaestroBundleID) == 0 { + bundleID := string(resultMaestroBundle.UID) + existingMaestroBundleRef.MaestroAPIMaestroBundleID = bundleID + err = existingServiceProviderNodePool.Status.MaestroReadonlyBundles.Set(existingMaestroBundleRef) + if err != nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to set Maestro Bundle reference: %w", err)) + } + existingServiceProviderNodePool, err = serviceProviderNodePoolsDBClient.Replace(ctx, existingServiceProviderNodePool, nil) + if err != nil { + return lastPersistedSPNP, utils.TrackError(fmt.Errorf("failed to replace ServiceProviderNodePool in database: %w", err)) + } + lastPersistedSPNP = existingServiceProviderNodePool + } + + return lastPersistedSPNP, nil +} + +// buildClusterEmptyNodePool returns an empty node pool representing a Cluster's Hypershift NodePool resource. +// It strictly contains the type information and the object meta information necessary to identify the resource in the management cluster. +// It can be used to provide as the input of a Maestro resource bundle. +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildClusterEmptyNodePool(csClusterID string, csClusterDomainPrefix string, csNodePoolID string) *hsv1beta1.NodePool { + // TODO To calculate the NodePool namespace we pass the maestro source ID because it turns out to have the same + // value as the envName in CS. This is not accurate but it is good enough. + // I would decouple what is the maestro source ID envname part from the envname. The reason being that they are + // conceptually different things, they just happen to have the same value for the envName part. + // I am hesitant to provide a generic "environment name" deployment parameter to backend because people might introduce conditional logic based + // on the environment name which is fragile. The options I see are: + // * Provide a deployment parameter to backend that is named something concrete like "k8s-names-calculations-env-name" or similar to indicate + // that is something that is used to calculate names/namespaces of some k8s resources. + // * Expose in the CS API Cluster payload the "CDNamespace" associated to the cluster and start storing it in cosmos. This would allow to fully + // decouple from this concept of CDNamespace and we would use the stored value when needed. However, if we want to + // create resources in the same namespace as the old ones then we would still need to keep forever the concept of "env name part used to calculate + // some k8s resource names/namespaces". + nodePoolNamespace := c.getNodePoolNamespace(c.maestroSourceEnvironmentIdentifier, csClusterID) + nodePoolName := c.getNodePoolName(csClusterDomainPrefix, csNodePoolID) + + // We first build the resource (manifest) that we want to put within the Maestro Bundle. + // The resource is empty and it only has the type information and the object meta + // information necessary to identify the resource in the management cluster. + nodePool := &hsv1beta1.NodePool{ + TypeMeta: metav1.TypeMeta{ + Kind: "NodePool", + APIVersion: hsv1beta1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: nodePoolName, + Namespace: nodePoolNamespace, + }, + } + + return nodePool +} + +// buildInitialReadonlyMaestroBundleForNodePool builds an initial readonly Maestro Bundle a the Cluster's Hypershift NodePool. +// Used to create the readonly Maestro bundle associated to it. +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) buildInitialReadonlyMaestroBundleForNodePool(nodePool *api.HCPOpenShiftClusterNodePool, csClusterDomainPrefix string, maestroBundleNamespacedName types.NamespacedName) *workv1.ManifestWork { + csClusterID := nodePool.ServiceProviderProperties.ClusterServiceID.ClusterID() + hypershiftNodePool := c.buildClusterEmptyNodePool(csClusterID, csClusterDomainPrefix, nodePool.ID.Name) + maestroBundleResourceIdentifier := workv1.ResourceIdentifier{ + Group: hsv1beta1.SchemeGroupVersion.Group, + Resource: "nodepools", + Name: hypershiftNodePool.Name, + Namespace: hypershiftNodePool.Namespace, + } + + return buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName, maestroBundleResourceIdentifier, hypershiftNodePool, readonlyBundleManagedByK8sLabelValueNodePoolScoped) +} + +// getNodePoolNamespace gets the namespace for the node pool based on the environment name and the cluster service OCM Cluster ID. +// For example, if the Node Pool URL is /api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111/nodepools/XXXX then the +// cluster service OCM Cluster ID is 11111111111111111111111111111111. +// The namespace is of the format ocm--. This is how CS calculates Hypershift's NodePool namespace. +// Internally in CS this is the "CDNamespace" attribute associated to the cluster. +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) getNodePoolNamespace(envName string, csClusterID string) string { + return fmt.Sprintf("ocm-%s-%s", envName, csClusterID) +} + +// getNodePoolName gets the name for the node pool based on the cluster domain prefix and the node pool service OCM Node Pool ID. +// For example, if the Node Pool URL is /api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111/nodepools/XXXX and the +// cluster's domain prefix is test-domprefix then the name is test-domprefix-XXXX. +// The name is of the format -. +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) getNodePoolName(csClusterDomainPrefix string, csNodePoolID string) string { + return fmt.Sprintf("%s-%s", csClusterDomainPrefix, csNodePoolID) +} + +func (c *createNodePoolScopedMaestroReadonlyBundlesSyncer) CooldownChecker() controllerutils.CooldownChecker { + return c.cooldownChecker +} diff --git a/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go new file mode 100644 index 00000000000..db6b10b37c7 --- /dev/null +++ b/backend/pkg/controllers/create_nodepool_scoped_maestro_readonly_bundles_controller_test.go @@ -0,0 +1,862 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + workv1 "open-cluster-management.io/api/work/v1" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + + arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" + hsv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + + "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" + "github.com/Azure/ARO-HCP/backend/pkg/maestro" + "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/api/arm" + "github.com/Azure/ARO-HCP/internal/database" + "github.com/Azure/ARO-HCP/internal/databasetesting" + "github.com/Azure/ARO-HCP/internal/ocm" +) + +// errorInjectingDBClientForNodePoolCreate wraps MockDBClient to return error-injecting CRUDs. +type errorInjectingDBClientForNodePoolCreate struct { + *databasetesting.MockDBClient + spnpCRUD database.ServiceProviderNodePoolCRUD +} + +func (e *errorInjectingDBClientForNodePoolCreate) ServiceProviderNodePools(subscriptionID, resourceGroupName, clusterName, nodePoolName string) database.ServiceProviderNodePoolCRUD { + if e.spnpCRUD != nil { + return e.spnpCRUD + } + return e.MockDBClient.ServiceProviderNodePools(subscriptionID, resourceGroupName, clusterName, nodePoolName) +} + +var _ database.DBClient = &errorInjectingDBClientForNodePoolCreate{} + +// errorInjectingSPNPCRUDForCreate wraps ServiceProviderNodePoolCRUD to allow error injection. +type errorInjectingSPNPCRUDForCreate struct { + database.ServiceProviderNodePoolCRUD + getErr error +} + +func (e *errorInjectingSPNPCRUDForCreate) Get(ctx context.Context, resourceID string) (*api.ServiceProviderNodePool, error) { + if e.getErr != nil { + return nil, e.getErr + } + return e.ServiceProviderNodePoolCRUD.Get(ctx, resourceID) +} + +var _ database.ServiceProviderNodePoolCRUD = &errorInjectingSPNPCRUDForCreate{} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_buildClusterEmptyNodePool(t *testing.T) { + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + maestroSourceEnvironmentIdentifier: "testenv", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + csClusterID := "11111111111111111111111111111111" + csClusterDomainPrefix := "test-domprefix" + csNodePoolID := "nodepool-id-1234" + expectedNodePoolName := fmt.Sprintf("%s-%s", csClusterDomainPrefix, csNodePoolID) + np := syncer.buildClusterEmptyNodePool(csClusterID, csClusterDomainPrefix, csNodePoolID) + + assert.NotNil(t, np) + assert.Equal(t, "NodePool", np.Kind) + assert.Equal(t, hsv1beta1.SchemeGroupVersion.String(), np.APIVersion) + assert.Equal(t, expectedNodePoolName, np.Name) + assert.Equal(t, fmt.Sprintf("ocm-%s-%s", syncer.maestroSourceEnvironmentIdentifier, csClusterID), np.Namespace) +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_getNodePoolNamespace(t *testing.T) { + envName := "testenv" + csClusterID := "11111111111111111111111111111111" + expected := fmt.Sprintf("ocm-%s-%s", envName, csClusterID) + + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + maestroSourceEnvironmentIdentifier: envName, + } + + result := syncer.getNodePoolNamespace(envName, csClusterID) + assert.Equal(t, expected, result) +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_getNodePoolName(t *testing.T) { + envName := "testenv" + csClusterDomainPrefix := "test-domprefix" + csNodePoolID := "nodepool-abc" + expected := fmt.Sprintf("%s-%s", csClusterDomainPrefix, csNodePoolID) + + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + maestroSourceEnvironmentIdentifier: envName, + } + + result := syncer.getNodePoolName(csClusterDomainPrefix, csNodePoolID) + assert.Equal(t, expected, result) +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_buildInitialMaestroBundleReferenceForNodePool(t *testing.T) { + generator := maestro.NewMaestroAPIMaestroBundleNameGenerator() + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool + + ref, err := buildInitialMaestroBundleReference(bundleInternalName, generator) + require.NoError(t, err) + + assert.NotNil(t, ref) + assert.Equal(t, bundleInternalName, ref.Name) + assert.NotEmpty(t, ref.MaestroAPIMaestroBundleName) + assert.Empty(t, ref.MaestroAPIMaestroBundleID) + + // Verify the name is a valid UUID + _, err = uuid.Parse(ref.MaestroAPIMaestroBundleName) + assert.NoError(t, err, "MaestroAPIMaestroBundleName should be a valid UUID") +} + +func TestBuildInitialReadonlyMaestroBundleForNodePool(t *testing.T) { + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + maestroSourceEnvironmentIdentifier: "testenv", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + + csClusterDomainPrefix := "test-domprefix" + maestroBundleNamespacedName := types.NamespacedName{ + Name: "test-maestro-api-maestro-bundle-name", + Namespace: "test-maestro-consumer", + } + expectedNodePoolName := fmt.Sprintf("%s-%s", csClusterDomainPrefix, strings.ToLower(nodepool.Name)) + + bundle := syncer.buildInitialReadonlyMaestroBundleForNodePool(nodepool, csClusterDomainPrefix, maestroBundleNamespacedName) + require.NotNil(t, bundle) + + assert.Equal(t, "test-maestro-api-maestro-bundle-name", bundle.Name) + assert.Equal(t, "test-maestro-consumer", bundle.Namespace) + require.Len(t, bundle.Spec.Workload.Manifests, 1) + require.Len(t, bundle.Spec.ManifestConfigs, 1) + + // Verify manifest config + manifestConfig := bundle.Spec.ManifestConfigs[0] + assert.Equal(t, "nodepools", manifestConfig.ResourceIdentifier.Resource) + assert.Equal(t, hsv1beta1.SchemeGroupVersion.Group, manifestConfig.ResourceIdentifier.Group) + assert.Equal(t, expectedNodePoolName, manifestConfig.ResourceIdentifier.Name) + expectedNamespace := "ocm-testenv-11111111111111111111111111111111" + assert.Equal(t, expectedNamespace, manifestConfig.ResourceIdentifier.Namespace) + + expectedNodePool := syncer.buildClusterEmptyNodePool( + nodepool.ServiceProviderProperties.ClusterServiceID.ID(), + csClusterDomainPrefix, + nodepool.ID.Name, + ) + assert.Equal(t, expectedNodePool, bundle.Spec.Workload.Manifests[0].Object) +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_syncMaestroBundle(t *testing.T) { + syncMaestroBundleTestDeterministicUUID := uuid.MustParse("aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee") + var syncMaestroBundleTestOtherBundleName api.MaestroBundleInternalName = "otherReadonlyBundle" + + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool + + tests := []struct { + name string + initialSPNP *api.ServiceProviderNodePool + maestroClientSetupMock func(*maestro.MockClient) + wantServiceProviderNodePool *api.ServiceProviderNodePool + wantErr bool + wantErrSubstr string + }{ + { + name: "existing reference but no ID - sets new ID and preserves name", + initialSPNP: &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "existing-bundle-name", + MaestroAPIMaestroBundleID: "", + }, + }, + }, + }, + maestroClientSetupMock: func(m *maestro.MockClient) { + createdBundle := &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{Name: "existing-bundle-name", Namespace: "test-consumer", UID: "new-bundle-uid"}, + } + m.EXPECT().Get(gomock.Any(), "existing-bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "not-found")) + m.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any()).Return(createdBundle, nil) + }, + wantServiceProviderNodePool: &api.ServiceProviderNodePool{ + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "existing-bundle-name", + MaestroAPIMaestroBundleID: "new-bundle-uid", + }, + }, + }, + }, + }, + { + name: "complete bundle reference - ID unchanged", + initialSPNP: &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "complete-bundle-name", + MaestroAPIMaestroBundleID: "complete-bundle-id", + }, + }, + }, + }, + maestroClientSetupMock: func(m *maestro.MockClient) { + existingBundle := &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{Name: "complete-bundle-name", Namespace: "test-consumer", UID: "complete-bundle-id"}, + } + m.EXPECT().Get(gomock.Any(), "complete-bundle-name", gomock.Any()).Return(existingBundle, nil) + }, + wantServiceProviderNodePool: &api.ServiceProviderNodePool{ + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "complete-bundle-name", + MaestroAPIMaestroBundleID: "complete-bundle-id", + }, + }, + }, + }, + }, + { + name: "multiple refs - only synced ref is updated, other refs unchanged", + initialSPNP: &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: syncMaestroBundleTestOtherBundleName, + MaestroAPIMaestroBundleName: "other-bundle-name", + MaestroAPIMaestroBundleID: "other-bundle-id", + }, + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "nodepool-bundle-name", + MaestroAPIMaestroBundleID: "", + }, + }, + }, + }, + maestroClientSetupMock: func(m *maestro.MockClient) { + createdBundle := &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{Name: "nodepool-bundle-name", Namespace: "test-consumer", UID: "nodepool-bundle-uid"}, + } + m.EXPECT().Get(gomock.Any(), "nodepool-bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "not-found")) + m.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any()).Return(createdBundle, nil) + }, + wantServiceProviderNodePool: &api.ServiceProviderNodePool{ + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: syncMaestroBundleTestOtherBundleName, + MaestroAPIMaestroBundleName: "other-bundle-name", + MaestroAPIMaestroBundleID: "other-bundle-id", + }, + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "nodepool-bundle-name", + MaestroAPIMaestroBundleID: "nodepool-bundle-uid", + }, + }, + }, + }, + }, + { + name: "maestro get or create error - returns last persisted SPNP", + initialSPNP: &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "bundle-name", + MaestroAPIMaestroBundleID: "", + }, + }, + }, + }, + maestroClientSetupMock: func(m *maestro.MockClient) { + m.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("maestro connection error")) + }, + wantServiceProviderNodePool: &api.ServiceProviderNodePool{ + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "bundle-name", + MaestroAPIMaestroBundleID: "", + }, + }, + }, + }, + wantErr: true, + wantErrSubstr: "failed to get or create Maestro Bundle", + }, + { + name: "no bundle reference initially - creates ref with deterministic UUID name and new ID", + initialSPNP: &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{}, + }, + }, + maestroClientSetupMock: func(m *maestro.MockClient) { + deterministicName := syncMaestroBundleTestDeterministicUUID.String() + createdBundle := &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{Name: deterministicName, Namespace: "test-consumer", UID: "new-bundle-uid"}, + } + m.EXPECT().Get(gomock.Any(), deterministicName, gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "not-found")) + m.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, mw *workv1.ManifestWork, opts metav1.CreateOptions) (*workv1.ManifestWork, error) { + assert.Equal(t, deterministicName, mw.Name) + assert.Equal(t, "test-consumer", mw.Namespace) + assert.Len(t, mw.Spec.Workload.Manifests, 1) + assert.Len(t, mw.Spec.ManifestConfigs, 1) + assert.Equal(t, workv1.UpdateStrategyTypeReadOnly, mw.Spec.ManifestConfigs[0].UpdateStrategy.Type) + return createdBundle, nil + }, + ) + }, + wantServiceProviderNodePool: &api.ServiceProviderNodePool{ + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: syncMaestroBundleTestDeterministicUUID.String(), + MaestroAPIMaestroBundleID: "new-bundle-uid", + }, + }, + }, + }, + }, + { + name: "no bundle ref initially - bundle name persisted then getOrCreate fails - returns SPNP with name set, no ID", + initialSPNP: &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{}, + }, + }, + maestroClientSetupMock: func(m *maestro.MockClient) { + deterministicName := syncMaestroBundleTestDeterministicUUID.String() + m.EXPECT().Get(gomock.Any(), deterministicName, gomock.Any()).Return(nil, fmt.Errorf("maestro connection error")) + }, + wantServiceProviderNodePool: &api.ServiceProviderNodePool{ + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: syncMaestroBundleTestDeterministicUUID.String(), + MaestroAPIMaestroBundleID: "", + }, + }, + }, + }, + wantErr: true, + wantErrSubstr: "failed to get or create Maestro Bundle", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + tt.maestroClientSetupMock(mockMaestro) + + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewAlwaysSameNameMaestroAPIMaestroBundleNameGenerator(syncMaestroBundleTestDeterministicUUID.String()), + } + ctx := context.Background() + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + + mockDB := databasetesting.NewMockDBClient() + spnpCRUD := mockDB.ServiceProviderNodePools("test-sub", "test-rg", "test-cluster", "test-nodepool") + createdSPNP, err := spnpCRUD.Create(ctx, tt.initialSPNP, nil) + require.NoError(t, err) + provisionShard := buildTestProvisionShard("test-consumer") + + result, err := syncer.syncMaestroBundle( + ctx, + bundleInternalName, + createdSPNP, + nodepool, + mockMaestro, + spnpCRUD, + provisionShard, + "test-domain", + ) + + assert.Equal(t, tt.wantErr, err != nil) + if tt.wantErr { + assert.Contains(t, err.Error(), tt.wantErrSubstr) + } + require.NotNil(t, result) + + wantList := tt.wantServiceProviderNodePool.Status.MaestroReadonlyBundles + gotList := result.Status.MaestroReadonlyBundles + require.Len(t, gotList, len(wantList), "result should have the same number of bundle refs as want") + for _, wantRef := range wantList { + gotRef, err := gotList.Get(wantRef.Name) + require.NoError(t, err) + require.NotNil(t, gotRef, "result missing bundle ref for name %q", wantRef.Name) + assert.Equal(t, wantRef.Name, gotRef.Name) + assert.Equal(t, wantRef.MaestroAPIMaestroBundleName, gotRef.MaestroAPIMaestroBundleName) + assert.Equal(t, wantRef.MaestroAPIMaestroBundleID, gotRef.MaestroAPIMaestroBundleID) + } + }) + } +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_NodePoolNotFound(t *testing.T) { + mockDBClient := databasetesting.NewMockDBClient() + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + // No nodepool in DB -> Get returns NotFound -> SyncOnce returns nil (no work to do) + err := syncer.SyncOnce(context.Background(), key) + assert.NoError(t, err) +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_EmptyClusterServiceID(t *testing.T) { + ctrl := gomock.NewController(t) + ctx := context.Background() + + mockDBClient := databasetesting.NewMockDBClient() + mockClusterService := ocm.NewMockClusterServiceClientSpec(ctrl) + + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + clusterServiceClient: mockClusterService, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.InternalID{}, + }, + } + nodepoolsCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{}, + }, + } + spnpCRUD := mockDBClient.ServiceProviderNodePools(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName, key.HCPNodePoolName) + _, err = spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + + // Cluster service ID not yet populated: skip sync (no OCM / Maestro calls), even though a bundle still needs syncing. + err = syncer.SyncOnce(ctx, key) + assert.NoError(t, err) + + // SPNP should be unchanged (sync did not run). + got, err := spnpCRUD.Get(ctx, api.ServiceProviderNodePoolResourceName) + require.NoError(t, err) + ref, err := got.Status.MaestroReadonlyBundles.Get(bundleInternalName) + require.NoError(t, err) + assert.Nil(t, ref, "Maestro bundle ref should not be created when ClusterServiceID is empty") +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_GetServiceProviderNodePoolError(t *testing.T) { + ctx := context.Background() + + baseMockDB := databasetesting.NewMockDBClient() + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + + nodepoolsCRUD := baseMockDB.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + expectedError := fmt.Errorf("database error") + mockDBClient := &errorInjectingDBClientForNodePoolCreate{ + MockDBClient: baseMockDB, + spnpCRUD: &errorInjectingSPNPCRUDForCreate{ + getErr: expectedError, + }, + } + + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + err = syncer.SyncOnce(ctx, key) + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to get or create ServiceProviderNodePool") +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_AllBundlesAlreadySynced(t *testing.T) { + ctx := context.Background() + mockDBClient := databasetesting.NewMockDBClient() + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + nodepoolsCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "bundle-name", + MaestroAPIMaestroBundleID: "bundle-id", + }, + }, + }, + } + spnpCRUD := mockDBClient.ServiceProviderNodePools(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName, key.HCPNodePoolName) + _, err = spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + + // Since all bundles are synced, no cluster service or maestro calls should be made + err = syncer.SyncOnce(ctx, key) + assert.NoError(t, err) +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_SyncLoopExecutesWithBundleCreation(t *testing.T) { + ctrl := gomock.NewController(t) + ctx := context.Background() + + mockDBClient := databasetesting.NewMockDBClient() + mockClusterService := ocm.NewMockClusterServiceClientSpec(ctrl) + mockMaestroBuilder := maestro.NewMockMaestroClientBuilder(ctrl) + mockMaestroClient := maestro.NewMockClient(ctrl) + + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + clusterServiceClient: mockClusterService, + maestroClientBuilder: mockMaestroBuilder, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + nodepoolsCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + // SPNP with no bundle reference (needs syncing) + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{}, + }, + } + spnpCRUD := mockDBClient.ServiceProviderNodePools(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName, key.HCPNodePoolName) + _, err = spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + + provisionShard := buildTestProvisionShard("test-consumer") + mockClusterService.EXPECT(). + GetClusterProvisionShard(gomock.Any(), nodepool.ServiceProviderProperties.ClusterServiceID). + Return(provisionShard, nil) + + csCluster, err := arohcpv1alpha1.NewCluster(). + DomainPrefix("test-domain"). + Build() + require.NoError(t, err) + mockClusterService.EXPECT(). + GetCluster(gomock.Any(), nodepool.ServiceProviderProperties.ClusterServiceID). + Return(csCluster, nil) + + restEndpoint := provisionShard.MaestroConfig().RestApiConfig().Url() + grpcEndpoint := provisionShard.MaestroConfig().GrpcApiConfig().Url() + consumerName := provisionShard.MaestroConfig().ConsumerName() + sourceID := maestro.GenerateMaestroSourceID("test-env", provisionShard.ID()) + mockMaestroBuilder.EXPECT(). + NewClient(gomock.Any(), restEndpoint, grpcEndpoint, consumerName, sourceID). + Return(mockMaestroClient, nil) + + mockMaestroClient.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, k8serrors.NewNotFound(workv1.Resource("manifestwork"), "test-bundle")) + + createdBundle := &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "new-bundle-name", + Namespace: "test-consumer", + UID: "new-bundle-id", + }, + } + mockMaestroClient.EXPECT().Create(gomock.Any(), gomock.Any(), gomock.Any()).Return(createdBundle, nil) + + err = syncer.SyncOnce(ctx, key) + require.NoError(t, err) + + updatedSPNP, err := spnpCRUD.Get(ctx, api.ServiceProviderNodePoolResourceName) + require.NoError(t, err) + require.NotNil(t, updatedSPNP) + + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool + bundleRef, err := updatedSPNP.Status.MaestroReadonlyBundles.Get(bundleInternalName) + require.NoError(t, err) + require.NotNil(t, bundleRef) + assert.NotEmpty(t, bundleRef.MaestroAPIMaestroBundleName) + assert.Equal(t, string(createdBundle.UID), bundleRef.MaestroAPIMaestroBundleID) +} + +func TestCreateNodePoolScopedMaestroReadonlyBundlesSyncer_SyncOnce_ProcessesPartiallySyncedBundles(t *testing.T) { + ctrl := gomock.NewController(t) + ctx := context.Background() + + mockDBClient := databasetesting.NewMockDBClient() + mockClusterService := ocm.NewMockClusterServiceClientSpec(ctrl) + mockMaestroBuilder := maestro.NewMockMaestroClientBuilder(ctrl) + mockMaestroClient := maestro.NewMockClient(ctrl) + + syncer := &createNodePoolScopedMaestroReadonlyBundlesSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + clusterServiceClient: mockClusterService, + maestroClientBuilder: mockMaestroBuilder, + maestroSourceEnvironmentIdentifier: "test-env", + maestroAPIMaestroBundleNameGenerator: maestro.NewMaestroAPIMaestroBundleNameGenerator(), + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + nodepoolsCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + { + Name: api.MaestroBundleInternalName("otherReadonlyBundle"), + MaestroAPIMaestroBundleName: "other-bundle-name", + MaestroAPIMaestroBundleID: "other-bundle-id", // fully synced - never touched + }, + { + Name: bundleInternalName, + MaestroAPIMaestroBundleName: "nodepool-bundle-name", + MaestroAPIMaestroBundleID: "", // partially synced - syncMaestroBundle will be called + }, + }, + }, + } + spnpCRUD := mockDBClient.ServiceProviderNodePools(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName, key.HCPNodePoolName) + _, err = spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + + provisionShard := buildTestProvisionShard("test-consumer") + mockClusterService.EXPECT(). + GetClusterProvisionShard(gomock.Any(), nodepool.ServiceProviderProperties.ClusterServiceID). + Return(provisionShard, nil) + + csCluster, err := arohcpv1alpha1.NewCluster().DomainPrefix("test-domain").Build() + require.NoError(t, err) + mockClusterService.EXPECT(). + GetCluster(gomock.Any(), nodepool.ServiceProviderProperties.ClusterServiceID). + Return(csCluster, nil) + + restEndpoint := provisionShard.MaestroConfig().RestApiConfig().Url() + grpcEndpoint := provisionShard.MaestroConfig().GrpcApiConfig().Url() + consumerName := provisionShard.MaestroConfig().ConsumerName() + sourceID := maestro.GenerateMaestroSourceID("test-env", provisionShard.ID()) + mockMaestroBuilder.EXPECT(). + NewClient(gomock.Any(), restEndpoint, grpcEndpoint, consumerName, sourceID). + Return(mockMaestroClient, nil) + + // Only the partially-synced bundle triggers a sync; maestro Get fails so we get an error + mockMaestroClient.EXPECT(). + Get(gomock.Any(), "nodepool-bundle-name", gomock.Any()). + Return(nil, fmt.Errorf("maestro API error")) + + err = syncer.SyncOnce(ctx, key) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to sync Maestro Bundle") + assert.Contains(t, err.Error(), "maestro API error") +} diff --git a/backend/pkg/controllers/cs_maestro_utils.go b/backend/pkg/controllers/cs_maestro_utils.go new file mode 100644 index 00000000000..4c71aa3b0ef --- /dev/null +++ b/backend/pkg/controllers/cs_maestro_utils.go @@ -0,0 +1,44 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "context" + + arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" + + "github.com/Azure/ARO-HCP/backend/pkg/maestro" +) + +// createMaestroClientFromCSProvisionShard creates a Maestro client for the given cluster provision shard. +// the client is scoped to the Consumer Name associated to the provision shard, and to +// the source ID associated to the provision shard and the environment specified +// in c.maestroSourceEnvironmentIdentifier, which is a configuration parameter at +// deployment time. +func createMaestroClientFromCSProvisionShard( + ctx context.Context, maestroSourceEnvironmentIdentifier string, maestroClientBuilder maestro.MaestroClientBuilder, clusterProvisionShard *arohcpv1alpha1.ProvisionShard, +) (maestro.Client, error) { + provisionShardMaestroConsumerName := clusterProvisionShard.MaestroConfig().ConsumerName() + provisionShardMaestroRESTAPIEndpoint := clusterProvisionShard.MaestroConfig().RestApiConfig().Url() + provisionShardMaestroGRPCAPIEndpoint := clusterProvisionShard.MaestroConfig().GrpcApiConfig().Url() + // This allows us to be able to have visibility on the Maestro Bundles owned by the same source ID for a given + // provision shard and environment. This should have the same source ID as what CS has in each corresponding environment + // because otherwise we would not have visibility on the Maestro Bundles owned + maestroSourceID := maestro.GenerateMaestroSourceID(maestroSourceEnvironmentIdentifier, clusterProvisionShard.ID()) + + maestroClient, err := maestroClientBuilder.NewClient(ctx, provisionShardMaestroRESTAPIEndpoint, provisionShardMaestroGRPCAPIEndpoint, provisionShardMaestroConsumerName, maestroSourceID) + + return maestroClient, err +} diff --git a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go index e7e8e60b858..fc2ea05c54e 100644 --- a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go +++ b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller.go @@ -26,9 +26,6 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/util/workqueue" - "k8s.io/utils/ptr" - - arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" "github.com/Azure/ARO-HCP/backend/pkg/maestro" @@ -54,7 +51,8 @@ type deleteOrphanedMaestroReadonlyBundles struct { maestroSourceEnvironmentIdentifier string } -// NewDeleteOrphanedMaestroReadonlyBundlesController periodically looks for cosmos objs that don't have an owning cluster and deletes them. +// NewDeleteOrphanedMaestroReadonlyBundlesController periodically looks for Maestro readonly bundles in the Maestro API that are not referenced +// by any of the supported cosmos resources by this controller and deletes them. func NewDeleteOrphanedMaestroReadonlyBundlesController(cosmosClient database.DBClient, csClient ocm.ClusterServiceClientSpec, maestroClientBuilder maestro.MaestroClientBuilder, maestroSourceEnvironmentIdentifier string) controllerutils.Controller { c := &deleteOrphanedMaestroReadonlyBundles{ name: "DeleteOrphanedMaestroReadonlyBundles", @@ -73,36 +71,39 @@ func NewDeleteOrphanedMaestroReadonlyBundlesController(cosmosClient database.DBC return c } -// SyncOnce current algorithm is: -// 1. List all ServiceProviderClusters (initial snapshot). -// 2. Build a map from Cluster Service provision shard ID to Maestro client (one client per registered provision shard). -// 3. Build initialShardToSPCs: map provision shard ID to the ServiceProviderClusters on that shard (from the initial list). -// This assumes the Maestro server for a provision shard uses a single Maestro source ID for resources we list; if not, -// client construction would need to change (Maestro Consumer Name + Maestro Source ID scope). -// 4. For each shard, list Maestro bundles (paginated, same label selector as today). A bundle is a delete candidate if -// it passes the readonly managed-by label filter and its name is not referenced by any SPC on that shard in initialShardToSPCs. -// Each candidate records the provision shard id and a pointer to the listed ManifestWork. -// 5. List all ServiceProviderClusters again (fresh snapshot), rebuild freshShardToSPCs the same way as (3). -// 6. For each candidate, if the bundle name is still not referenced on that shard in the fresh snapshot, delete it via Maestro -// +// SyncOnce algorithm: +// 1. Build a map from Cluster Service provision shard ID to Maestro client (one client per registered provision shard). +// 2. For each resource type that contains Maestro readonly bundles that we decide to include here: +// 2.1 List all documents (initial snapshot). +// 2.2 Build a map from Cluster Service provision shard ID to the documents on that shard (from the initial list). +// 2.3 For each shard, list Maestro bundles (paginated, and with a label selector that filters for the readonly managed-by label associated to +// the specifc resource type we are processing). A bundle is a delete candidate if it passes the resource-scoped readonly managed-by label +// filter and its name is not referenced by any document on that shard in the initial map. Each candidate records the provision shard +// id and a pointer to the listed ManifestWork. +// 2.4 List all documents again (fresh snapshot), rebuild the map from it in the same way as 2.2. +// 2.5 For each candidate, if the bundle name is still not referenced on that shard in the fresh snapshot, delete it via Maestro +// bundle deletion API. + // Cross-store: The fresh SPC list and per-shard reference set (steps 5-6) prevent deleting a bundle that is already referenced // in committed Cosmos documents by the time that snapshot is built, so a stale initial list alone does not cause accidental // delete. - +// // IMPORTANT NOTE: This assumes that the maestro server associated to the provision shard // has resources with always the same source ID. If it turns out we cannot have this assumption this logic would not // be good enough. In that case it might be necessary to store to what source ID a Maestro Bundle/set of Maestro Bundles // belongs to but then the instantiation of the Maestro client needs to be done differently as its scoped to // Maestro Consumer Name + Maestro Source ID. We know for example that in the CSPR environment different CS instances // have different Maestro source IDs using the same Maestro Server. +// +// Note: We considered using the Maestro API Maestro UID which is globally unique but it's possible that there's a scenario +// where a maestro create readonly bundles controller creates a bundle, then creates the Maestro bundle using the Maestro API +// but then for some reason fails to persist it in the database, which in that case the cluster ended up being deleted by the +// orphan controller accidentally. In that scenario we would not have the Maestro UID to identify the Maestro Bundle and +// we would not be able to delete it. Furthermore we should not use the fact of the UID being empty as the trigger +// to delete because it could be that it's being created and not yet persisted in Cosmos. func (c *deleteOrphanedMaestroReadonlyBundles) SyncOnce(ctx context.Context, _ any) error { logger := utils.LoggerFromContext(ctx) logger.Info("Syncing orphaned Maestro Readonly Bundles") - initialServiceProviderClusters, err := c.getAllServiceProviderClusters(ctx) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to get all ServiceProviderClusters: %w", err)) - } - logger.Info(fmt.Sprintf("Found %d ServiceProviderClusters (initial)", len(initialServiceProviderClusters))) logger.Info("Building Maestro clients per Cluster Service provision shard") maestroClientsByShard, err := c.buildMaestroClientsByProvisionShard(ctx) @@ -113,55 +114,60 @@ func (c *deleteOrphanedMaestroReadonlyBundles) SyncOnce(ctx context.Context, _ a } logger.Info(fmt.Sprintf("Built Maestro clients for %d provision shards", len(maestroClientsByShard))) - logger.Info("Mapping initial ServiceProviderClusters to provision shards") - initialShardToSPCs, err := c.mapServiceProviderClustersByProvisionShard(ctx, initialServiceProviderClusters, maestroClientsByShard) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to map ServiceProviderClusters to provision shards: %w", err)) + var syncErrors []error + + logger.Info("Ensuring orphaned cluster scoped Maestro Readonly Bundles are deleted") + if err := c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, maestroClientsByShard); err != nil { + syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to ensure orphaned cluster-scoped Maestro Bundles are deleted: %w", err))) } - logger.Info(fmt.Sprintf("Initial ServiceProviderClusters mapped to %d provision shards", len(initialShardToSPCs))) - logger.Info("Ensuring orphaned Maestro Readonly Bundles are deleted") - err = c.ensureOrphanedMaestroReadonlyBundlesAreDeleted(ctx, maestroClientsByShard, initialShardToSPCs) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to ensure orphaned Maestro Bundles are deleted: %w", err)) + logger.Info("Ensuring orphaned nodepool scoped Maestro Readonly Bundles are deleted") + if err := c.ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted(ctx, maestroClientsByShard); err != nil { + syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to ensure orphaned nodepool-scoped Maestro Bundles are deleted: %w", err))) } + logger.Info("End of orphaned Maestro Readonly Bundles sync") + return errors.Join(syncErrors...) +} - return nil +// ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted ensures that Maestro readonly bundles managed by the cluster-scoped +// controller are deleted when no ServiceProviderCluster on that provision shard references them. +func (c *deleteOrphanedMaestroReadonlyBundles) ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient) error { + logger := utils.LoggerFromContext(ctx) + logger = logger.WithValues("maestroReadonlyBundleReferencesResourceType", api.ServiceProviderClusterResourceType) + ctx = utils.ContextWithLogger(ctx, logger) + + return c.ensureOrphanedReadonlyBundlesDeleted(ctx, maestroClientsByShard, readonlyBundleManagedByK8sLabelValueClusterScoped, c.clusterScopedPersistedMaestroBundleRefsByShardFromCosmos) +} + +// ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted ensures that Maestro readonly bundles managed by the +// nodepool-scoped controller are deleted when no ServiceProviderNodePool on that provision shard references them. +func (c *deleteOrphanedMaestroReadonlyBundles) ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient) error { + logger := utils.LoggerFromContext(ctx) + logger = logger.WithValues("maestroReadonlyBundleReferencesResourceType", api.ServiceProviderNodePoolResourceType) + ctx = utils.ContextWithLogger(ctx, logger) + + return c.ensureOrphanedReadonlyBundlesDeleted(ctx, maestroClientsByShard, readonlyBundleManagedByK8sLabelValueNodePoolScoped, c.nodePoolScopedPersistedMaestroBundleRefsByShardFromCosmos) } -// getAllServiceProviderClusters returns the list of all ServiceProviderClusters in the database. +// getAllServiceProviderClusters returns all ServiceProviderClusters via database.ListAll. func (c *deleteOrphanedMaestroReadonlyBundles) getAllServiceProviderClusters(ctx context.Context) ([]*api.ServiceProviderCluster, error) { - // We list all ServiceProviderClusters in chunks of 500 to avoid putting + // We list all ServiceProviderClusters in chunks of 500 at most to avoid putting // too much pressure on the Cosmos DB. // Any failure to iterate over the ServiceProviderclusters ends the sync process because otherwise // we would not have the complete information to evaluate the deletion and we could // accidentally delete Maestro Bundles that are still in use. - listOptions := &database.DBClientListResourceDocsOptions{ - PageSizeHint: ptr.To(int32(500)), - } - allServiceProviderClusters := []*api.ServiceProviderCluster{} - for { - iterator, err := c.cosmosClient.GlobalListers().ServiceProviderClusters().List(ctx, listOptions) - if err != nil { - return nil, utils.TrackError(fmt.Errorf("failed to list ServiceProviderClusters: %w", err)) - } - for _, spc := range iterator.Items(ctx) { - allServiceProviderClusters = append(allServiceProviderClusters, spc) - } - err = iterator.GetError() - if err != nil { - return nil, utils.TrackError(fmt.Errorf("failed iterating ServiceProviderClusters: %w", err)) - } - - continuationToken := iterator.GetContinuationToken() - if continuationToken == "" { - break - } - listOptions.ContinuationToken = &continuationToken - } + return database.ListAll(ctx, 500, c.cosmosClient.GlobalListers().ServiceProviderClusters().List) +} - return allServiceProviderClusters, nil +// getAllServiceProviderNodePools returns all ServiceProviderNodePools via database.ListAll. +func (c *deleteOrphanedMaestroReadonlyBundles) getAllServiceProviderNodePools(ctx context.Context) ([]*api.ServiceProviderNodePool, error) { + // We list all ServiceProviderNodePools in chunks of 500 at most to avoid putting + // too much pressure on the Cosmos DB. + // Any failure to iterate over the ServiceProviderNodePools ends the sync process because otherwise + // we would not have the complete information to evaluate the deletion and we could + // accidentally delete Maestro Bundles that are still in use. + return database.ListAll(ctx, 500, c.cosmosClient.GlobalListers().ServiceProviderNodePools().List) } // shardMaestroClient holds a Maestro API client for one Cluster Service provision shard and its teardown cancel func. @@ -179,7 +185,6 @@ func cancelMaestroClientsByProvisionShard(maestroClientsByProvisionShard map[str // buildMaestroClientsByProvisionShard lists registered provision shards from Cluster Service and builds a map of // provision shard ID to Maestro client. The key of the map is the CS provision shard ID. -// // On error the returned map may be partial (clients created before the error). The caller must defer cancelMaestroClientsByProvisionShard unconditionally. func (c *deleteOrphanedMaestroReadonlyBundles) buildMaestroClientsByProvisionShard(ctx context.Context) (map[string]*shardMaestroClient, error) { maestroClientsByProvisionShard := map[string]*shardMaestroClient{} @@ -188,11 +193,12 @@ func (c *deleteOrphanedMaestroReadonlyBundles) buildMaestroClientsByProvisionSha // the information in Cosmos and this should be changed to use that instead. // TODO should we take into account the provision shard status on what to consider (active, maintenance, offline, ...)? // for now we consider all provision shards independently of their status. - for provisionShard := range c.clusterServiceClient.ListProvisionShards().Items(ctx) { + provisionShardIter := c.clusterServiceClient.ListProvisionShards() + for provisionShard := range provisionShardIter.Items(ctx) { // We create a new context with a cancel function so we can cancel the Maestro client when the sync is done. // This is important to avoid leaking resources when the sync is done. maestroClientCtx, cancel := context.WithCancel(ctx) - maestroClient, err := c.createMaestroClientFromProvisionShard(maestroClientCtx, provisionShard) + maestroClient, err := createMaestroClientFromCSProvisionShard(maestroClientCtx, c.maestroSourceEnvironmentIdentifier, c.maestroClientBuilder, provisionShard) if err != nil { cancel() // on error creating the Maestro client we ensure we cancel the context that we just created too return maestroClientsByProvisionShard, utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) @@ -202,19 +208,32 @@ func (c *deleteOrphanedMaestroReadonlyBundles) buildMaestroClientsByProvisionSha maestroClientCancelFunc: cancel, } } + err := provisionShardIter.GetError() + if err != nil { + return maestroClientsByProvisionShard, utils.TrackError(fmt.Errorf("failed to list Cluster Service provision shards: %w", err)) + } return maestroClientsByProvisionShard, nil } // mapServiceProviderClustersByProvisionShard groups ServiceProviderClusters by Cluster Service provision shard ID. // Every resolved shard must exist in maestroClientsByShard (registered provision shards). +// ServiceProviderClusters whose parent cluster has no ClusterServiceID yet (pre–Cluster Service registration) are skipped +// so the syncer doesn't fail if there are some resources that still don't have it set func (c *deleteOrphanedMaestroReadonlyBundles) mapServiceProviderClustersByProvisionShard(ctx context.Context, spcs []*api.ServiceProviderCluster, maestroClientsByShard map[string]*shardMaestroClient) (map[string][]*api.ServiceProviderCluster, error) { res := make(map[string][]*api.ServiceProviderCluster) for _, spc := range spcs { - shardID, err := c.clusterProvisionShardIDForServiceProviderCluster(ctx, spc) + shardID, skip, err := c.clusterProvisionShardIDForServiceProviderCluster(ctx, spc) if err != nil { return nil, err } + if skip { + // It should be safe to skip those because if a maestro bundle in the maestro API exists it means that there should be a corresponding Cosmos + // resource with a maestro bundle reference. If for some reason during the orphan calculation inbetween the first read of cosmos + // resources and the read in the Maestro api there's a new bundle in maestro, the second read of cosmos resources will catch that + // and prevent accidental deletion. + continue + } if _, ok := maestroClientsByShard[shardID]; !ok { return nil, utils.TrackError(fmt.Errorf("provision shard %s for ServiceProviderCluster %s is not present in provision shards map", shardID, spc.ResourceID.String())) } @@ -223,84 +242,177 @@ func (c *deleteOrphanedMaestroReadonlyBundles) mapServiceProviderClustersByProvi return res, nil } +// mapServiceProviderNodePoolsByProvisionShard groups ServiceProviderNodePools by Cluster Service provision shard ID +// of their owning cluster. Every resolved shard must exist in maestroClientsByShard (registered provision shards). +// ServiceProviderNodePools whose parent cluster has no ClusterServiceID yet (pre–Cluster Service registration) are skipped +// so the syncer doesn't fail if there are some resources that still don't have it set. +func (c *deleteOrphanedMaestroReadonlyBundles) mapServiceProviderNodePoolsByProvisionShard(ctx context.Context, spnps []*api.ServiceProviderNodePool, maestroClientsByShard map[string]*shardMaestroClient) (map[string][]*api.ServiceProviderNodePool, error) { + res := make(map[string][]*api.ServiceProviderNodePool) + for _, spnp := range spnps { + shardID, skip, err := c.clusterProvisionShardIDForServiceProviderNodePool(ctx, spnp) + if err != nil { + return nil, err + } + if skip { + // It should be safe to skip those because if a maestro bundle in the maestro API exists it means that there should be a corresponding Cosmos + // resource with a maestro bundle reference. If for some reason during the orphan calculation inbetween the first read of cosmos + // resources and the read in the Maestro api there's a new bundle in maestro, the second read of cosmos resources will catch that + // and prevent accidental deletion. + continue + } + if _, ok := maestroClientsByShard[shardID]; !ok { + return nil, utils.TrackError(fmt.Errorf("provision shard %s for ServiceProviderNodePool %s is not present in provision shards map", shardID, spnp.ResourceID.String())) + } + res[shardID] = append(res[shardID], spnp) + } + return res, nil +} + +// maestroBundleNamesByShard maps Cluster Service provision shard IDs to a set of Maestro API Maestro bundle names +// The outer map key is the Cluster Service provision shard ID. The inner map key is the Maestro API Maestro bundle name. +// The inner map value is a struct{} to indicate the presence of the bundle name. +type maestroBundleNamesByShard map[string]map[string]struct{} + +// maestroBundleNamesByShardRetrieverFunc retrieves a map of Cluster Service provision shard IDs to a set of Maestro API Maestro bundle names. +type maestroBundleNamesByShardRetrieverFunc func(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient) (bundleNamesByShard maestroBundleNamesByShard, err error) + // orphanReadonlyBundleDeleteCandidate is a Maestro bundle listed on a provision shard that was not referenced by the -// initial SPC snapshot for that shard; delete still requires a fresh snapshot check. +// initial snapshot for that shard; delete still requires a fresh snapshot check. type orphanReadonlyBundleDeleteCandidate struct { csShardID string bundle *workv1.ManifestWork } -// ensureOrphanedMaestroReadonlyBundlesAreDeleted ensures that Maestro readonly bundles managed by the cluster-scoped -// controller are deleted when no ServiceProviderCluster on that provision shard references them. -// -// 1. From initialShardToSPCs, build per-shard sets of referenced Maestro bundle names. -// 2. For each shard with a Maestro client, list bundles (paginated) and add candidates when the bundle is not referenced on that shard. -// 3. List all ServiceProviderClusters again (fresh), map them by shard, rebuild referenced sets. -// 4. Delete each candidate that is still unreferenced on its shard in the fresh snapshot. -func (c *deleteOrphanedMaestroReadonlyBundles) ensureOrphanedMaestroReadonlyBundlesAreDeleted(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient, initialShardToSPCs map[string][]*api.ServiceProviderCluster) error { +// listOrphanReadonlyBundleCandidates lists Maestro bundles for each shard using the Maestro API +// and it returns a list of candidate orphan maestro readonly bundles for deletion. The criteria to consider a +// maestro readonly bundle as a candidate for deletion is that it matches the managedByLabelValue and is not +// referenced in bundleNamesByShard for that shard. +func (c *deleteOrphanedMaestroReadonlyBundles) listOrphanReadonlyBundleCandidates(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient, + bundleNamesByShard maestroBundleNamesByShard, managedByLabelValue string, +) ([]orphanReadonlyBundleDeleteCandidate, error) { logger := utils.LoggerFromContext(ctx) - var syncErrors []error - - referencedByShardInitial, err := referencedMaestroAPIMaestroBundleNamesByShard(initialShardToSPCs) - if err != nil { - return utils.TrackError(fmt.Errorf("error building referenced Maestro API Maestro bundle names by shard (initial snapshot): %w", err)) - } - var deleteCandidates []orphanReadonlyBundleDeleteCandidate for csShardID, shardEntry := range maestroClientsByShard { shardLogger := logger.WithValues("csProvisionShardID", csShardID) ctxShard := utils.ContextWithLogger(ctx, shardLogger) - initialOnShard := initialShardToSPCs[csShardID] - shardLogger.Info(fmt.Sprintf("listing Maestro bundles on cluster service provision shard %s (%d ServiceProviderClusters in initial shard map)", csShardID, len(initialOnShard))) maestroClient := shardEntry.maestroClient - listOptions := metav1.ListOptions{Limit: 400, Continue: "", LabelSelector: fmt.Sprintf("%s=%s", readonlyBundleManagedByK8sLabelKey, readonlyBundleManagedByK8sLabelValueClusterScoped)} - for { - maestroBundles, err := maestroClient.List(ctxShard, listOptions) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to list Maestro Bundles for shard %s: %w", csShardID, err)) + // We list all the Maestro Bundles in chunks of 400 to avoid putting too much pressure on the Maestro API. + // We filter by the K8s label that identifies which controller manages the bundle. + listOpts := metav1.ListOptions{ + Limit: 400, + LabelSelector: fmt.Sprintf("%s=%s", readonlyBundleManagedByK8sLabelKey, managedByLabelValue), + } + err := maestro.ForEachMaestroBundle(ctxShard, maestroClient, listOpts, func(maestroBundle *workv1.ManifestWork) error { + // Even though Maestro should filter by the K8s label we specified we double check it here to be sure. + if maestroBundle.Labels[readonlyBundleManagedByK8sLabelKey] != managedByLabelValue { + return nil } - for i := range maestroBundles.Items { - maestroBundle := &maestroBundles.Items[i] - // Even though Maestro should filter by the K8s label we specified we double check it here to be sure - if maestroBundle.Labels[readonlyBundleManagedByK8sLabelKey] != readonlyBundleManagedByK8sLabelValueClusterScoped { - continue + // Check if the bundle is referenced by any resource allocated to this shard. + if shardRefSet := bundleNamesByShard[csShardID]; shardRefSet != nil { + if _, referenced := shardRefSet[maestroBundle.Name]; referenced { + // The Maestro API Maestro Bundle Name should be unique within a given Maestro Consumer Name and Maestro Source ID. + // If we find a match, it means the Maestro Bundle is referenced and we should not delete it. + return nil } - // We check if the Maestro bundle is referenced by any of the ServiceProviderClusters on the shard in the initial snapshot. - // If it is referenced we skip it as it is not an orphan. - // The Maestro API Maestro Bundle Name should be unique within a given Maestro Consumer Name and Maestro Source ID. - if shardRefSet := referencedByShardInitial[csShardID]; shardRefSet != nil { - if _, referenced := shardRefSet[maestroBundle.Name]; referenced { - continue - } - } - deleteCandidates = append(deleteCandidates, orphanReadonlyBundleDeleteCandidate{ - csShardID: csShardID, - bundle: maestroBundle, - }) - } - continuationToken := maestroBundles.GetContinue() - if continuationToken == "" { - break } - listOptions.Continue = continuationToken + deleteCandidates = append(deleteCandidates, orphanReadonlyBundleDeleteCandidate{ + csShardID: csShardID, + bundle: maestroBundle, + }) + return nil + }) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to list Maestro Bundles for shard %s: %w", csShardID, err)) } } - freshServiceProviderClusters, err := c.getAllServiceProviderClusters(ctx) + return deleteCandidates, nil +} + +// clusterScopedPersistedMaestroBundleRefsByShardFromCosmos lists ServiceProviderClusters from Cosmos, maps them by +// provision shard, and returns referenced Maestro API bundle names per shard. +func (c *deleteOrphanedMaestroReadonlyBundles) clusterScopedPersistedMaestroBundleRefsByShardFromCosmos(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient) (maestroBundleNamesByShard, error) { + clusters, err := c.getAllServiceProviderClusters(ctx) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to get all ServiceProviderClusters: %w", err)) + } + shardDocs, err := c.mapServiceProviderClustersByProvisionShard(ctx, clusters, maestroClientsByShard) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to map ServiceProviderClusters to provision shards: %w", err)) + } + refs, err := c.buildClusterScopedMaestroAPIMaestroBundleNamesByShard(shardDocs) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("error building cluster scoped Maestro API Maestro bundle names by shard: %w", err)) + } + return refs, nil +} + +// nodePoolScopedPersistedMaestroBundleRefsByShardFromCosmos lists ServiceProviderNodePools from Cosmos, maps them by +// provision shard, and returns referenced Maestro API bundle names per shard. +func (c *deleteOrphanedMaestroReadonlyBundles) nodePoolScopedPersistedMaestroBundleRefsByShardFromCosmos(ctx context.Context, maestroClientsByShard map[string]*shardMaestroClient) (maestroBundleNamesByShard, error) { + pools, err := c.getAllServiceProviderNodePools(ctx) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to get all ServiceProviderNodePools: %w", err)) + } + shardDocs, err := c.mapServiceProviderNodePoolsByProvisionShard(ctx, pools, maestroClientsByShard) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to map ServiceProviderNodePools to provision shards: %w", err)) + } + refs, err := c.buildNodePoolScopedMaestroAPIMaestroBundleNamesByShard(shardDocs) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("error building nodepool scoped Maestro API Maestro bundle names by shard: %w", err)) + } + return refs, nil +} + +// ensureOrphanedReadonlyBundlesDeleted runs the two-phase list/compare/delete flow shared by every Cosmos-backed resource type +// that can reference Maestro readonly bundles: +// 1. Initial Cosmos snapshot of all the instances of that resource type that can reference Maestro readonly bundles +// 2. For each shard, use its Maestro client and list the maestro bundles including them as deletion candidates +// when the bundle is not referenced on that shard and the readonlyBundleManagedByK8sLabelKey label matches the managedByLabelValue +// 3. Retrieve a fresh snapshot of all the instances of that resource type that can reference Maestro readonly bundles +// 4. Delete each candidate that is still unreferenced on that shard in the fresh snapshot +func (c *deleteOrphanedMaestroReadonlyBundles) ensureOrphanedReadonlyBundlesDeleted( + ctx context.Context, + maestroClientsByShard map[string]*shardMaestroClient, + managedByLabelValue string, + persistedMaestroBundleRefsByShardRetriever maestroBundleNamesByShardRetrieverFunc, +) error { + initialPersistedMaestroBundlesByShard, err := persistedMaestroBundleRefsByShardRetriever(ctx, maestroClientsByShard) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to retrieve initial persisted Maestro bundle references by shard: %w", err)) + } + + deleteCandidates, err := c.listOrphanReadonlyBundleCandidates(ctx, maestroClientsByShard, initialPersistedMaestroBundlesByShard, managedByLabelValue) if err != nil { - return utils.TrackError(fmt.Errorf("error getting all ServiceProviderClusters (fresh snapshot): %w", err)) + return utils.TrackError(fmt.Errorf("failed to list orphaned Maestro readonly bundle candidates: %w", err)) } - freshShardToSPCs, err := c.mapServiceProviderClustersByProvisionShard(ctx, freshServiceProviderClusters, maestroClientsByShard) + + freshPersistedMaestroBundlesByShard, err := persistedMaestroBundleRefsByShardRetriever(ctx, maestroClientsByShard) if err != nil { - return utils.TrackError(fmt.Errorf("error mapping fresh ServiceProviderClusters to provision shards (fresh snapshot): %w", err)) + return utils.TrackError(fmt.Errorf("failed to retrieve fresh persisted Maestro bundle references by shard: %w", err)) } - referencedByShardFresh, err := referencedMaestroAPIMaestroBundleNamesByShard(freshShardToSPCs) + + err = c.conditionallyDeleteOrphanReadonlyBundleCandidates(ctx, maestroClientsByShard, deleteCandidates, freshPersistedMaestroBundlesByShard) if err != nil { - return utils.TrackError(fmt.Errorf("error building referenced Maestro API Maestro bundle names by shard (fresh snapshot): %w", err)) + return utils.TrackError(fmt.Errorf("failed to delete orphaned Maestro readonly bundle candidates: %w", err)) } - for _, cand := range deleteCandidates { + return nil +} + +// conditionallyDeleteOrphanReadonlyBundleCandidates processes a list of Maestro readonly bundle delete candidates. An +// orphan Maestro readonly bundle delete candidate is deleted only if it is not referenced in persistedMaestroBundlesByShard +// for the same shard. +func (c *deleteOrphanedMaestroReadonlyBundles) conditionallyDeleteOrphanReadonlyBundleCandidates( + ctx context.Context, + maestroClientsByShard map[string]*shardMaestroClient, + candidates []orphanReadonlyBundleDeleteCandidate, + persistedMaestroBundlesByShard maestroBundleNamesByShard, +) error { + var syncErrors []error + for _, cand := range candidates { csShardID := cand.csShardID candidateMaestroBundle := cand.bundle shardEntry, ok := maestroClientsByShard[csShardID] @@ -312,15 +424,15 @@ func (c *deleteOrphanedMaestroReadonlyBundles) ensureOrphanedMaestroReadonlyBund shardLogger := utils.LoggerFromContext(ctx).WithValues("csProvisionShardID", csShardID) ctxShard := utils.ContextWithLogger(ctx, shardLogger) - if shardRefSet := referencedByShardFresh[csShardID]; shardRefSet != nil { + if shardRefSet := persistedMaestroBundlesByShard[csShardID]; shardRefSet != nil { if _, referenced := shardRefSet[candidateMaestroBundle.Name]; referenced { - // If the Maestro bundle is referenced by any of the ServiceProviderClusters on the shard in the fresh snapshot we skip it as it is not an orphan. + // If the Maestro bundle is referenced by any persisted Maestro bundle references for that shard, skip it. continue } } shardLogger.Info("Deleting orphaned Maestro readonly Bundle", "maestroConsumerName", candidateMaestroBundle.Namespace, "maestroAPIMaestroBundleName", candidateMaestroBundle.Name, "maestroAPIMaestroBundleID", candidateMaestroBundle.UID) - err = maestroClient.Delete(ctxShard, candidateMaestroBundle.Name, metav1.DeleteOptions{}) + err := maestroClient.Delete(ctxShard, candidateMaestroBundle.Name, metav1.DeleteOptions{}) if err != nil { // Failure to delete does not end the sync process. We log the error and we continue with the processing of other Maestro bundle deletion candidates. syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to delete Maestro Bundle: %w", err))) @@ -328,19 +440,46 @@ func (c *deleteOrphanedMaestroReadonlyBundles) ensureOrphanedMaestroReadonlyBund shardLogger.Info("Deleted orphaned Maestro readonly Bundle", "maestroConsumerName", candidateMaestroBundle.Namespace, "maestroAPIMaestroBundleName", candidateMaestroBundle.Name, "maestroAPIMaestroBundleID", candidateMaestroBundle.UID) } } - - return errors.Join(syncErrors...) + return utils.TrackError(errors.Join(syncErrors...)) } // clusterProvisionShardIDForServiceProviderCluster returns the Cluster Service provision shard ID for the cluster that owns the SPC. -func (c *deleteOrphanedMaestroReadonlyBundles) clusterProvisionShardIDForServiceProviderCluster(ctx context.Context, spc *api.ServiceProviderCluster) (string, error) { +// skip is true when the parent cluster has no ClusterServiceID yet. +func (c *deleteOrphanedMaestroReadonlyBundles) clusterProvisionShardIDForServiceProviderCluster(ctx context.Context, spc *api.ServiceProviderCluster) (shardID string, skip bool, err error) { clusterResourceID := spc.ResourceID.Parent if clusterResourceID == nil { - return "", utils.TrackError(fmt.Errorf("ServiceProviderCluster %s has no parent resource ID", spc.ResourceID.String())) + return "", false, utils.TrackError(fmt.Errorf("ServiceProviderCluster %s has no parent resource ID", spc.ResourceID.String())) } cluster, err := c.cosmosClient.HCPClusters(clusterResourceID.SubscriptionID, clusterResourceID.ResourceGroupName).Get(ctx, clusterResourceID.Name) if err != nil { - return "", utils.TrackError(fmt.Errorf("failed to get Cluster: %w", err)) + return "", false, utils.TrackError(fmt.Errorf("failed to get Cluster: %w", err)) + } + return c.provisionShardIDFromCluster(ctx, cluster) +} + +// clusterProvisionShardIDForServiceProviderNodePool returns the Cluster Service provision shard ID for the cluster that owns the node pool. +// skip is true when the parent cluster has no ClusterServiceID yet. +func (c *deleteOrphanedMaestroReadonlyBundles) clusterProvisionShardIDForServiceProviderNodePool(ctx context.Context, spnp *api.ServiceProviderNodePool) (shardID string, skip bool, err error) { + nodePoolResourceID := spnp.ResourceID.Parent + if nodePoolResourceID == nil { + return "", false, utils.TrackError(fmt.Errorf("ServiceProviderNodePool %s has no parent resource ID", spnp.ResourceID.String())) + } + clusterResourceID := nodePoolResourceID.Parent + if clusterResourceID == nil { + return "", false, utils.TrackError(fmt.Errorf("ServiceProviderNodePool %s has no grandparent cluster resource ID", spnp.ResourceID.String())) + } + cluster, err := c.cosmosClient.HCPClusters(clusterResourceID.SubscriptionID, clusterResourceID.ResourceGroupName).Get(ctx, clusterResourceID.Name) + if err != nil { + return "", false, utils.TrackError(fmt.Errorf("failed to get Cluster: %w", err)) + } + return c.provisionShardIDFromCluster(ctx, cluster) +} + +// provisionShardIDFromCluster resolves the provision shard for a Cosmos cluster document. skip is true when ClusterServiceID +// is unset so the cluster is not yet registered with Cluster Service (same gate as create-*-scoped Maestro bundle controllers). +func (c *deleteOrphanedMaestroReadonlyBundles) provisionShardIDFromCluster(ctx context.Context, cluster *api.HCPOpenShiftCluster) (shardID string, skip bool, err error) { + if len(cluster.ServiceProviderProperties.ClusterServiceID.String()) == 0 { + return "", true, nil } // TODO We get the provision shard ID from CS but at some point we should have // the information in Cosmos and this should be changed to use that instead. @@ -349,16 +488,16 @@ func (c *deleteOrphanedMaestroReadonlyBundles) clusterProvisionShardIDForService // we assume that the cluster is associated to a single provision shard at a time. clusterCSShard, err := c.clusterServiceClient.GetClusterProvisionShard(ctx, cluster.ServiceProviderProperties.ClusterServiceID) if err != nil { - return "", utils.TrackError(fmt.Errorf("failed to get Cluster Provision Shard: %w", err)) + return "", false, utils.TrackError(fmt.Errorf("failed to get Cluster Provision Shard: %w", err)) } - return clusterCSShard.ID(), nil + return clusterCSShard.ID(), false, nil } -// referencedMaestroAPIMaestroBundleNamesByShard maps provision shard ID to the set of Maestro API bundle names referenced by -// SPCs grouped under that shard (shard assignment is already resolved in spcsByShard). Nil list entries or empty +// buildClusterScopedMaestroAPIMaestroBundleNamesByShard maps provision shard ID to the set of Maestro API bundle names referenced by +// ServiceProviderClusters grouped under that shard (shard assignment is already resolved in spcsByShard). Nil list entries or empty // maestroAPIMaestroBundleName return an error so the reference set cannot silently omit in-use bundles. -func referencedMaestroAPIMaestroBundleNamesByShard(spcsByShard map[string][]*api.ServiceProviderCluster) (map[string]map[string]struct{}, error) { - out := make(map[string]map[string]struct{}) +func (c *deleteOrphanedMaestroReadonlyBundles) buildClusterScopedMaestroAPIMaestroBundleNamesByShard(spcsByShard map[string][]*api.ServiceProviderCluster) (maestroBundleNamesByShard, error) { + out := make(maestroBundleNamesByShard) for shardID, spcs := range spcsByShard { // If it is the first time we are processing this shard we initialize the map entry for it @@ -385,6 +524,37 @@ func referencedMaestroAPIMaestroBundleNamesByShard(spcsByShard map[string][]*api return out, nil } +// buildNodePoolScopedMaestroAPIMaestroBundleNamesByShard builds a map of provision shard ID to the set of Maestro API bundle names referenced by +// ServiceProviderNodePools grouped under that shard (shard assignment is already resolved in spnpsByShard). Nil list entries or empty +// maestroAPIMaestroBundleName return an error so the reference set cannot silently omit in-use bundles. +func (c *deleteOrphanedMaestroReadonlyBundles) buildNodePoolScopedMaestroAPIMaestroBundleNamesByShard(spnpsByShard map[string][]*api.ServiceProviderNodePool) (maestroBundleNamesByShard, error) { + out := make(maestroBundleNamesByShard) + + for shardID, spnps := range spnpsByShard { + // If it is the first time we are processing this shard we initialize the map entry for it + if out[shardID] == nil { + out[shardID] = make(map[string]struct{}) + } + // We iterate over the ServiceProviderNodePools on the shard and we add the Maestro API Maestro bundle names to the map. + for _, spnp := range spnps { + if spnp == nil { + return nil, utils.TrackError(fmt.Errorf("nil ServiceProviderNodePool under provision shard %s", shardID)) + } + for i, ref := range spnp.Status.MaestroReadonlyBundles { + if ref == nil { + return nil, utils.TrackError(fmt.Errorf("serviceProviderNodePool %s: MaestroReadonlyBundles[%d] is nil", spnp.ResourceID.String(), i)) + } + if ref.MaestroAPIMaestroBundleName == "" { + return nil, utils.TrackError(fmt.Errorf("serviceProviderNodePool %s: MaestroReadonlyBundles[%d] (internal name %q) has empty maestroAPIMaestroBundleName", spnp.ResourceID.String(), i, ref.Name)) + } + out[shardID][ref.MaestroAPIMaestroBundleName] = struct{}{} + } + } + } + + return out, nil +} + func (c *deleteOrphanedMaestroReadonlyBundles) Run(ctx context.Context, threadiness int) { // don't let panics crash the process defer utilruntime.HandleCrash() @@ -441,25 +611,3 @@ func (c *deleteOrphanedMaestroReadonlyBundles) processNextWorkItem(ctx context.C return true } - -// createMaestroClientFromProvisionShard creates a Maestro client for the given provision shard. -// The client is scoped to the Maestro Consumer associated to the provision shard, as well -// as to the the Maestro Source ID associated to the provision shard which is calculated from the provision shard ID and the -// environment specified in c.maestroSourceEnvironmentIdentifier. -func (c *deleteOrphanedMaestroReadonlyBundles) createMaestroClientFromProvisionShard( - ctx context.Context, provisionShard *arohcpv1alpha1.ProvisionShard, -) (maestro.Client, error) { - provisionShardMaestroConsumerName := provisionShard.MaestroConfig().ConsumerName() - provisionShardMaestroRESTAPIEndpoint := provisionShard.MaestroConfig().RestApiConfig().Url() - provisionShardMaestroGRPCAPIEndpoint := provisionShard.MaestroConfig().GrpcApiConfig().Url() - // This allows us to be able to have visibility on the Maestro Bundles owned by the same source ID for a given - // provision shard and environment. This should have the same source ID as what CS has in each corresponding environment - // because otherwise we would not have visibility on the Maestro Bundles owned - // TODO do we want to use the same source ID that CS uses or do we want intentionally a different one? This has consequences - // on the visibility of the Maestro Bundles, including processing of events sent by Maestro. - maestroSourceID := maestro.GenerateMaestroSourceID(c.maestroSourceEnvironmentIdentifier, provisionShard.ID()) - - maestroClient, err := c.maestroClientBuilder.NewClient(ctx, provisionShardMaestroRESTAPIEndpoint, provisionShardMaestroGRPCAPIEndpoint, provisionShardMaestroConsumerName, maestroSourceID) - - return maestroClient, err -} diff --git a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller_test.go b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller_test.go index b3923d7d3a0..d94d92ec828 100644 --- a/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller_test.go +++ b/backend/pkg/controllers/delete_orphaned_maestro_readonly_bundles_controller_test.go @@ -41,55 +41,7 @@ import ( "github.com/Azure/ARO-HCP/internal/utils" ) -func TestDeleteOrphanedMaestroReadonlyBundles_getAllServiceProviderClusters(t *testing.T) { - ctx := context.Background() - clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) - spcResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/serviceProviderClusters/default")) - - tests := []struct { - name string - setupDB func(t *testing.T, ctx context.Context, mockDB *databasetesting.MockDBClient) - wantLen int - wantFirstResourceID string - }{ - { - name: "empty DB returns no SPCs", - setupDB: nil, - wantLen: 0, - }, - { - name: "returns SPCs created via CRUD", - setupDB: func(t *testing.T, ctx context.Context, mockDB *databasetesting.MockDBClient) { - spc := &api.ServiceProviderCluster{ - CosmosMetadata: arm.CosmosMetadata{ResourceID: spcResourceID}, - ResourceID: *spcResourceID, - } - spcCRUD := mockDB.ServiceProviderClusters(clusterResourceID.SubscriptionID, clusterResourceID.ResourceGroupName, clusterResourceID.Name) - _, err := spcCRUD.Create(ctx, spc, nil) - require.NoError(t, err) - }, - wantLen: 1, - wantFirstResourceID: spcResourceID.String(), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mockDB := databasetesting.NewMockDBClient() - if tt.setupDB != nil { - tt.setupDB(t, ctx, mockDB) - } - c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB} - all, err := c.getAllServiceProviderClusters(ctx) - require.NoError(t, err) - require.Len(t, all, tt.wantLen) - if tt.wantFirstResourceID != "" { - assert.Equal(t, tt.wantFirstResourceID, all[0].ResourceID.String()) - } - }) - } -} - -func TestReferencedMaestroAPIMaestroBundleNamesByShard(t *testing.T) { +func TestBuildClusterScopedMaestroAPIMaestroBundleNamesByShard(t *testing.T) { spcRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/serviceProviderClusters/default")) tests := []struct { @@ -158,9 +110,10 @@ func TestReferencedMaestroAPIMaestroBundleNamesByShard(t *testing.T) { wantBundleName: "bundle-one", }, } + c := &deleteOrphanedMaestroReadonlyBundles{} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - out, err := referencedMaestroAPIMaestroBundleNamesByShard(tt.spcsByShard) + out, err := c.buildClusterScopedMaestroAPIMaestroBundleNamesByShard(tt.spcsByShard) if tt.wantErr { require.Error(t, err) assert.Contains(t, err.Error(), tt.errSubstr) @@ -175,34 +128,91 @@ func TestReferencedMaestroAPIMaestroBundleNamesByShard(t *testing.T) { } } -// paginationSPCGlobalLister returns different iterators based on ContinuationToken to simulate pagination. -type paginationSPCGlobalLister struct { - iter1 database.DBClientIterator[api.ServiceProviderCluster] - iter2 database.DBClientIterator[api.ServiceProviderCluster] - token string -} +func TestBuildNodePoolScopedMaestroAPIMaestroBundleNamesByShard(t *testing.T) { + spnpRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/nodePools/np/serviceProviderNodePools/default")) -func (p *paginationSPCGlobalLister) List(ctx context.Context, opts *database.DBClientListResourceDocsOptions) (database.DBClientIterator[api.ServiceProviderCluster], error) { - tok := "" - if opts != nil && opts.ContinuationToken != nil { - tok = *opts.ContinuationToken - } - if tok == "" { - return p.iter1, nil + tests := []struct { + name string + spnpsByShard map[string][]*api.ServiceProviderNodePool + wantErr bool + errSubstr string + wantShard string + wantBundleName string + }{ + { + name: "empty maestroAPIMaestroBundleName", + spnpsByShard: map[string][]*api.ServiceProviderNodePool{ + "shard1": { + { + ResourceID: *spnpRID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + {Name: "logical-name", MaestroAPIMaestroBundleName: ""}, + }, + }, + }, + }, + }, + wantErr: true, + errSubstr: "has empty maestroAPIMaestroBundleName", + }, + { + name: "nil ref entry", + spnpsByShard: map[string][]*api.ServiceProviderNodePool{ + "shard1": { + { + ResourceID: *spnpRID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{nil}, + }, + }, + }, + }, + wantErr: true, + errSubstr: "is nil", + }, + { + name: "nil ServiceProviderNodePool", + spnpsByShard: map[string][]*api.ServiceProviderNodePool{ + "shard1": {nil}, + }, + wantErr: true, + errSubstr: "nil ServiceProviderNodePool", + }, + { + name: "success", + spnpsByShard: map[string][]*api.ServiceProviderNodePool{ + "s": { + { + ResourceID: *spnpRID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + {MaestroAPIMaestroBundleName: "bundle-np"}, + }, + }, + }, + }, + }, + wantShard: "s", + wantBundleName: "bundle-np", + }, } - if tok == p.token { - return p.iter2, nil + c := &deleteOrphanedMaestroReadonlyBundles{} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out, err := c.buildNodePoolScopedMaestroAPIMaestroBundleNamesByShard(tt.spnpsByShard) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errSubstr) + return + } + require.NoError(t, err) + shardSet := out[tt.wantShard] + require.NotNil(t, shardSet, "expected shard %q in result", tt.wantShard) + _, ok := shardSet[tt.wantBundleName] + assert.True(t, ok, "expected bundle name %q under shard %q", tt.wantBundleName, tt.wantShard) + }) } - return nil, fmt.Errorf("unexpected continuation token %q", tok) -} - -// iteratorErrorSPCGlobalLister returns an iterator that reports an error from GetError(). -type iteratorErrorSPCGlobalLister struct { - iter database.DBClientIterator[api.ServiceProviderCluster] -} - -func (i *iteratorErrorSPCGlobalLister) List(ctx context.Context, opts *database.DBClientListResourceDocsOptions) (database.DBClientIterator[api.ServiceProviderCluster], error) { - return i.iter, nil } // panicGlobalLister is a GlobalLister that panics if List is called (used for unused listers in test doubles). @@ -212,46 +222,45 @@ func (n *panicGlobalLister[T]) List(context.Context, *database.DBClientListResou panic("panicGlobalLister.List should not be called") } -// paginationGlobalListers implements GlobalListers with pagination only for ServiceProviderClusters. -type paginationGlobalListers struct { - spcLister database.GlobalLister[api.ServiceProviderCluster] -} +// defaultPanicGlobalListers implements database.GlobalListers with panic-on-List listers for every resource type. +// Embed it in a test double and override only the accessors the test cares about. +type defaultPanicGlobalListers struct{} -func (p *paginationGlobalListers) Subscriptions() database.GlobalLister[arm.Subscription] { +func (defaultPanicGlobalListers) Subscriptions() database.GlobalLister[arm.Subscription] { return &panicGlobalLister[arm.Subscription]{} } -func (p *paginationGlobalListers) Clusters() database.GlobalLister[api.HCPOpenShiftCluster] { +func (defaultPanicGlobalListers) Clusters() database.GlobalLister[api.HCPOpenShiftCluster] { return &panicGlobalLister[api.HCPOpenShiftCluster]{} } -func (p *paginationGlobalListers) NodePools() database.GlobalLister[api.HCPOpenShiftClusterNodePool] { +func (defaultPanicGlobalListers) NodePools() database.GlobalLister[api.HCPOpenShiftClusterNodePool] { return &panicGlobalLister[api.HCPOpenShiftClusterNodePool]{} } -func (p *paginationGlobalListers) ExternalAuths() database.GlobalLister[api.HCPOpenShiftClusterExternalAuth] { +func (defaultPanicGlobalListers) ExternalAuths() database.GlobalLister[api.HCPOpenShiftClusterExternalAuth] { return &panicGlobalLister[api.HCPOpenShiftClusterExternalAuth]{} } -func (p *paginationGlobalListers) ServiceProviderClusters() database.GlobalLister[api.ServiceProviderCluster] { - return p.spcLister +func (defaultPanicGlobalListers) ServiceProviderClusters() database.GlobalLister[api.ServiceProviderCluster] { + return &panicGlobalLister[api.ServiceProviderCluster]{} } -func (p *paginationGlobalListers) Operations() database.GlobalLister[api.Operation] { - return &panicGlobalLister[api.Operation]{} -} -func (p *paginationGlobalListers) ActiveOperations() database.GlobalLister[api.Operation] { - return &panicGlobalLister[api.Operation]{} +func (defaultPanicGlobalListers) ServiceProviderNodePools() database.GlobalLister[api.ServiceProviderNodePool] { + return &panicGlobalLister[api.ServiceProviderNodePool]{} } -func (p *paginationGlobalListers) Controllers() database.GlobalLister[api.Controller] { +func (defaultPanicGlobalListers) Controllers() database.GlobalLister[api.Controller] { return &panicGlobalLister[api.Controller]{} } -func (p *paginationGlobalListers) ManagementClusterContents() database.GlobalLister[api.ManagementClusterContent] { +func (defaultPanicGlobalListers) ManagementClusterContents() database.GlobalLister[api.ManagementClusterContent] { return &panicGlobalLister[api.ManagementClusterContent]{} } -func (p *paginationGlobalListers) ServiceProviderNodePools() database.GlobalLister[api.ServiceProviderNodePool] { - return &panicGlobalLister[api.ServiceProviderNodePool]{} +func (defaultPanicGlobalListers) Operations() database.GlobalLister[api.Operation] { + return &panicGlobalLister[api.Operation]{} +} +func (defaultPanicGlobalListers) ActiveOperations() database.GlobalLister[api.Operation] { + return &panicGlobalLister[api.Operation]{} } -func (p *paginationGlobalListers) BillingDocs() database.GlobalLister[database.BillingDocument] { +func (defaultPanicGlobalListers) BillingDocs() database.GlobalLister[database.BillingDocument] { return &panicGlobalLister[database.BillingDocument]{} } -var _ database.GlobalListers = (*paginationGlobalListers)(nil) +var _ database.GlobalListers = defaultPanicGlobalListers{} // simpleIterator is a simple iterator implementation for testing that doesn't use gomock. type simpleIterator[T any] struct { @@ -281,68 +290,102 @@ func (s *simpleIterator[T]) GetError() error { var _ database.DBClientIterator[api.ServiceProviderCluster] = &simpleIterator[api.ServiceProviderCluster]{} -func TestDeleteOrphanedMaestroReadonlyBundles_getAllServiceProviderClusters_Pagination(t *testing.T) { +func TestDeleteOrphanedMaestroReadonlyBundles_getAllServiceProviderClusters(t *testing.T) { ctx := context.Background() - spc1ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/serviceProviderClusters/default")) - spc2ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub2/resourceGroups/rg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster2/serviceProviderClusters/default")) - page1SPC := &api.ServiceProviderCluster{ - CosmosMetadata: arm.CosmosMetadata{ResourceID: spc1ResourceID}, - ResourceID: *spc1ResourceID, - } - page2SPC := &api.ServiceProviderCluster{ - CosmosMetadata: arm.CosmosMetadata{ResourceID: spc2ResourceID}, - ResourceID: *spc2ResourceID, - } - - iter1 := &simpleIterator[api.ServiceProviderCluster]{ - ids: []string{"id1"}, - items: []*api.ServiceProviderCluster{page1SPC}, - continuationToken: "token1", - err: nil, - } + clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) + spcResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/serviceProviderClusters/default")) - iter2 := &simpleIterator[api.ServiceProviderCluster]{ - ids: []string{"id2"}, - items: []*api.ServiceProviderCluster{page2SPC}, - continuationToken: "", - err: nil, + tests := []struct { + name string + setupDB func(t *testing.T, ctx context.Context, mockDB *databasetesting.MockDBClient) + wantLen int + wantFirstResourceID string + }{ + { + name: "empty DB returns no SPCs", + setupDB: nil, + wantLen: 0, + }, + { + name: "returns SPCs created via CRUD", + setupDB: func(t *testing.T, ctx context.Context, mockDB *databasetesting.MockDBClient) { + spc := &api.ServiceProviderCluster{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spcResourceID}, + ResourceID: *spcResourceID, + } + spcCRUD := mockDB.ServiceProviderClusters(clusterResourceID.SubscriptionID, clusterResourceID.ResourceGroupName, clusterResourceID.Name) + _, err := spcCRUD.Create(ctx, spc, nil) + require.NoError(t, err) + }, + wantLen: 1, + wantFirstResourceID: spcResourceID.String(), + }, } - - paginationListers := &paginationGlobalListers{ - spcLister: &paginationSPCGlobalLister{iter1: iter1, iter2: iter2, token: "token1"}, + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockDB := databasetesting.NewMockDBClient() + if tt.setupDB != nil { + tt.setupDB(t, ctx, mockDB) + } + c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB} + all, err := c.getAllServiceProviderClusters(ctx) + require.NoError(t, err) + require.Len(t, all, tt.wantLen) + if tt.wantFirstResourceID != "" { + assert.Equal(t, tt.wantFirstResourceID, all[0].ResourceID.String()) + } + }) } - mockDB := databasetesting.NewMockDBClient() - mockDB.SetGlobalListers(paginationListers) - - c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB} - all, err := c.getAllServiceProviderClusters(ctx) - require.NoError(t, err) - require.Len(t, all, 2) - assert.Equal(t, spc1ResourceID.String(), all[0].ResourceID.String()) - assert.Equal(t, spc2ResourceID.String(), all[1].ResourceID.String()) } -func TestDeleteOrphanedMaestroReadonlyBundles_getAllServiceProviderClusters_IteratorError(t *testing.T) { - ctx := context.Background() - iterErr := fmt.Errorf("iteration error") - iter := &simpleIterator[api.ServiceProviderCluster]{ - ids: []string{}, - items: []*api.ServiceProviderCluster{}, - continuationToken: "", - err: iterErr, - } +var _ database.DBClientIterator[api.ServiceProviderNodePool] = &simpleIterator[api.ServiceProviderNodePool]{} - mockDB := databasetesting.NewMockDBClient() - mockDB.SetGlobalListers(&paginationGlobalListers{ - spcLister: &iteratorErrorSPCGlobalLister{iter: iter}, - }) +func TestDeleteOrphanedMaestroReadonlyBundles_getAllServiceProviderNodePools(t *testing.T) { + ctx := context.Background() + clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/nodePools/worker/serviceProviderNodePools/default")) - c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB} - all, err := c.getAllServiceProviderClusters(ctx) - require.Error(t, err) - assert.Nil(t, all) - assert.Contains(t, err.Error(), "failed iterating ServiceProviderClusters") - assert.Contains(t, err.Error(), "iteration error") + tests := []struct { + name string + setupDB func(t *testing.T, ctx context.Context, mockDB *databasetesting.MockDBClient) + wantLen int + wantFirstResourceID string + }{ + { + name: "empty DB returns no SPNPs", + setupDB: nil, + wantLen: 0, + }, + { + name: "returns SPNPs created via CRUD", + setupDB: func(t *testing.T, ctx context.Context, mockDB *databasetesting.MockDBClient) { + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + } + spnpCRUD := mockDB.ServiceProviderNodePools(clusterResourceID.SubscriptionID, clusterResourceID.ResourceGroupName, clusterResourceID.Name, "worker") + _, err := spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + }, + wantLen: 1, + wantFirstResourceID: spnpResourceID.String(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockDB := databasetesting.NewMockDBClient() + if tt.setupDB != nil { + tt.setupDB(t, ctx, mockDB) + } + c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB} + all, err := c.getAllServiceProviderNodePools(ctx) + require.NoError(t, err) + require.Len(t, all, tt.wantLen) + if tt.wantFirstResourceID != "" { + assert.Equal(t, tt.wantFirstResourceID, all[0].ResourceID.String()) + } + }) + } } // alwaysErrorGlobalListers is a test double that makes the returned global listers @@ -398,20 +441,151 @@ func (f *alwaysErrorGlobalLister[T]) List(ctx context.Context, options *database var _ database.GlobalLister[any] = (*alwaysErrorGlobalLister[any])(nil) +// emptyGlobalLister returns an empty page for global list tests. +type emptyGlobalLister[T any] struct{} + +func (e *emptyGlobalLister[T]) List(ctx context.Context, opts *database.DBClientListResourceDocsOptions) (database.DBClientIterator[T], error) { + return &simpleIterator[T]{}, nil +} + +// failOnSecondSPNPGlobalLister fails ServiceProviderNodePools.List starting on the second call (initial list inside nodepool ensure succeeds, fresh list inside nodepool delete pass fails). +type failOnSecondSPNPGlobalLister struct { + call int + err error +} + +func (f *failOnSecondSPNPGlobalLister) List(ctx context.Context, opts *database.DBClientListResourceDocsOptions) (database.DBClientIterator[api.ServiceProviderNodePool], error) { + f.call++ + if f.call >= 2 { + return nil, f.err + } + return &simpleIterator[api.ServiceProviderNodePool]{}, nil +} + +// syncOnceSPCOKFailSecondSPNPGlobalListers lists empty SPCs always, and fails the second global SPNP list during nodepool ensure (for SyncOnce integration). +type syncOnceSPCOKFailSecondSPNPGlobalListers struct { + defaultPanicGlobalListers + spnp *failOnSecondSPNPGlobalLister +} + +func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) ServiceProviderClusters() database.GlobalLister[api.ServiceProviderCluster] { + return &emptyGlobalLister[api.ServiceProviderCluster]{} +} + +func (g *syncOnceSPCOKFailSecondSPNPGlobalListers) ServiceProviderNodePools() database.GlobalLister[api.ServiceProviderNodePool] { + return g.spnp +} + +var _ database.GlobalListers = (*syncOnceSPCOKFailSecondSPNPGlobalListers)(nil) + +// failOnSecondServiceProviderClusterGlobalLister fails ServiceProviderClusters.List starting on the second call. +type failOnSecondServiceProviderClusterGlobalLister struct { + call int + err error +} + +func (f *failOnSecondServiceProviderClusterGlobalLister) List(ctx context.Context, opts *database.DBClientListResourceDocsOptions) (database.DBClientIterator[api.ServiceProviderCluster], error) { + f.call++ + if f.call >= 2 { + return nil, f.err + } + return &simpleIterator[api.ServiceProviderCluster]{}, nil +} + +// emptyFirstThenServiceProviderClusterGlobalLister returns an empty first ServiceProviderClusters list, then yields items on subsequent calls. +type emptyFirstThenServiceProviderClusterGlobalLister struct { + call int + items []*api.ServiceProviderCluster +} + +func (e *emptyFirstThenServiceProviderClusterGlobalLister) List(ctx context.Context, opts *database.DBClientListResourceDocsOptions) (database.DBClientIterator[api.ServiceProviderCluster], error) { + e.call++ + if e.call == 1 { + return &simpleIterator[api.ServiceProviderCluster]{}, nil + } + ids := make([]string, len(e.items)) + for i, spc := range e.items { + ids[i] = spc.ResourceID.String() + } + return &simpleIterator[api.ServiceProviderCluster]{ids: ids, items: e.items}, nil +} + +// orphanTestGlobalListersSPCOnly is a GlobalListers test double that only customizes ServiceProviderClusters(). +type orphanTestGlobalListersSPCOnly struct { + defaultPanicGlobalListers + spc database.GlobalLister[api.ServiceProviderCluster] +} + +func newOrphanTestGlobalListersSPCOnly(spc database.GlobalLister[api.ServiceProviderCluster]) *orphanTestGlobalListersSPCOnly { + return &orphanTestGlobalListersSPCOnly{spc: spc} +} + +func (g *orphanTestGlobalListersSPCOnly) ServiceProviderClusters() database.GlobalLister[api.ServiceProviderCluster] { + return g.spc +} + +func (g *orphanTestGlobalListersSPCOnly) ServiceProviderNodePools() database.GlobalLister[api.ServiceProviderNodePool] { + return &emptyGlobalLister[api.ServiceProviderNodePool]{} +} + +var _ database.GlobalListers = (*orphanTestGlobalListersSPCOnly)(nil) + +// orphanTestGlobalListersSPNPOnly is a GlobalListers test double that only customizes ServiceProviderNodePools(). +type orphanTestGlobalListersSPNPOnly struct { + defaultPanicGlobalListers + spnp database.GlobalLister[api.ServiceProviderNodePool] +} + +func newOrphanTestGlobalListersSPNPOnly(spnp database.GlobalLister[api.ServiceProviderNodePool]) *orphanTestGlobalListersSPNPOnly { + return &orphanTestGlobalListersSPNPOnly{spnp: spnp} +} + +func (g *orphanTestGlobalListersSPNPOnly) ServiceProviderClusters() database.GlobalLister[api.ServiceProviderCluster] { + return &emptyGlobalLister[api.ServiceProviderCluster]{} +} + +func (g *orphanTestGlobalListersSPNPOnly) ServiceProviderNodePools() database.GlobalLister[api.ServiceProviderNodePool] { + return g.spnp +} + +var _ database.GlobalListers = (*orphanTestGlobalListersSPNPOnly)(nil) + func TestDeleteOrphanedMaestroReadonlyBundles_SyncOnce_ListServiceProviderClustersError(t *testing.T) { + ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) + ctrl := gomock.NewController(t) mockDB := databasetesting.NewMockDBClient() listErr := fmt.Errorf("list SPCs error") mockDB.SetGlobalListers(&alwaysErrorGlobalListers{err: listErr}) - c := &deleteOrphanedMaestroReadonlyBundles{ - cosmosClient: mockDB, - } - err := c.SyncOnce(context.Background(), nil) + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + mockCS.EXPECT().ListProvisionShards().Return(ocm.NewSimpleProvisionShardListIterator(nil, nil)) + + c := NewDeleteOrphanedMaestroReadonlyBundlesController(mockDB, mockCS, nil, "test-env") + err := c.SyncOnce(ctx, nil) require.Error(t, err) assert.Contains(t, err.Error(), "failed to get all ServiceProviderClusters") assert.Contains(t, err.Error(), "list SPCs error") } +func TestDeleteOrphanedMaestroReadonlyBundles_SyncOnce_ListServiceProviderNodePoolsError(t *testing.T) { + ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) + ctrl := gomock.NewController(t) + mockDB := databasetesting.NewMockDBClient() + listErr := fmt.Errorf("list SPNPs error") + mockDB.SetGlobalListers(&syncOnceSPCOKFailSecondSPNPGlobalListers{ + spnp: &failOnSecondSPNPGlobalLister{err: listErr}, + }) + + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + mockCS.EXPECT().ListProvisionShards().Return(ocm.NewSimpleProvisionShardListIterator(nil, nil)) + + c := NewDeleteOrphanedMaestroReadonlyBundlesController(mockDB, mockCS, nil, "test-env") + err := c.SyncOnce(ctx, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to get all ServiceProviderNodePools") + assert.Contains(t, err.Error(), "list SPNPs error") +} + func TestDeleteOrphanedMaestroReadonlyBundles_SyncOnce_NoServiceProviderClusters_Success(t *testing.T) { ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) ctrl := gomock.NewController(t) @@ -445,6 +619,17 @@ func TestDeleteOrphanedMaestroReadonlyBundles_buildMaestroClientsByProvisionShar require.Empty(t, clients) }, }, + { + name: "list provision shards iterator reports error after iteration", + setup: func(ctrl *gomock.Controller) *deleteOrphanedMaestroReadonlyBundles { + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + listErr := fmt.Errorf("CS list provision shards failed") + mockCS.EXPECT().ListProvisionShards().Return(ocm.NewSimpleProvisionShardListIterator(nil, listErr)) + return &deleteOrphanedMaestroReadonlyBundles{clusterServiceClient: mockCS} + }, + wantErr: true, + errSubstr: "failed to list Cluster Service provision shards", + }, { name: "success single shard", setup: func(ctrl *gomock.Controller) *deleteOrphanedMaestroReadonlyBundles { @@ -521,16 +706,234 @@ func TestDeleteOrphanedMaestroReadonlyBundles_buildMaestroClientsByProvisionShar maestroSourceEnvironmentIdentifier: "test-env", } }, - wantErr: true, - errSubstr: "failed to create Maestro client", + wantErr: true, + errSubstr: "failed to create Maestro client", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + c := tt.setup(ctrl) + clients, err := c.buildMaestroClientsByProvisionShard(ctx) + defer cancelMaestroClientsByProvisionShard(clients) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errSubstr) + return + } + require.NoError(t, err) + if tt.validateOut != nil { + tt.validateOut(t, clients) + } + }) + } +} + +func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderClustersByProvisionShard(t *testing.T) { + ctx := context.Background() + // mapServiceProviderClustersByProvisionShard does not use the Maestro client; tests only need shard IDs in the map. + noopMaestroShardClient := &shardMaestroClient{ + maestroClient: nil, + maestroClientCancelFunc: func() {}, + } + clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) + spcResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/serviceProviderClusters/default")) + + tests := []struct { + name string + setup func(ctrl *gomock.Controller) (c *deleteOrphanedMaestroReadonlyBundles, clients map[string]*shardMaestroClient, spcs []*api.ServiceProviderCluster) + wantErr bool + errSubstr string + validateOut func(t *testing.T, shardToSPCs map[string][]*api.ServiceProviderCluster) + }{ + { + name: "Get cluster error", + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderCluster) { + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + spc := &api.ServiceProviderCluster{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spcResourceID}, + ResourceID: *spcResourceID, + } + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, + map[string]*shardMaestroClient{"unused-shard": noopMaestroShardClient}, + []*api.ServiceProviderCluster{spc} + }, + wantErr: true, + errSubstr: "failed to get Cluster", + }, + { + name: "Get provision shard error", + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderCluster) { + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + cluster := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterResourceID}}, + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid")), + }, + } + _, err := mockDB.HCPClusters("sub", "rg").Create(ctx, cluster, nil) + require.NoError(t, err) + spc := &api.ServiceProviderCluster{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spcResourceID}, + ResourceID: *spcResourceID, + } + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster.ServiceProviderProperties.ClusterServiceID).Return(nil, fmt.Errorf("provision shard error")) + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, + map[string]*shardMaestroClient{"unused-shard": noopMaestroShardClient}, + []*api.ServiceProviderCluster{spc} + }, + wantErr: true, + errSubstr: "failed to get Cluster Provision Shard", + }, + { + name: "provision shard not in clients map", + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderCluster) { + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + cluster := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterResourceID}}, + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid")), + }, + } + _, err := mockDB.HCPClusters("sub", "rg").Create(ctx, cluster, nil) + require.NoError(t, err) + spc := &api.ServiceProviderCluster{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spcResourceID}, + ResourceID: *spcResourceID, + } + shardInClientsMap := buildTestProvisionShard("consumer-in-list") + shard2ID := "33333333333333333333333333333333" + shardReturnedByCS, err := arohcpv1alpha1.NewProvisionShard(). + ID(shard2ID). + MaestroConfig( + arohcpv1alpha1.NewProvisionShardMaestroConfig(). + ConsumerName("other-consumer"). + RestApiConfig(arohcpv1alpha1.NewProvisionShardMaestroRestApiConfig().Url("https://other.example.com:443")). + GrpcApiConfig(arohcpv1alpha1.NewProvisionShardMaestroGrpcApiConfig().Url("https://other.example.com:444")), + ). + Build() + require.NoError(t, err) + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster.ServiceProviderProperties.ClusterServiceID).Return(shardReturnedByCS, nil) + clients := map[string]*shardMaestroClient{ + shardInClientsMap.ID(): noopMaestroShardClient, + } + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, clients, []*api.ServiceProviderCluster{spc} + }, + wantErr: true, + errSubstr: "not present in provision shards map", + }, + { + name: "success single shard", + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderCluster) { + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + cluster := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterResourceID}}, + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid")), + }, + } + _, err := mockDB.HCPClusters("sub", "rg").Create(ctx, cluster, nil) + require.NoError(t, err) + spc := &api.ServiceProviderCluster{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spcResourceID}, + ResourceID: *spcResourceID, + } + provisionShard := buildTestProvisionShard("test-consumer") + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster.ServiceProviderProperties.ClusterServiceID).Return(provisionShard, nil) + clients := map[string]*shardMaestroClient{provisionShard.ID(): noopMaestroShardClient} + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, clients, []*api.ServiceProviderCluster{spc} + }, + validateOut: func(t *testing.T, shardToSPCs map[string][]*api.ServiceProviderCluster) { + provisionShard := buildTestProvisionShard("test-consumer") + require.Len(t, shardToSPCs, 1) + spcs := shardToSPCs[provisionShard.ID()] + require.Len(t, spcs, 1) + assert.Equal(t, spcResourceID.String(), spcs[0].ResourceID.String()) + }, + }, + { + name: "multiple provision shards each get own map entry", + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderCluster) { + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + + cluster1ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub1/resourceGroups/rg1/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster1")) + cluster2ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub2/resourceGroups/rg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster2")) + cluster1 := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: cluster1ResourceID}}, + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid1")), + }, + } + cluster2 := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: cluster2ResourceID}}, + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid2")), + }, + } + _, err := mockDB.HCPClusters("sub1", "rg1").Create(ctx, cluster1, nil) + require.NoError(t, err) + _, err = mockDB.HCPClusters("sub2", "rg2").Create(ctx, cluster2, nil) + require.NoError(t, err) + + spc1ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub1/resourceGroups/rg1/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster1/serviceProviderClusters/default")) + spc2ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub2/resourceGroups/rg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster2/serviceProviderClusters/default")) + spc1 := &api.ServiceProviderCluster{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spc1ResourceID}, + ResourceID: *spc1ResourceID, + } + spc2 := &api.ServiceProviderCluster{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spc2ResourceID}, + ResourceID: *spc2ResourceID, + } + + shard1 := buildTestProvisionShard("consumer1") + shard2ID := "33333333333333333333333333333333" + shard2, err := arohcpv1alpha1.NewProvisionShard(). + ID(shard2ID). + MaestroConfig( + arohcpv1alpha1.NewProvisionShardMaestroConfig(). + ConsumerName("consumer2"). + RestApiConfig(arohcpv1alpha1.NewProvisionShardMaestroRestApiConfig().Url("https://maestro2.example.com:443")). + GrpcApiConfig(arohcpv1alpha1.NewProvisionShardMaestroGrpcApiConfig().Url("https://maestro2.example.com:444")), + ). + Build() + require.NoError(t, err) + + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster1.ServiceProviderProperties.ClusterServiceID).Return(shard1, nil) + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster2.ServiceProviderProperties.ClusterServiceID).Return(shard2, nil) + + clients := map[string]*shardMaestroClient{ + shard1.ID(): noopMaestroShardClient, + shard2.ID(): noopMaestroShardClient, + } + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, clients, []*api.ServiceProviderCluster{spc1, spc2} + }, + validateOut: func(t *testing.T, shardToSPCs map[string][]*api.ServiceProviderCluster) { + require.Len(t, shardToSPCs, 2) + shard1 := buildTestProvisionShard("consumer1") + spcs1 := shardToSPCs[shard1.ID()] + require.Len(t, spcs1, 1) + spc1RID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub1/resourceGroups/rg1/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster1/serviceProviderClusters/default")) + assert.Equal(t, spc1RID.String(), spcs1[0].ResourceID.String()) + shard2ID := "33333333333333333333333333333333" + spcs2 := shardToSPCs[shard2ID] + require.Len(t, spcs2, 1) + spc2RID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub2/resourceGroups/rg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster2/serviceProviderClusters/default")) + assert.Equal(t, spc2RID.String(), spcs2[0].ResourceID.String()) + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) - c := tt.setup(ctrl) - clients, err := c.buildMaestroClientsByProvisionShard(ctx) + c, clients, spcs := tt.setup(ctrl) defer cancelMaestroClientsByProvisionShard(clients) + shardToSPCs, err := c.mapServiceProviderClustersByProvisionShard(ctx, spcs, clients) if tt.wantErr { require.Error(t, err) assert.Contains(t, err.Error(), tt.errSubstr) @@ -538,48 +941,123 @@ func TestDeleteOrphanedMaestroReadonlyBundles_buildMaestroClientsByProvisionShar } require.NoError(t, err) if tt.validateOut != nil { - tt.validateOut(t, clients) + tt.validateOut(t, shardToSPCs) } }) } } -func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderClustersByProvisionShard(t *testing.T) { +func TestDeleteOrphanedMaestroReadonlyBundles_provisionShardIDFromCluster(t *testing.T) { ctx := context.Background() - // mapServiceProviderClustersByProvisionShard does not use the Maestro client; tests only need shard IDs in the map. + + t.Run("skip when ClusterServiceID is empty", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + c := &deleteOrphanedMaestroReadonlyBundles{clusterServiceClient: mockCS} + cluster := &api.HCPOpenShiftCluster{ + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.InternalID{}, + }, + } + shardID, skip, err := c.provisionShardIDFromCluster(ctx, cluster) + require.NoError(t, err) + assert.True(t, skip) + assert.Empty(t, shardID) + }) + + t.Run("returns shard ID when ClusterServiceID is set", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + c := &deleteOrphanedMaestroReadonlyBundles{clusterServiceClient: mockCS} + csID := api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid")) + cluster := &api.HCPOpenShiftCluster{ + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: csID, + }, + } + provisionShard := buildTestProvisionShard("consumer") + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), csID).Return(provisionShard, nil) + shardID, skip, err := c.provisionShardIDFromCluster(ctx, cluster) + require.NoError(t, err) + assert.False(t, skip) + assert.Equal(t, provisionShard.ID(), shardID) + }) +} + +func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderNodePoolsByProvisionShard(t *testing.T) { + ctx := context.Background() + // mapServiceProviderNodePoolsByProvisionShard does not use the Maestro client; tests only need shard IDs in the map. noopMaestroShardClient := &shardMaestroClient{ maestroClient: nil, maestroClientCancelFunc: func() {}, } clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) - spcResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/serviceProviderClusters/default")) + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/nodePools/worker/serviceProviderNodePools/default")) tests := []struct { name string - setup func(ctrl *gomock.Controller) (c *deleteOrphanedMaestroReadonlyBundles, clients map[string]*shardMaestroClient, spcs []*api.ServiceProviderCluster) + setup func(ctrl *gomock.Controller) (c *deleteOrphanedMaestroReadonlyBundles, clients map[string]*shardMaestroClient, spnps []*api.ServiceProviderNodePool) wantErr bool errSubstr string - validateOut func(t *testing.T, shardToSPCs map[string][]*api.ServiceProviderCluster) + validateOut func(t *testing.T, shardToSPNPs map[string][]*api.ServiceProviderNodePool) }{ + { + name: "no parent resource ID", + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderNodePool) { + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + broken := *spnpResourceID + broken.Parent = nil + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: &broken}, + ResourceID: broken, + } + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, + map[string]*shardMaestroClient{"unused-shard": noopMaestroShardClient}, + []*api.ServiceProviderNodePool{spnp} + }, + wantErr: true, + errSubstr: "has no parent resource ID", + }, + { + name: "no grandparent cluster resource ID", + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderNodePool) { + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + noGrand := *spnpResourceID + npOnly := *noGrand.Parent + npOnly.Parent = nil + noGrand.Parent = &npOnly + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: &noGrand}, + ResourceID: noGrand, + } + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, + map[string]*shardMaestroClient{"unused-shard": noopMaestroShardClient}, + []*api.ServiceProviderNodePool{spnp} + }, + wantErr: true, + errSubstr: "has no grandparent cluster resource ID", + }, { name: "Get cluster error", - setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderCluster) { + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderNodePool) { mockDB := databasetesting.NewMockDBClient() mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) - spc := &api.ServiceProviderCluster{ - CosmosMetadata: arm.CosmosMetadata{ResourceID: spcResourceID}, - ResourceID: *spcResourceID, + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, } return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, map[string]*shardMaestroClient{"unused-shard": noopMaestroShardClient}, - []*api.ServiceProviderCluster{spc} + []*api.ServiceProviderNodePool{spnp} }, wantErr: true, errSubstr: "failed to get Cluster", }, { name: "Get provision shard error", - setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderCluster) { + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderNodePool) { mockDB := databasetesting.NewMockDBClient() mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) cluster := &api.HCPOpenShiftCluster{ @@ -590,21 +1068,21 @@ func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderClustersByProvis } _, err := mockDB.HCPClusters("sub", "rg").Create(ctx, cluster, nil) require.NoError(t, err) - spc := &api.ServiceProviderCluster{ - CosmosMetadata: arm.CosmosMetadata{ResourceID: spcResourceID}, - ResourceID: *spcResourceID, + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, } mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster.ServiceProviderProperties.ClusterServiceID).Return(nil, fmt.Errorf("provision shard error")) return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, map[string]*shardMaestroClient{"unused-shard": noopMaestroShardClient}, - []*api.ServiceProviderCluster{spc} + []*api.ServiceProviderNodePool{spnp} }, wantErr: true, errSubstr: "failed to get Cluster Provision Shard", }, { name: "provision shard not in clients map", - setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderCluster) { + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderNodePool) { mockDB := databasetesting.NewMockDBClient() mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) cluster := &api.HCPOpenShiftCluster{ @@ -615,9 +1093,9 @@ func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderClustersByProvis } _, err := mockDB.HCPClusters("sub", "rg").Create(ctx, cluster, nil) require.NoError(t, err) - spc := &api.ServiceProviderCluster{ - CosmosMetadata: arm.CosmosMetadata{ResourceID: spcResourceID}, - ResourceID: *spcResourceID, + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, } shardInClientsMap := buildTestProvisionShard("consumer-in-list") shard2ID := "33333333333333333333333333333333" @@ -635,14 +1113,14 @@ func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderClustersByProvis clients := map[string]*shardMaestroClient{ shardInClientsMap.ID(): noopMaestroShardClient, } - return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, clients, []*api.ServiceProviderCluster{spc} + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, clients, []*api.ServiceProviderNodePool{spnp} }, wantErr: true, errSubstr: "not present in provision shards map", }, { name: "success single shard", - setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderCluster) { + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderNodePool) { mockDB := databasetesting.NewMockDBClient() mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) cluster := &api.HCPOpenShiftCluster{ @@ -653,26 +1131,26 @@ func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderClustersByProvis } _, err := mockDB.HCPClusters("sub", "rg").Create(ctx, cluster, nil) require.NoError(t, err) - spc := &api.ServiceProviderCluster{ - CosmosMetadata: arm.CosmosMetadata{ResourceID: spcResourceID}, - ResourceID: *spcResourceID, + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, } provisionShard := buildTestProvisionShard("test-consumer") mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster.ServiceProviderProperties.ClusterServiceID).Return(provisionShard, nil) clients := map[string]*shardMaestroClient{provisionShard.ID(): noopMaestroShardClient} - return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, clients, []*api.ServiceProviderCluster{spc} + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, clients, []*api.ServiceProviderNodePool{spnp} }, - validateOut: func(t *testing.T, shardToSPCs map[string][]*api.ServiceProviderCluster) { + validateOut: func(t *testing.T, shardToSPNPs map[string][]*api.ServiceProviderNodePool) { provisionShard := buildTestProvisionShard("test-consumer") - require.Len(t, shardToSPCs, 1) - spcs := shardToSPCs[provisionShard.ID()] - require.Len(t, spcs, 1) - assert.Equal(t, spcResourceID.String(), spcs[0].ResourceID.String()) + require.Len(t, shardToSPNPs, 1) + spnps := shardToSPNPs[provisionShard.ID()] + require.Len(t, spnps, 1) + assert.Equal(t, spnpResourceID.String(), spnps[0].ResourceID.String()) }, }, { name: "multiple provision shards each get own map entry", - setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderCluster) { + setup: func(ctrl *gomock.Controller) (*deleteOrphanedMaestroReadonlyBundles, map[string]*shardMaestroClient, []*api.ServiceProviderNodePool) { mockDB := databasetesting.NewMockDBClient() mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) @@ -695,15 +1173,15 @@ func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderClustersByProvis _, err = mockDB.HCPClusters("sub2", "rg2").Create(ctx, cluster2, nil) require.NoError(t, err) - spc1ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub1/resourceGroups/rg1/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster1/serviceProviderClusters/default")) - spc2ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub2/resourceGroups/rg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster2/serviceProviderClusters/default")) - spc1 := &api.ServiceProviderCluster{ - CosmosMetadata: arm.CosmosMetadata{ResourceID: spc1ResourceID}, - ResourceID: *spc1ResourceID, + spnp1ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub1/resourceGroups/rg1/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster1/nodePools/worker/serviceProviderNodePools/default")) + spnp2ResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub2/resourceGroups/rg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster2/nodePools/worker/serviceProviderNodePools/default")) + spnp1 := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnp1ResourceID}, + ResourceID: *spnp1ResourceID, } - spc2 := &api.ServiceProviderCluster{ - CosmosMetadata: arm.CosmosMetadata{ResourceID: spc2ResourceID}, - ResourceID: *spc2ResourceID, + spnp2 := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnp2ResourceID}, + ResourceID: *spnp2ResourceID, } shard1 := buildTestProvisionShard("consumer1") @@ -726,29 +1204,29 @@ func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderClustersByProvis shard1.ID(): noopMaestroShardClient, shard2.ID(): noopMaestroShardClient, } - return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, clients, []*api.ServiceProviderCluster{spc1, spc2} + return &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS}, clients, []*api.ServiceProviderNodePool{spnp1, spnp2} }, - validateOut: func(t *testing.T, shardToSPCs map[string][]*api.ServiceProviderCluster) { - require.Len(t, shardToSPCs, 2) + validateOut: func(t *testing.T, shardToSPNPs map[string][]*api.ServiceProviderNodePool) { + require.Len(t, shardToSPNPs, 2) shard1 := buildTestProvisionShard("consumer1") - spcs1 := shardToSPCs[shard1.ID()] - require.Len(t, spcs1, 1) - spc1RID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub1/resourceGroups/rg1/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster1/serviceProviderClusters/default")) - assert.Equal(t, spc1RID.String(), spcs1[0].ResourceID.String()) + spnps1 := shardToSPNPs[shard1.ID()] + require.Len(t, spnps1, 1) + spnp1RID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub1/resourceGroups/rg1/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster1/nodePools/worker/serviceProviderNodePools/default")) + assert.Equal(t, spnp1RID.String(), spnps1[0].ResourceID.String()) shard2ID := "33333333333333333333333333333333" - spcs2 := shardToSPCs[shard2ID] - require.Len(t, spcs2, 1) - spc2RID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub2/resourceGroups/rg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster2/serviceProviderClusters/default")) - assert.Equal(t, spc2RID.String(), spcs2[0].ResourceID.String()) + spnps2 := shardToSPNPs[shard2ID] + require.Len(t, spnps2, 1) + spnp2RID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub2/resourceGroups/rg2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster2/nodePools/worker/serviceProviderNodePools/default")) + assert.Equal(t, spnp2RID.String(), spnps2[0].ResourceID.String()) }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) - c, clients, spcs := tt.setup(ctrl) + c, clients, spnps := tt.setup(ctrl) defer cancelMaestroClientsByProvisionShard(clients) - shardToSPCs, err := c.mapServiceProviderClustersByProvisionShard(ctx, spcs, clients) + shardToSPNPs, err := c.mapServiceProviderNodePoolsByProvisionShard(ctx, spnps, clients) if tt.wantErr { require.Error(t, err) assert.Contains(t, err.Error(), tt.errSubstr) @@ -756,57 +1234,56 @@ func TestDeleteOrphanedMaestroReadonlyBundles_mapServiceProviderClustersByProvis } require.NoError(t, err) if tt.validateOut != nil { - tt.validateOut(t, shardToSPCs) + tt.validateOut(t, shardToSPNPs) } }) } } -func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundlesAreDeleted(t *testing.T) { +func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(t *testing.T) { ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) spcResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/serviceProviderClusters/default")) tests := []struct { name string - setupMock func(*testing.T, *maestro.MockClient, *databasetesting.MockDBClient, *ocm.MockClusterServiceClientSpec, *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) + setupMock func(*testing.T, *maestro.MockClient, *databasetesting.MockDBClient, *ocm.MockClusterServiceClientSpec, *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient wantErr bool errSubstr string }{ { name: "empty provision shards map does not perform anything", - setupMock: func(t *testing.T, _ *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, _ *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) { - return nil, nil + setupMock: func(t *testing.T, _ *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, _ *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { + return nil }, wantErr: false, }, { name: "second global SPC list error", - setupMock: func(_ *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) { + setupMock: func(_ *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { listErr := fmt.Errorf("fresh list SPCs error") - mockDB.SetGlobalListers(&alwaysErrorGlobalListers{err: listErr}) - // We mock the Maestro client list to return an empty list to avoid processing any Maestro bundles and going straight to the SPC list call. + mockDB.SetGlobalListers(newOrphanTestGlobalListersSPCOnly(&failOnSecondServiceProviderClusterGlobalLister{err: listErr})) m.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{}, nil) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderCluster{} + } }, wantErr: true, - errSubstr: "error getting all ServiceProviderClusters (fresh snapshot)", + errSubstr: "failed to get all ServiceProviderClusters", }, { name: "an error is returned when listing Maestro bundles fails", - setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) { + setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { m.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("maestro list error")) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderCluster{} + } }, wantErr: true, errSubstr: "failed to list Maestro Bundles", }, { name: "skips bundle without readonly managed-by label", - setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) { + setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { bundleList := &workv1.ManifestWorkList{ Items: []workv1.ManifestWork{ { @@ -821,12 +1298,12 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl m.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderCluster{} + } }, }, { name: "skips Maestro bundle that is referenced by a ServiceProviderCluster on the shard", - setupMock: func(t *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, mockCS *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) { + setupMock: func(t *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, mockCS *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { clusterRID := spcResourceID.Parent cluster := &api.HCPOpenShiftCluster{ TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterRID}}, @@ -862,12 +1339,12 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl m.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderCluster{shard.ID(): {spc}} + } }, }, { name: "deletes Maestro bundle that is not referenced by any ServiceProviderCluster on the shard", - setupMock: func(t *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, mockCS *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) { + setupMock: func(t *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, mockCS *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { clusterRID := spcResourceID.Parent cluster := &api.HCPOpenShiftCluster{ TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterRID}}, @@ -904,14 +1381,14 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl m.EXPECT().Delete(gomock.Any(), "orphaned-bundle", metav1.DeleteOptions{}).Return(nil) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderCluster{shard.ID(): {spc}} + } }, }, { // Two orphans: first Delete fails (appended to syncErrors, loop continues), second Delete succeeds; // gomock call order proves the second delete was still attempted; errors.Join still returns an error. name: "continues deleting remaining orphans after a delete failure", - setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) { + setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { metaA := metav1.ObjectMeta{ Name: "orphan-a", Namespace: "consumer", @@ -935,14 +1412,14 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl m.EXPECT().Delete(gomock.Any(), "orphan-b", metav1.DeleteOptions{}).Return(nil) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderCluster{} + } }, wantErr: true, errSubstr: "failed to delete Maestro Bundle", }, { name: "pagination lists and deletes across pages", - setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) (map[string]*shardMaestroClient, map[string][]*api.ServiceProviderCluster) { + setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { page1Meta := metav1.ObjectMeta{ Name: "orphan-page1", Namespace: "consumer", @@ -962,7 +1439,168 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl m.EXPECT().Delete(gomock.Any(), "orphan-page1", metav1.DeleteOptions{}).Return(nil) return map[string]*shardMaestroClient{ shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, - }, map[string][]*api.ServiceProviderCluster{} + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + mockDB := databasetesting.NewMockDBClient() + mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) + shard := buildTestProvisionShard("test-consumer") + clients := tt.setupMock(t, mockMaestro, mockDB, mockCS, shard) + c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} + err := c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errSubstr) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted(t *testing.T) { + ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/nodePools/worker/serviceProviderNodePools/default")) + + tests := []struct { + name string + setupMock func(*testing.T, *maestro.MockClient, *databasetesting.MockDBClient, *ocm.MockClusterServiceClientSpec, *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient + wantErr bool + errSubstr string + }{ + { + name: "empty maestro clients map does not perform anything", + setupMock: func(t *testing.T, _ *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, _ *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { + return nil + }, + wantErr: false, + }, + { + name: "second global ServiceProviderNodePools list error", + setupMock: func(_ *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { + listErr := fmt.Errorf("fresh list SPNPs error") + mockDB.SetGlobalListers(newOrphanTestGlobalListersSPNPOnly(&failOnSecondSPNPGlobalLister{err: listErr})) + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{}, nil) + return map[string]*shardMaestroClient{ + shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, + } + }, + wantErr: true, + errSubstr: "failed to get all ServiceProviderNodePools", + }, + { + name: "an error is returned when listing Maestro bundles fails", + setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("maestro list error")) + return map[string]*shardMaestroClient{ + shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, + } + }, + wantErr: true, + errSubstr: "failed to list Maestro Bundles for shard", + }, + { + name: "skips bundle without nodepool readonly managed-by label", + setupMock: func(_ *testing.T, m *maestro.MockClient, _ *databasetesting.MockDBClient, _ *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { + bundleList := &workv1.ManifestWorkList{ + Items: []workv1.ManifestWork{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "other-bundle", + Namespace: "consumer", + Labels: map[string]string{readonlyBundleManagedByK8sLabelKey: readonlyBundleManagedByK8sLabelValueClusterScoped}, + }, + }, + }, + } + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) + return map[string]*shardMaestroClient{ + shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, + } + }, + }, + { + name: "skips Maestro bundle referenced by a ServiceProviderNodePool on the shard", + setupMock: func(t *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, mockCS *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { + clusterRID := spnpResourceID.Parent.Parent + cluster := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterRID}}, + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid")), + }, + } + _, err := mockDB.HCPClusters(clusterRID.SubscriptionID, clusterRID.ResourceGroupName).Create(ctx, cluster, nil) + require.NoError(t, err) + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster.ServiceProviderProperties.ClusterServiceID).Return(shard, nil).AnyTimes() + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + {MaestroAPIMaestroBundleName: "referenced-np-bundle"}, + }, + }, + } + _, err = mockDB.ServiceProviderNodePools(clusterRID.SubscriptionID, clusterRID.ResourceGroupName, clusterRID.Name, spnpResourceID.Parent.Name).Create(context.Background(), spnp, nil) + require.NoError(t, err) + bundleList := &workv1.ManifestWorkList{ + Items: []workv1.ManifestWork{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "referenced-np-bundle", + Namespace: "consumer", + Labels: map[string]string{readonlyBundleManagedByK8sLabelKey: readonlyBundleManagedByK8sLabelValueNodePoolScoped}, + }, + }, + }, + } + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) + return map[string]*shardMaestroClient{ + shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, + } + }, + }, + { + name: "deletes Maestro bundle not referenced by any ServiceProviderNodePool on the shard", + setupMock: func(t *testing.T, m *maestro.MockClient, mockDB *databasetesting.MockDBClient, mockCS *ocm.MockClusterServiceClientSpec, shard *arohcpv1alpha1.ProvisionShard) map[string]*shardMaestroClient { + clusterRID := spnpResourceID.Parent.Parent + cluster := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterRID}}, + ServiceProviderProperties: api.HCPOpenShiftClusterServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/csid")), + }, + } + _, err := mockDB.HCPClusters(clusterRID.SubscriptionID, clusterRID.ResourceGroupName).Create(ctx, cluster, nil) + require.NoError(t, err) + mockCS.EXPECT().GetClusterProvisionShard(gomock.Any(), cluster.ServiceProviderProperties.ClusterServiceID).Return(shard, nil).AnyTimes() + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + {MaestroAPIMaestroBundleName: "kept-np-bundle"}, + }, + }, + } + _, err = mockDB.ServiceProviderNodePools(clusterRID.SubscriptionID, clusterRID.ResourceGroupName, clusterRID.Name, spnpResourceID.Parent.Name).Create(context.Background(), spnp, nil) + require.NoError(t, err) + orphanMeta := metav1.ObjectMeta{ + Name: "orphaned-np-bundle", + Namespace: "consumer", + UID: types.UID("orphan-np-uid"), + Labels: map[string]string{readonlyBundleManagedByK8sLabelKey: readonlyBundleManagedByK8sLabelValueNodePoolScoped}, + } + bundleList := &workv1.ManifestWorkList{Items: []workv1.ManifestWork{{ObjectMeta: orphanMeta}}} + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) + m.EXPECT().Delete(gomock.Any(), "orphaned-np-bundle", metav1.DeleteOptions{}).Return(nil) + return map[string]*shardMaestroClient{ + shard.ID(): {maestroClient: m, maestroClientCancelFunc: func() {}}, + } }, }, } @@ -973,9 +1611,9 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl mockDB := databasetesting.NewMockDBClient() mockCS := ocm.NewMockClusterServiceClientSpec(ctrl) shard := buildTestProvisionShard("test-consumer") - clients, initialShardToSPCs := tt.setupMock(t, mockMaestro, mockDB, mockCS, shard) + clients := tt.setupMock(t, mockMaestro, mockDB, mockCS, shard) c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} - err := c.ensureOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPCs) + err := c.ensureOrphanedNodePoolScopedMaestroReadonlyBundlesAreDeleted(ctx, clients) if tt.wantErr { require.Error(t, err) assert.Contains(t, err.Error(), tt.errSubstr) @@ -986,10 +1624,10 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl } } -// TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundlesAreDeleted_shardScopedDeletes +// TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted_shardScopedDeletes // verifies that when processing a Maestro bundle in shard A, if it doesn't have a corresponding SPC associated to shard A // it is deleted, even if there's a SPC associated to other shards that contains the same maestro bundle name (per-shard reference scope). -func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundlesAreDeleted_shardScopedDeletes(t *testing.T) { +func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted_clusterScopedDeletes(t *testing.T) { ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) ctrl := gomock.NewController(t) mockShard1 := maestro.NewMockClient(ctrl) @@ -1043,23 +1681,10 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl _, err = mockDB.ServiceProviderClusters(parent.SubscriptionID, parent.ResourceGroupName, parent.Name).Create(ctx, spcOnShard1, nil) require.NoError(t, err) - spcOnShard1Lite := &api.ServiceProviderCluster{ - ResourceID: *spcOnShard1ResourceID, - Status: api.ServiceProviderClusterStatus{ - MaestroReadonlyBundles: api.MaestroBundleReferenceList{ - {MaestroAPIMaestroBundleName: "bundle-X"}, - }, - }, - } - clients := map[string]*shardMaestroClient{ shard1.ID(): {maestroClient: mockShard1, maestroClientCancelFunc: func() {}}, shard2.ID(): {maestroClient: mockShard2, maestroClientCancelFunc: func() {}}, } - initialShardToSPCs := map[string][]*api.ServiceProviderCluster{ - shard1.ID(): {spcOnShard1Lite}, - shard2.ID(): {}, - } mockShard1.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{Items: []workv1.ManifestWork{}}, nil) @@ -1078,14 +1703,14 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl mockShard2.EXPECT().Delete(gomock.Any(), "bundle-X", metav1.DeleteOptions{}).Return(nil) c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} - err = c.ensureOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPCs) + err = c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients) require.NoError(t, err) } -// TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundlesAreDeleted_bundleOnlyOnShardANoDeleteOnShardB +// TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted_bundleOnlyOnShardANoDeleteOnShardB // verifies that bundle N exists only on shard A's Maestro and is referenced by an SPC on shard A, while shard B's // Maestro lists no such bundle: processing shard B never issues a Delete for N (and shard A skips N as referenced). -func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundlesAreDeleted_bundleOnlyOnShardANoDeleteOnShardB(t *testing.T) { +func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted_bundleOnlyOnShardANoDeleteOnShardB(t *testing.T) { ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) ctrl := gomock.NewController(t) mockShardA := maestro.NewMockClient(ctrl) @@ -1139,23 +1764,10 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl _, err = mockDB.ServiceProviderClusters(parent.SubscriptionID, parent.ResourceGroupName, parent.Name).Create(ctx, spc, nil) require.NoError(t, err) - spcLite := &api.ServiceProviderCluster{ - ResourceID: *spcResourceID, - Status: api.ServiceProviderClusterStatus{ - MaestroReadonlyBundles: api.MaestroBundleReferenceList{ - {MaestroAPIMaestroBundleName: "bundle-N"}, - }, - }, - } - clients := map[string]*shardMaestroClient{ shardA.ID(): {maestroClient: mockShardA, maestroClientCancelFunc: func() {}}, shardB.ID(): {maestroClient: mockShardB, maestroClientCancelFunc: func() {}}, } - initialShardToSPCs := map[string][]*api.ServiceProviderCluster{ - shardA.ID(): {spcLite}, - shardB.ID(): {}, - } bundleNMeta := metav1.ObjectMeta{ Name: "bundle-N", @@ -1170,14 +1782,14 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl mockShardB.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{Items: []workv1.ManifestWork{}}, nil) c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} - err = c.ensureOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPCs) + err = c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients) require.NoError(t, err) } -// TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundlesAreDeleted_ReferenceOnlyOnFreshGlobalList -// verifies that an SPC document present only on the second global Cosmos list (not in the cached SPC snapshot) +// TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted_ReferenceOnlyOnFreshGlobalList +// verifies that an SPC document present only on the second global Cosmos list (not on the first) // still prevents deletion of the referenced bundle. -func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundlesAreDeleted_ReferenceOnlyOnFreshGlobalList(t *testing.T) { +func TestDeleteOrphanedMaestroReadonlyBundles_ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted_ReferenceOnlyOnFreshGlobalList(t *testing.T) { ctx := utils.ContextWithLogger(context.Background(), testr.New(t)) ctrl := gomock.NewController(t) mockMaestro := maestro.NewMockClient(ctrl) @@ -1209,15 +1821,14 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl _, err = mockDB.ServiceProviderClusters(clusterRID.SubscriptionID, clusterRID.ResourceGroupName, clusterRID.Name).Create(ctx, spc, nil) require.NoError(t, err) + mockDB.SetGlobalListers(newOrphanTestGlobalListersSPCOnly(&emptyFirstThenServiceProviderClusterGlobalLister{items: []*api.ServiceProviderCluster{spc}})) + clients := map[string]*shardMaestroClient{ shard.ID(): { maestroClient: mockMaestro, maestroClientCancelFunc: func() {}, }, } - initialShardToSPCs := map[string][]*api.ServiceProviderCluster{ - shard.ID(): {}, // stale: no SPC rows for this shard in the initial map - } bundleList := &workv1.ManifestWorkList{ Items: []workv1.ManifestWork{ @@ -1233,7 +1844,7 @@ func TestDeleteOrphanedMaestroReadonlyBundles_ensureOrphanedMaestroReadonlyBundl mockMaestro.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) c := &deleteOrphanedMaestroReadonlyBundles{cosmosClient: mockDB, clusterServiceClient: mockCS} - err = c.ensureOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients, initialShardToSPCs) + err = c.ensureClusterScopedOrphanedMaestroReadonlyBundlesAreDeleted(ctx, clients) require.NoError(t, err) } @@ -1299,6 +1910,8 @@ func TestDeleteOrphanedMaestroReadonlyBundles_SyncOnce_FullFlow_DeletesOrphanedB } mockMaestro.EXPECT().List(gomock.Any(), gomock.Any()).Return(bundleList, nil) mockMaestro.EXPECT().Delete(gomock.Any(), "orphaned-bundle", metav1.DeleteOptions{}).Return(nil) + // Second Maestro list: nodepool-scoped bundles pass after cluster-scoped orphan cleanup. + mockMaestro.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{}, nil) controller := NewDeleteOrphanedMaestroReadonlyBundlesController(mockDB, mockCS, mockMaestroBuilder, "test-env") err = controller.SyncOnce(ctx, nil) diff --git a/backend/pkg/controllers/maestro_readonly_bundle_helpers.go b/backend/pkg/controllers/maestro_readonly_bundle_helpers.go new file mode 100644 index 00000000000..a92340be031 --- /dev/null +++ b/backend/pkg/controllers/maestro_readonly_bundle_helpers.go @@ -0,0 +1,335 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package controllers + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + workv1 "open-cluster-management.io/api/work/v1" + + "k8s.io/apimachinery/pkg/api/equality" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + + "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" + "github.com/Azure/ARO-HCP/backend/pkg/maestro" + "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/database" + "github.com/Azure/ARO-HCP/internal/utils" +) + +const ( + // readonlyBundleManagedByK8sLabelKey is the key of the K8s label that is used to identify the controller that manages the readonly Maestro bundle. + readonlyBundleManagedByK8sLabelKey = "aro-hcp.azure.com/readonly-bundle-managed-by" +) + +// kubeContentMaxSizeBytes is the maximum serialized size (in bytes) stored as KubeContent in Cosmos. +// 2MB is the maximum size of a Cosmos DB item (https://learn.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-item-limits). +const kubeContentMaxSizeBytes = 1887436 // 2MB * 0.9 + +// buildInitialReadonlyMaestroBundle builds an initial readonly Maestro Bundle for a given resource specified in obj. +// objResourceIdentifier is the resource identifier of the resource specified in obj. +// maestroBundleNamespacedName is the namespaced name of the Maestro Bundle. +// managedByLabelValue is the value of the readonlyBundleManagedByK8sLabelKey label to apply to the bundle. +// Used to create the readonly Maestro bundle associated to the resource specified in obj. Some controllers consider +// the readonlyBundleManagedByK8sLabelKey label to perform their own filtering of Maestro Bundles. +func buildInitialReadonlyMaestroBundle(maestroBundleNamespacedName types.NamespacedName, objResourceIdentifier workv1.ResourceIdentifier, obj runtime.Object, managedByLabelValue string) *workv1.ManifestWork { + maestroBundleObjMeta := metav1.ObjectMeta{ + Name: maestroBundleNamespacedName.Name, + Namespace: maestroBundleNamespacedName.Namespace, + ResourceVersion: "0", // TODO is this needed when creating a maestro bundle? + Labels: map[string]string{ + // We define it as a K8s label because Maestro supports server-side filtering based on K8s labels. + // We can define it as a K8s label because for this specific use case we can comply with + // K8s labels length and charset restrictions https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set. + readonlyBundleManagedByK8sLabelKey: managedByLabelValue, + }, + } + + // We build the Maestro Bundle that will contain the resource specified in obj. + // Aside from putting the resource (manifest) previously built above, we + // also define a FeedbackRule that will allow us to retrieve the whole content + // from the management cluster + maestroBundle := &workv1.ManifestWork{ + ObjectMeta: maestroBundleObjMeta, + Spec: workv1.ManifestWorkSpec{ + Workload: workv1.ManifestsTemplate{ + Manifests: []workv1.Manifest{ + { + RawExtension: runtime.RawExtension{ + // We put the resource (manifest) specified in obj. + // In Maestro only the desired `spec` as defined in the bundle can be retrieved + // from here when querying the Maestro Bundle. + // To retrieve another section other than the desired spec Maestro + // requires defining FeedbackRule(s) in the Maestro bundle. + // For maestro readonly resources, not even the desired spec can be retrieved from here. For + // those type of resources it needs to be retrieved via status feedback rule(s) too. + // For owned resources, here the desired spec can be retrieved but that + // is not necessarily the actual spec in the management cluster side. If that is + // desired it is again necessary to get the spec via FeedbackRule(s). + Object: obj, + }, + }, + }, + }, + ManifestConfigs: []workv1.ManifestConfigOption{ + // We also need to define the ManifestConfig associated to the resource(manifest) + // that is being put within the Maestro Bundle. + { + // ResourceIdentifier needs to be specified and it is the information + // associated to the manifest that is being put within the Maestro Bundle. + ResourceIdentifier: objResourceIdentifier, + // We need to set the UpdateStrategy to read only. This + // creates a "readonly maestro bundle". + UpdateStrategy: &workv1.UpdateStrategy{ + Type: workv1.UpdateStrategyTypeReadOnly, + }, + // We define a feedbackrule based on JSONPath. We alias the name + // of this JSONPath as "resource" and its real JSONPath is "@" which + // signals the whole object is retrieved. This includes both spec + // and status. + FeedbackRules: []workv1.FeedbackRule{ + { + Type: workv1.JSONPathsType, + JsonPaths: []workv1.JsonPath{ + { + Name: "resource", + Path: "@", + }, + }, + }, + }, + }, + }, + }, + } + + return maestroBundle +} + +// buildInitialMaestroBundleReference builds an initial Maestro Bundle reference for a given maestro bundle internal name. +func buildInitialMaestroBundleReference(internalName api.MaestroBundleInternalName, generator maestro.MaestroAPIMaestroBundleNameGenerator) (*api.MaestroBundleReference, error) { + maestroAPIMaestroBundleName, err := generator.NewMaestroAPIMaestroBundleName() + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to generate Maestro API Maestro Bundle name: %w", err)) + } + return &api.MaestroBundleReference{ + Name: internalName, + MaestroAPIMaestroBundleName: maestroAPIMaestroBundleName, + MaestroAPIMaestroBundleID: "", + }, nil +} + +// buildObjectsFromUnstructuredObj builds the list of objects from the given unstructured object. +// If the unstructured object is a list, it flattens the list of objects from the list of items. Nested lists are not flattened. +// If the unstructured object is not a list, it returns a list with a single item being the single object. +func buildObjectsFromUnstructuredObj(unstructuredObj *unstructured.Unstructured) ([]runtime.RawExtension, error) { + if !unstructuredObj.IsList() { + return []runtime.RawExtension{{Object: unstructuredObj}}, nil + } + + objs := []runtime.RawExtension{} + err := unstructuredObj.EachListItem(func(o runtime.Object) error { + objs = append(objs, runtime.RawExtension{Object: o}) + return nil + }) + if err != nil { + return nil, utils.TrackError(err) + } + + return objs, nil +} + +func buildDegradedCondition(conditionStatus api.ConditionStatus, conditionReason string, conditionMessage string) api.Condition { + return api.Condition{ + Type: "Degraded", + Status: conditionStatus, + Reason: conditionReason, + Message: conditionMessage, + } +} + +// getSingleResourceStatusFeedbackRawJSONFromMaestroBundle gets the single resource status feedback raw JSON from a Maestro Bundle. +// Used to extract the content of the resource from the Maestro Bundle. +// An error is returned if the Maestro Bundle does not contain a single resource or if the resource does not contain a single status feedback value +// with its name being "resource" and its type being JsonRaw. +func getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(maestroBundle *workv1.ManifestWork) (json.RawMessage, error) { + resourceStatusManifests := maestroBundle.Status.ResourceStatus.Manifests + if len(resourceStatusManifests) != 1 { + return nil, utils.TrackError(fmt.Errorf("expected exactly one resource within the Maestro Bundle, got %d", len(resourceStatusManifests))) + } + + statusFeedbackValues := resourceStatusManifests[0].StatusFeedbacks.Values + if len(statusFeedbackValues) == 0 { + return nil, utils.TrackError(fmt.Errorf("expected exactly one status feedback value within the Maestro Bundle resource, got %d", len(statusFeedbackValues))) + } + if len(statusFeedbackValues) > 1 { + return nil, utils.TrackError(fmt.Errorf("expected exactly one status feedback value within the Maestro Bundle resource, got %d", len(statusFeedbackValues))) + } + statusFeedbackValue := statusFeedbackValues[0] + if statusFeedbackValue.Name != "resource" { + return nil, utils.TrackError(fmt.Errorf("expected status feedback value name to be 'resource', got %s", statusFeedbackValue.Name)) + } + if statusFeedbackValue.Value.Type != workv1.JsonRaw { + return nil, utils.TrackError(fmt.Errorf("expected status feedback value type to be JsonRaw, got %s", statusFeedbackValue.Value.Type)) + } + if statusFeedbackValue.Value.JsonRaw == nil { + return nil, utils.TrackError(fmt.Errorf("expected status feedback value JsonRaw to be not nil")) + } + + return []byte(*statusFeedbackValue.Value.JsonRaw), nil +} + +// calculateManagementClusterContentFromMaestroBundle builds the desired ManagementClusterContent from a Maestro +// bundle reference. parentResourceID is the ARM ID of the document parent +func calculateManagementClusterContentFromMaestroBundle( + ctx context.Context, + parentResourceID *azcorearm.ResourceID, + maestroBundleReference *api.MaestroBundleReference, + maestroClient maestro.Client, +) (*api.ManagementClusterContent, error) { + managementClusterContentResourceID := controllerutils.ManagementClusterContentResourceIDFromParentResourceID(parentResourceID, maestroBundleReference.Name) + desired := controllerutils.NewInitialManagementClusterContent(managementClusterContentResourceID) + + existingMaestroBundle, err := maestroClient.Get(ctx, maestroBundleReference.MaestroAPIMaestroBundleName, metav1.GetOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return nil, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle: %w", err)) + } + if k8serrors.IsNotFound(err) { + degradedCondition := buildDegradedCondition(api.ConditionTrue, "MaestroBundleNotFound", err.Error()) + controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) + return desired, nil + } + + rawBytes, err := getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(existingMaestroBundle) + if err != nil { + degradedCondition := buildDegradedCondition(api.ConditionTrue, "MaestroBundleStatusFeedbackNotAvailable", err.Error()) + controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) + return desired, nil + } + + kubeContentMaxSizeExceeded := len(rawBytes) > kubeContentMaxSizeBytes + var kubeContextMaxSizeExceededConditionMessage string + // We only set the retrieved content if it is within the size limit. If it + // is outside the limit we set the Degraded condition communicating the issue. + // We use unstructuredObj.Unstructured to deserialize the content so we can + // implement logic agnostic to the type of the content being retrieved. + unstructuredObj := &unstructured.Unstructured{} + err = json.Unmarshal(rawBytes, unstructuredObj) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to unmarshal object from status feedback value: %w", err)) + } + kind := unstructuredObj.GetKind() + if kind == "" { + return nil, utils.TrackError(fmt.Errorf("expected kind to be not empty")) + } + + objs, err := buildObjectsFromUnstructuredObj(unstructuredObj) + if err != nil { + return nil, utils.TrackError(fmt.Errorf("failed to build objects from unstructured object: %w", err)) + } + var degradedCondition api.Condition + if !kubeContentMaxSizeExceeded { + // TODO is ListMeta or TypeMeta required at the metav1.List level? + desired.Status.KubeContent = &metav1.List{Items: objs} + degradedCondition = buildDegradedCondition(api.ConditionFalse, "NoErrors", "As expected.") + } else { + kubeContextMaxSizeExceededConditionMessage = fmt.Sprintf("%s serialized size %.2f MiB exceeds Kube content max size %.2f MiB;", kind, float64(len(rawBytes))/(1024*1024), float64(kubeContentMaxSizeBytes)/(1024*1024)) + degradedCondition = buildDegradedCondition(api.ConditionTrue, "KubeContentMaxSizeExceeded", kubeContextMaxSizeExceededConditionMessage) + } + controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) + + return desired, nil +} + +// readAndPersistMaestroReadonlyBundleContent reads a Maestro readonly bundle and creates or updates the corresponding +// ManagementClusterContent in Cosmos. parentResourceID is the resource id of the parent resource of the ManagementClusterContent. +func readAndPersistMaestroReadonlyBundleContent( + ctx context.Context, + parentResourceID *azcorearm.ResourceID, + maestroBundleReference *api.MaestroBundleReference, + maestroClient maestro.Client, + managementClusterContentsDBClient database.ManagementClusterContentCRUD, +) error { + desired, err := calculateManagementClusterContentFromMaestroBundle(ctx, parentResourceID, maestroBundleReference, maestroClient) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to calculate ManagementClusterContent from Maestro Bundle: %w", err)) + } + + existing, err := managementClusterContentsDBClient.Get(ctx, desired.CosmosMetadata.ResourceID.Name) + if err != nil && !database.IsResponseError(err, http.StatusNotFound) { + return utils.TrackError(fmt.Errorf("failed to get ManagementClusterContent: %w", err)) + } + if database.IsResponseError(err, http.StatusNotFound) { + _, err := managementClusterContentsDBClient.Create(ctx, desired, nil) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to create ManagementClusterContent: %w", err)) + } + return nil + } + + // We set the Cosmos ETag to the existing one to avoid conflicts when replacing the document + // unless someone else has modified the document since we last read it. + desired.CosmosETag = existing.CosmosETag + + // If we haven't been able to retrieve the content but there was already content + // stored we keep the previously existing stored content. + if desired.Status.KubeContent == nil && existing.Status.KubeContent != nil { + desired.Status.KubeContent = existing.Status.KubeContent + } + + // The existing ManagementClusterContent in Cosmos might include conditions that already exist beforehand and that + // have been calculated in the new desired content. To preserve the LastTransitionTime of those conditions in the case + // where the status of them hasn't changed, what we do is: + // 1. Deep copy of the existing status from Cosmos, which includes the conditions + // 2. Iterate over the newly calculated desired conditions, and for each condition: + // 2.1. Check if the condition already exists in the existing status + // 2.2. If it does, update the condition in the existing status with the new values using SetCondition. This + // will update the LastTransitionTime to the current time if there's been a change or keep the existing + // LastTransitionTime if the condition hasn't changed its status. Then, use the newly updated condition as + // the desired one. + // 2.3. If it does not, then keep the condition as is + // 3. Assign the merged conditions to the desired status. + tmpExistingStatus := existing.Status.DeepCopy() + mergedConditions := make([]api.Condition, 0, len(desired.Status.Conditions)) + for _, desiredCondition := range desired.Status.Conditions { + if controllerutils.GetCondition(tmpExistingStatus.Conditions, desiredCondition.Type) != nil { + controllerutils.SetCondition(&tmpExistingStatus.Conditions, desiredCondition) + merged := controllerutils.GetCondition(tmpExistingStatus.Conditions, desiredCondition.Type) + mergedConditions = append(mergedConditions, *merged) + continue + } + mergedConditions = append(mergedConditions, desiredCondition) + } + desired.Status.Conditions = mergedConditions + + if equality.Semantic.DeepEqual(existing, desired) { + return nil + } + + _, err = managementClusterContentsDBClient.Replace(ctx, desired, nil) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to replace ManagementClusterContent: %w", err)) + } + + return nil +} diff --git a/backend/pkg/controllers/maestro_readonly_bundle_helpers_test.go b/backend/pkg/controllers/maestro_readonly_bundle_helpers_test.go new file mode 100644 index 00000000000..baee76842da --- /dev/null +++ b/backend/pkg/controllers/maestro_readonly_bundle_helpers_test.go @@ -0,0 +1,643 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + workv1 "open-cluster-management.io/api/work/v1" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos" + + hsv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + + "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" + "github.com/Azure/ARO-HCP/backend/pkg/maestro" + "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/api/arm" + "github.com/Azure/ARO-HCP/internal/database" + "github.com/Azure/ARO-HCP/internal/databasetesting" +) + +// buildTestMaestroBundleWithStatusFeedback builds a ManifestWork with exactly one resource status manifest +// and one status feedback value named "resource" with JsonRaw type. +func buildTestMaestroBundleWithStatusFeedback(name, namespace, rawJSON string) *workv1.ManifestWork { + jsonRaw := rawJSON + return &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Status: workv1.ManifestWorkStatus{ + ResourceStatus: workv1.ManifestResourceStatus{ + Manifests: []workv1.ManifestCondition{ + { + ResourceMeta: workv1.ManifestResourceMeta{ + Group: "hypershift.openshift.io", + Version: "v1beta1", + Kind: "HostedCluster", + Name: "test-hc", + Namespace: "test-ns", + }, + StatusFeedbacks: workv1.StatusFeedbackResult{ + Values: []workv1.FeedbackValue{ + { + Name: "resource", + Value: workv1.FieldValue{ + Type: workv1.JsonRaw, + JsonRaw: &jsonRaw, + }, + }, + }, + }, + Conditions: []metav1.Condition{}, + }, + }, + }, + }, + } +} + +func TestMaestroReadonlyBundleHelpers_buildDegradedCondition(t *testing.T) { + cond := buildDegradedCondition(api.ConditionTrue, "MaestroBundleNotFound", "bundle not found") + assert.Equal(t, "Degraded", cond.Type) + assert.Equal(t, api.ConditionTrue, cond.Status) + assert.Equal(t, "MaestroBundleNotFound", cond.Reason) + assert.Equal(t, "bundle not found", cond.Message) + + condFalse := buildDegradedCondition(api.ConditionFalse, "", "") + assert.Equal(t, api.ConditionFalse, condFalse.Status) + assert.Empty(t, condFalse.Reason) + assert.Empty(t, condFalse.Message) +} + +func TestMaestroReadonlyBundleHelpers_buildObjectsFromUnstructuredObj(t *testing.T) { + t.Run("single object returns one item", func(t *testing.T) { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "hypershift.openshift.io", Version: "v1beta1", Kind: "HostedCluster"}) + obj.SetName("test-hc") + obj.SetNamespace("test-ns") + + objs, err := buildObjectsFromUnstructuredObj(obj) + require.NoError(t, err) + require.Len(t, objs, 1) + assert.Equal(t, obj, objs[0].Object) + }) + + t.Run("list object flattens items", func(t *testing.T) { + // Build an Unstructured that represents a K8s ConfigMapList with two ConfigMap items. + item1 := map[string]interface{}{"kind": "HostedClusterList", "metadata": map[string]interface{}{"name": "cm1"}} + item2 := map[string]interface{}{"kind": "HostedClusterList", "metadata": map[string]interface{}{"name": "cm2"}} + listObj := &unstructured.Unstructured{} + listObj.SetUnstructuredContent(map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMapList", + "items": []interface{}{item1, item2}, + }) + objs, err := buildObjectsFromUnstructuredObj(listObj) + require.NoError(t, err) + require.Len(t, objs, 2) + + // RawExtension has Object set (not Raw) when coming from buildObjectsFromUnstructuredObj; unmarshal to typed. + require.NotNil(t, objs[0].Object, "Object should be set") + u1 := objs[0].Object.(*unstructured.Unstructured) + cm1 := &hsv1beta1.HostedCluster{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(u1.UnstructuredContent(), cm1) + require.NoError(t, err) + assert.Equal(t, "cm1", cm1.Name) + + u2 := objs[1].Object.(*unstructured.Unstructured) + cm2 := &hsv1beta1.HostedCluster{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(u2.UnstructuredContent(), cm2) + require.NoError(t, err) + assert.Equal(t, "cm2", cm2.Name) + }) +} + +func TestMaestroReadonlyBundleHelpers_getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(t *testing.T) { + validJSON := `{"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"test"}}` + + tests := []struct { + name string + bundle *workv1.ManifestWork + want string + wantErr bool + errSub string + }{ + { + name: "success - returns raw JSON", + bundle: buildTestMaestroBundleWithStatusFeedback("bundle-1", "ns", validJSON), + want: validJSON, + }, + { + name: "error - zero manifests", + bundle: &workv1.ManifestWork{ + Status: workv1.ManifestWorkStatus{ + ResourceStatus: workv1.ManifestResourceStatus{ + Manifests: []workv1.ManifestCondition{}, + }, + }, + }, + wantErr: true, + errSub: "expected exactly one resource within the Maestro Bundle, got 0", + }, + { + name: "error - two manifests", + bundle: &workv1.ManifestWork{ + Status: workv1.ManifestWorkStatus{ + ResourceStatus: workv1.ManifestResourceStatus{ + Manifests: []workv1.ManifestCondition{ + {ResourceMeta: workv1.ManifestResourceMeta{}, Conditions: []metav1.Condition{}}, + {ResourceMeta: workv1.ManifestResourceMeta{}, Conditions: []metav1.Condition{}}, + }, + }, + }, + }, + wantErr: true, + errSub: "expected exactly one resource within the Maestro Bundle, got 2", + }, + { + name: "error - zero status feedback values", + bundle: &workv1.ManifestWork{ + Status: workv1.ManifestWorkStatus{ + ResourceStatus: workv1.ManifestResourceStatus{ + Manifests: []workv1.ManifestCondition{ + { + ResourceMeta: workv1.ManifestResourceMeta{}, + StatusFeedbacks: workv1.StatusFeedbackResult{Values: []workv1.FeedbackValue{}}, + Conditions: []metav1.Condition{}, + }, + }, + }, + }, + }, + wantErr: true, + errSub: "expected exactly one status feedback value", + }, + { + name: "error - wrong feedback name", + bundle: func() *workv1.ManifestWork { + b := buildTestMaestroBundleWithStatusFeedback("b", "ns", validJSON) + b.Status.ResourceStatus.Manifests[0].StatusFeedbacks.Values[0].Name = "wrong" + return b + }(), + wantErr: true, + errSub: "expected status feedback value name to be 'resource', got wrong", + }, + { + name: "error - wrong feedback type", + bundle: func() *workv1.ManifestWork { + b := buildTestMaestroBundleWithStatusFeedback("b", "ns", validJSON) + b.Status.ResourceStatus.Manifests[0].StatusFeedbacks.Values[0].Value.Type = workv1.String + return b + }(), + wantErr: true, + errSub: "expected status feedback value type to be JsonRaw", + }, + { + name: "error - nil JsonRaw", + bundle: func() *workv1.ManifestWork { + b := buildTestMaestroBundleWithStatusFeedback("b", "ns", validJSON) + b.Status.ResourceStatus.Manifests[0].StatusFeedbacks.Values[0].Value.JsonRaw = nil + return b + }(), + wantErr: true, + errSub: "expected status feedback value JsonRaw to be not nil", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(tt.bundle) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errSub) + } else { + require.NoError(t, err) + assert.Equal(t, tt.want, string(got)) + } + }) + } +} + +// errorInjectingMCCCRUD wraps ManagementClusterContentCRUD to allow error injection for testing. +type errorInjectingMCCCRUD struct { + database.ManagementClusterContentCRUD + getResult *api.ManagementClusterContent + getErr error + replaceErr error +} + +func (e *errorInjectingMCCCRUD) Get(ctx context.Context, resourceID string) (*api.ManagementClusterContent, error) { + if e.getErr != nil { + return nil, e.getErr + } + if e.getResult != nil { + return e.getResult, nil + } + return e.ManagementClusterContentCRUD.Get(ctx, resourceID) +} + +func (e *errorInjectingMCCCRUD) Replace(ctx context.Context, obj *api.ManagementClusterContent, opts *azcosmos.ItemOptions) (*api.ManagementClusterContent, error) { + if e.replaceErr != nil { + return nil, e.replaceErr + } + return e.ManagementClusterContentCRUD.Replace(ctx, obj, opts) +} + +var _ database.ManagementClusterContentCRUD = &errorInjectingMCCCRUD{} + +// hcpClusterCRUDWithInjectedMCC wraps HCPClusterCRUD to return a fixed ManagementClusterContentCRUD (for tests). +type hcpClusterCRUDWithInjectedMCC struct { + database.HCPClusterCRUD + mccCRUD database.ManagementClusterContentCRUD +} + +func (e *hcpClusterCRUDWithInjectedMCC) ManagementClusterContents(hcpClusterName string) database.ManagementClusterContentCRUD { + return e.mccCRUD +} + +var _ database.HCPClusterCRUD = &hcpClusterCRUDWithInjectedMCC{} + +// errorInjectingDBClient wraps MockDBClient to return error-injecting CRUDs. +type errorInjectingDBClient struct { + *databasetesting.MockDBClient + mccCRUD database.ManagementClusterContentCRUD + clustersCRUD database.HCPClusterCRUD + spcCRUD database.ServiceProviderClusterCRUD +} + +func (e *errorInjectingDBClient) HCPClusters(subscriptionID, resourceGroupName string) database.HCPClusterCRUD { + var base database.HCPClusterCRUD + if e.clustersCRUD != nil { + base = e.clustersCRUD + } else { + base = e.MockDBClient.HCPClusters(subscriptionID, resourceGroupName) + } + if e.mccCRUD != nil { + return &hcpClusterCRUDWithInjectedMCC{HCPClusterCRUD: base, mccCRUD: e.mccCRUD} + } + return base +} + +func (e *errorInjectingDBClient) ServiceProviderClusters(subscriptionID, resourceGroupName, clusterName string) database.ServiceProviderClusterCRUD { + if e.spcCRUD != nil { + return e.spcCRUD + } + return e.MockDBClient.ServiceProviderClusters(subscriptionID, resourceGroupName, clusterName) +} + +var _ database.DBClient = &errorInjectingDBClient{} + +// errorInjectingSPCCRUD wraps ServiceProviderClusterCRUD to allow error injection. +type errorInjectingSPCCRUD struct { + database.ServiceProviderClusterCRUD + getErr error +} + +func (e *errorInjectingSPCCRUD) Get(ctx context.Context, resourceID string) (*api.ServiceProviderCluster, error) { + if e.getErr != nil { + return nil, e.getErr + } + return e.ServiceProviderClusterCRUD.Get(ctx, resourceID) +} + +var _ database.ServiceProviderClusterCRUD = &errorInjectingSPCCRUD{} + +func TestMaestroReadonlyBundleHelpers_calculateManagementClusterContentFromMaestroBundle(t *testing.T) { + clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) + cluster := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterResourceID}}, + } + ref := &api.MaestroBundleReference{ + Name: api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster, + MaestroAPIMaestroBundleName: "bundle-name", + } + + hc := hsv1beta1.HostedCluster{TypeMeta: metav1.TypeMeta{APIVersion: "hypershift.openshift.io/v1beta1", Kind: "HostedCluster"}, ObjectMeta: metav1.ObjectMeta{Name: "hc1", Namespace: "ns1"}} + hcJSONBytes, err := json.Marshal(hc) + require.NoError(t, err) + validHCJSON := string(hcJSONBytes) + + tests := []struct { + name string + maestroGet func(*maestro.MockClient) + wantDegraded bool + wantKubeContent bool + wantErr bool + errSub string + }{ + { + name: "bundle not found - desired with degraded condition", + maestroGet: func(m *maestro.MockClient) { + m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "bundle-name")) + }, + wantDegraded: true, + wantKubeContent: false, + }, + { + name: "maestro api maestro bundleget error - returns error", + maestroGet: func(m *maestro.MockClient) { + m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(nil, fmt.Errorf("connection error")) + }, + wantErr: true, + errSub: "failed to get Maestro Bundle", + }, + { + name: "bundle has invalid status feedback - desired with degraded", + maestroGet: func(m *maestro.MockClient) { + // Bundle with no status feedback values + b := &workv1.ManifestWork{ + Status: workv1.ManifestWorkStatus{ + ResourceStatus: workv1.ManifestResourceStatus{ + Manifests: []workv1.ManifestCondition{ + {ResourceMeta: workv1.ManifestResourceMeta{}, StatusFeedbacks: workv1.StatusFeedbackResult{Values: []workv1.FeedbackValue{}}, Conditions: []metav1.Condition{}}, + }, + }, + }, + } + m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + }, + wantDegraded: true, + wantKubeContent: false, + }, + { + name: "success - desired with kube content", + maestroGet: func(m *maestro.MockClient) { + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) + m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + }, + wantDegraded: false, + wantKubeContent: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + tt.maestroGet(mockMaestro) + + got, err := calculateManagementClusterContentFromMaestroBundle(context.Background(), cluster.ID, ref, mockMaestro) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errSub) + } else { + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, tt.wantKubeContent, got.Status.KubeContent != nil && len(got.Status.KubeContent.Items) > 0) + hasDegradedTrue := controllerutils.IsConditionTrue(got.Status.Conditions, "Degraded") + assert.Equal(t, tt.wantDegraded, hasDegradedTrue) + } + }) + } +} + +func TestMaestroReadonlyBundleHelpers_readAndPersistMaestroReadonlyBundleContent(t *testing.T) { + ctx := context.Background() + clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) + cluster := &api.HCPOpenShiftCluster{ + TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterResourceID}}, + } + ref := &api.MaestroBundleReference{ + Name: api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster, + MaestroAPIMaestroBundleName: "bundle-name", + } + hc := hsv1beta1.HostedCluster{TypeMeta: metav1.TypeMeta{APIVersion: "hypershift.openshift.io/v1beta1", Kind: "HostedCluster"}, ObjectMeta: metav1.ObjectMeta{Name: "hc1", Namespace: "ns1"}} + hcJSONBytes, err := json.Marshal(hc) + require.NoError(t, err) + validHCJSON := string(hcJSONBytes) + + t.Run("creates new ManagementClusterContent when not found", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + + mockDB := databasetesting.NewMockDBClient() + mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") + + err := readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) + require.NoError(t, err) + + // Content should have been created (name = bundle internal name) + got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.Status.KubeContent) + require.Len(t, got.Status.KubeContent.Items, 1) + }) + + t.Run("replaces existing when content changed", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + + mockDB := databasetesting.NewMockDBClient() + mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") + // Pre-create existing content with different payload + existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) + existing := &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, + ResourceID: *existingRID, + Status: api.ManagementClusterContentStatus{KubeContent: &metav1.List{Items: []runtime.RawExtension{}}}, + } + _, err := mccCRUD.Create(ctx, existing, nil) + require.NoError(t, err) + + err = readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) + require.NoError(t, err) + + got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) + require.NoError(t, err) + require.NotNil(t, got.Status.KubeContent) + require.Len(t, got.Status.KubeContent.Items, 1) + }) + + t.Run("keeps existing kube content when desired has no content (degraded)", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + // Return bundle that has no valid status feedback so desired has no KubeContent + b := &workv1.ManifestWork{ + Status: workv1.ManifestWorkStatus{ + ResourceStatus: workv1.ManifestResourceStatus{ + Manifests: []workv1.ManifestCondition{ + {ResourceMeta: workv1.ManifestResourceMeta{}, StatusFeedbacks: workv1.StatusFeedbackResult{Values: []workv1.FeedbackValue{}}, Conditions: []metav1.Condition{}}, + }, + }, + }, + } + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + + mockDB := databasetesting.NewMockDBClient() + mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") + existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) + existingContent := &metav1.List{Items: []runtime.RawExtension{{Raw: []byte(`{}`)}}} + existing := &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, + ResourceID: *existingRID, + Status: api.ManagementClusterContentStatus{KubeContent: existingContent}, + } + _, err := mccCRUD.Create(ctx, existing, nil) + require.NoError(t, err) + + err = readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) + require.NoError(t, err) + + got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) + require.NoError(t, err) + // Should have kept existing content + require.NotNil(t, got.Status.KubeContent) + assert.Equal(t, existingContent.Items[0].Raw, got.Status.KubeContent.Items[0].Raw) + }) + + t.Run("no replace occurs when content has not changed", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) + // Get is called once when building desired for pre-create, and once inside readAndPersistMaestroReadonlyBundleContent. + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil).Times(2) + + mockDB := databasetesting.NewMockDBClient() + mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") + existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) + // Pre-create content that matches exactly what the syncer would compute (same KubeContent and Degraded=False condition) + // so that DeepEqual(existing, desired) is true and Replace is not called. + desired, err := calculateManagementClusterContentFromMaestroBundle(ctx, cluster.ID, ref, mockMaestro) + require.NoError(t, err) + require.NotNil(t, desired) + existing := &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, + ResourceID: *existingRID, + Status: desired.Status, + } + _, err = mccCRUD.Create(ctx, existing, nil) + require.NoError(t, err) + + err = readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) + require.NoError(t, err) + + // Document should still exist with same content (no Replace was needed) + got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) + require.NoError(t, err) + require.NotNil(t, got.Status.KubeContent) + require.Len(t, got.Status.KubeContent.Items, 1) + }) + + t.Run("error occurs when object has been modified in Cosmos since we retrieved it", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + + existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) + existingDoc := &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, + ResourceID: *existingRID, + Status: api.ManagementClusterContentStatus{KubeContent: &metav1.List{Items: []runtime.RawExtension{{Raw: []byte(validHCJSON)}}}}, + } + + // Use error-injecting wrapper to simulate 412 Precondition Failed on Replace + mockDB := &errorInjectingDBClient{ + MockDBClient: databasetesting.NewMockDBClient(), + mccCRUD: &errorInjectingMCCCRUD{ + getResult: existingDoc, + replaceErr: databasetesting.NewPreconditionFailedError(), + }, + } + + err := readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mockDB.mccCRUD) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to replace ManagementClusterContent") + assert.True(t, database.IsResponseError(err, http.StatusPreconditionFailed), "expected 412 Precondition Failed") + }) + + t.Run("error occurs when managementClusterContentsDBClient.Get fails", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) + + getErr := fmt.Errorf("cosmos connection error") + // Use error-injecting wrapper to simulate Get error + mockDB := &errorInjectingDBClient{ + MockDBClient: databasetesting.NewMockDBClient(), + mccCRUD: &errorInjectingMCCCRUD{ + getErr: getErr, + }, + } + + err := readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mockDB.mccCRUD) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to get ManagementClusterContent") + assert.Contains(t, err.Error(), "cosmos connection error") + }) + + t.Run("preserves Degraded LastTransitionTime from Cosmos when status unchanged", func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := maestro.NewMockClient(ctrl) + b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) + mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil).Times(1) + + mockDB := databasetesting.NewMockDBClient() + mccCRUD := mockDB.HCPClusters("sub", "rg").ManagementClusterContents("cluster") + existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) + + u := &unstructured.Unstructured{} + require.NoError(t, json.Unmarshal([]byte(validHCJSON), u)) + historicLTT := time.Date(2020, 6, 15, 12, 0, 0, 0, time.UTC) + existing := &api.ManagementClusterContent{ + CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, + ResourceID: *existingRID, + Status: api.ManagementClusterContentStatus{ + KubeContent: &metav1.List{ + Items: []runtime.RawExtension{{Object: u}}, + }, + Conditions: []api.Condition{ + { + Type: "Degraded", + Status: api.ConditionFalse, + Reason: "NoErrors", + Message: "As expected.", + LastTransitionTime: historicLTT, + }, + }, + }, + } + _, err := mccCRUD.Create(ctx, existing, nil) + require.NoError(t, err) + + err = readAndPersistMaestroReadonlyBundleContent(ctx, cluster.ID, ref, mockMaestro, mccCRUD) + require.NoError(t, err) + + got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) + require.NoError(t, err) + degraded := controllerutils.GetCondition(got.Status.Conditions, "Degraded") + require.NotNil(t, degraded) + assert.True(t, degraded.LastTransitionTime.Equal(historicLTT), "expected LastTransitionTime from Cosmos to be preserved, got %v", degraded.LastTransitionTime) + }) + +} diff --git a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go index e050c014b14..c8fc6e3ad44 100644 --- a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go +++ b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller.go @@ -15,29 +15,15 @@ package controllers import ( "context" - "encoding/json" "errors" "fmt" "net/http" "time" - workv1 "open-cluster-management.io/api/work/v1" - - "k8s.io/apimachinery/pkg/api/equality" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - - azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" - - arohcpv1alpha1 "github.com/openshift-online/ocm-sdk-go/arohcp/v1alpha1" - "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" "github.com/Azure/ARO-HCP/backend/pkg/informers" "github.com/Azure/ARO-HCP/backend/pkg/listers" "github.com/Azure/ARO-HCP/backend/pkg/maestro" - "github.com/Azure/ARO-HCP/internal/api" "github.com/Azure/ARO-HCP/internal/database" "github.com/Azure/ARO-HCP/internal/ocm" "github.com/Azure/ARO-HCP/internal/utils" @@ -128,14 +114,16 @@ func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) SyncOnc // This is important to avoid leaking resources when the sync is done. ctx, cancel := context.WithCancel(ctx) defer cancel() - maestroClient, err := c.createMaestroClientFromProvisionShard(ctx, clusterProvisionShard) + maestroClient, err := createMaestroClientFromCSProvisionShard(ctx, c.maestroSourceEnvironmentIdentifier, c.maestroClientBuilder, clusterProvisionShard) if err != nil { return utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) } + managementClusterContentsDBClient := c.cosmosClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).ManagementClusterContents(key.HCPClusterName) + var syncErrors []error for _, maestroBundleReference := range existingServiceProviderCluster.Status.MaestroReadonlyBundles { - err = c.readAndPersistMaestroBundleContent(ctx, existingCluster, maestroBundleReference, maestroClient) + err = readAndPersistMaestroReadonlyBundleContent(ctx, existingCluster.ID, maestroBundleReference, maestroClient, managementClusterContentsDBClient) if err != nil { syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to read and persist HostedCluster: %w", err))) } @@ -145,227 +133,6 @@ func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) SyncOnc return utils.TrackError(errors.Join(syncErrors...)) } -// calculateManagementClusterContentFromMaestroBundle calculates the desired ManagementClusterContent from the given Maestro Bundle reference. -// It returns the desired ManagementClusterContent or an error if the calculation fails. -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) calculateManagementClusterContentFromMaestroBundle( - ctx context.Context, cluster *api.HCPOpenShiftCluster, hostedClusterMaestroBundleReference *api.MaestroBundleReference, - maestroClient maestro.Client, -) (*api.ManagementClusterContent, error) { - managementClusterContentResourceID := c.managementClusterContentResourceIDFromClusterResourceID(cluster.ID, hostedClusterMaestroBundleReference.Name) - desired := c.newInitialManagementClusterContent(managementClusterContentResourceID) - - existingMaestroBundle, err := maestroClient.Get(ctx, hostedClusterMaestroBundleReference.MaestroAPIMaestroBundleName, metav1.GetOptions{}) - if err != nil && !k8serrors.IsNotFound(err) { - return nil, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle: %w", err)) - } - if k8serrors.IsNotFound(err) { - degradedCondition := c.buildDegradedCondition(api.ConditionTrue, "MaestroBundleNotFound", err.Error()) - controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) - return desired, nil - } - - rawBytes, err := c.getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(existingMaestroBundle) - if err != nil { - degradedCondition := c.buildDegradedCondition(api.ConditionTrue, "MaestroBundleStatusFeedbackNotAvailable", err.Error()) - controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) - return desired, nil - } - - kubeContentMaxSizeExceeded := len(rawBytes) > c.kubeContentMaxSizeBytes() - var kubeContextMaxSizeExceededConditionMessage string - // We only set the retrieved content if it is within the size limit. If it - // is outside the limit we set the Degraded condition communicating the issue. - // We use unstructuredObj.Unstructured to deserialize the content so we can - // implement logic agnostic to the type of the content being retrieved. - unstructuredObj := &unstructured.Unstructured{} - err = json.Unmarshal(rawBytes, unstructuredObj) - if err != nil { - return nil, utils.TrackError(fmt.Errorf("failed to unmarshal object from status feedback value: %w", err)) - } - kind := unstructuredObj.GetKind() - if kind == "" { - return nil, utils.TrackError(fmt.Errorf("expected kind to be not empty")) - } - - objs, err := c.buildObjectsFromUnstructuredObj(unstructuredObj) - if err != nil { - return nil, utils.TrackError(fmt.Errorf("failed to build objects from unstructured object: %w", err)) - } - var degradedCondition api.Condition - if !kubeContentMaxSizeExceeded { - // TODO is ListMeta or TypeMeta required at the metav1.List level? - desired.Status.KubeContent = &metav1.List{Items: objs} - degradedCondition = c.buildDegradedCondition(api.ConditionFalse, "NoErrors", "As expected.") - } else { - kubeContextMaxSizeExceededConditionMessage = fmt.Sprintf("%s serialized size %.2f MiB exceeds Kube content max size %.2f MiB;", kind, float64(len(rawBytes))/(1024*1024), float64(c.kubeContentMaxSizeBytes())/(1024*1024)) - degradedCondition = c.buildDegradedCondition(api.ConditionTrue, "KubeContentMaxSizeExceeded", kubeContextMaxSizeExceededConditionMessage) - } - controllerutils.SetCondition(&desired.Status.Conditions, degradedCondition) - - return desired, nil -} - -// buildObjectsFromUnstructuredObj builds the list of objects from the given unstructured object. -// If the unstructured object is a list, it flattens the list of objects from the list of items. Nested lists are not flattened. -// If the unstructured object is not a list, it returns a list with a single item being the single object. -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) buildObjectsFromUnstructuredObj(unstructuredObj *unstructured.Unstructured) ([]runtime.RawExtension, error) { - if !unstructuredObj.IsList() { - return []runtime.RawExtension{{Object: unstructuredObj}}, nil - } - - objs := []runtime.RawExtension{} - err := unstructuredObj.EachListItem(func(o runtime.Object) error { - objs = append(objs, runtime.RawExtension{Object: o}) - return nil - }) - if err != nil { - return nil, utils.TrackError(err) - } - - return objs, nil -} - -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) buildDegradedCondition(conditionStatus api.ConditionStatus, conditionReason string, conditionMessage string) api.Condition { - return api.Condition{ - Type: "Degraded", - Status: conditionStatus, - Reason: conditionReason, - Message: conditionMessage, - } -} - -// readAndPersistMaestroBundleContent reads the Maestro Bundle content from the given Maestro Bundle reference -// and persists it in Cosmos. -// To achieve that, it gets the Maestro readonly bundle pointing to the Cluster's HostedCluster, it extracts the -// returned content by Maestro by taking it from the Maestro bundles's status feedback rule that contains the whole object and then it persists it -// in Cosmos. -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) readAndPersistMaestroBundleContent( - ctx context.Context, cluster *api.HCPOpenShiftCluster, hostedClusterMaestroBundleReference *api.MaestroBundleReference, - maestroClient maestro.Client, -) error { - - desired, err := c.calculateManagementClusterContentFromMaestroBundle(ctx, cluster, hostedClusterMaestroBundleReference, maestroClient) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to calculate ManagementClusterContent from Maestro Bundle: %w", err)) - } - - managementClusterContentsDBClient := c.cosmosClient.ManagementClusterContents( - cluster.ID.SubscriptionID, - cluster.ID.ResourceGroupName, - cluster.ID.Name, - ) - - existing, err := managementClusterContentsDBClient.Get(ctx, desired.CosmosMetadata.ResourceID.Name) - if err != nil && !database.IsResponseError(err, http.StatusNotFound) { - return utils.TrackError(fmt.Errorf("failed to get ManagementClusterContent: %w", err)) - } - if database.IsResponseError(err, http.StatusNotFound) { - _, err := managementClusterContentsDBClient.Create(ctx, desired, nil) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to create ManagementClusterContent: %w", err)) - } - return nil - } - - // We set the Cosmos ETag to the existing one to avoid conflicts when replacing the document - // unless someone else has modified the document since we last read it. - desired.CosmosETag = existing.CosmosETag - - // If we haven't been able to retrieve the content but there was already content - // stored we keep the previously existing stored content. - if desired.Status.KubeContent == nil && existing.Status.KubeContent != nil { - desired.Status.KubeContent = existing.Status.KubeContent - } - - if equality.Semantic.DeepEqual(existing, desired) { - return nil - } - - _, err = managementClusterContentsDBClient.Replace(ctx, desired, nil) - if err != nil { - return utils.TrackError(fmt.Errorf("failed to replace ManagementClusterContent: %w", err)) - } - - return nil -} - -// kubeContentMaxSizeBytes returns the maximum size of a Cosmos DB item in bytes. -// 2MB is the maximum size of a Cosmos DB item (https://learn.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-item-limits). -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) kubeContentMaxSizeBytes() int { - return 1887436 // 2MB * 0.9 -} - -// getSingleResourceStatusFeedbackRawJSONFromMaestroBundle gets the single resource status feedback raw JSON from a Maestro Bundle. -// Used to extract the content of the resource from the Maestro Bundle. -// An error is returned if the Maestro Bundle does not contain a single resource or if the resource does not contain a single status feedback value -// with its name being "resource" and its type being JsonRaw. -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(maestroBundle *workv1.ManifestWork) (json.RawMessage, error) { - resourceStatusManifests := maestroBundle.Status.ResourceStatus.Manifests - if len(resourceStatusManifests) != 1 { - return nil, utils.TrackError(fmt.Errorf("expected exactly one resource within the Maestro Bundle, got %d", len(resourceStatusManifests))) - } - - statusFeedbackValues := resourceStatusManifests[0].StatusFeedbacks.Values - if len(statusFeedbackValues) == 0 { - return nil, utils.TrackError(fmt.Errorf("expected exactly one status feedback value within the Maestro Bundle resource, got %d", len(statusFeedbackValues))) - } - if len(statusFeedbackValues) > 1 { - return nil, utils.TrackError(fmt.Errorf("expected exactly one status feedback value within the Maestro Bundle resource, got %d", len(statusFeedbackValues))) - } - statusFeedbackValue := statusFeedbackValues[0] - if statusFeedbackValue.Name != "resource" { - return nil, utils.TrackError(fmt.Errorf("expected status feedback value name to be 'resource', got %s", statusFeedbackValue.Name)) - } - if statusFeedbackValue.Value.Type != workv1.JsonRaw { - return nil, utils.TrackError(fmt.Errorf("expected status feedback value type to be JsonRaw, got %s", statusFeedbackValue.Value.Type)) - } - if statusFeedbackValue.Value.JsonRaw == nil { - return nil, utils.TrackError(fmt.Errorf("expected status feedback value JsonRaw to be not nil")) - } - - return []byte(*statusFeedbackValue.Value.JsonRaw), nil -} - func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) CooldownChecker() controllerutils.CooldownChecker { return c.cooldownChecker } - -// createMaestroClientFromProvisionShard creates a Maestro client for the given cluster provision shard. -// the client is scoped to the Consumer Name associated to the provision shard, and to -// the source ID associated to the provision shard and the environment specified -// in c.maestroSourceEnvironmentIdentifier, which is a configuration parameter at -// deployment time. -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) createMaestroClientFromProvisionShard( - ctx context.Context, clusterProvisionShard *arohcpv1alpha1.ProvisionShard, -) (maestro.Client, error) { - provisionShardMaestroConsumerName := clusterProvisionShard.MaestroConfig().ConsumerName() - provisionShardMaestroRESTAPIEndpoint := clusterProvisionShard.MaestroConfig().RestApiConfig().Url() - provisionShardMaestroGRPCAPIEndpoint := clusterProvisionShard.MaestroConfig().GrpcApiConfig().Url() - // This allows us to be able to have visibility on the Maestro Bundles owned by the same source ID for a given - // provision shard and environment. This should have the same source ID as what CS has in each corresponding environment - // because otherwise we would not have visibility on the Maestro Bundles owned - maestroSourceID := maestro.GenerateMaestroSourceID(c.maestroSourceEnvironmentIdentifier, clusterProvisionShard.ID()) - - maestroClient, err := c.maestroClientBuilder.NewClient(ctx, provisionShardMaestroRESTAPIEndpoint, provisionShardMaestroGRPCAPIEndpoint, provisionShardMaestroConsumerName, maestroSourceID) - - return maestroClient, err -} - -// newInitialManagementClusterContent returns a new ManagementClusterContent with -// the given resource ID as its parent. The resource ID is assumed to be a -// cluster resource ID. -// The returned value can be used to consistently initialize a new ManagementClusterContent -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) newInitialManagementClusterContent(managementClusterContentResourceID *azcorearm.ResourceID) *api.ManagementClusterContent { - return &api.ManagementClusterContent{ - CosmosMetadata: api.CosmosMetadata{ - ResourceID: managementClusterContentResourceID, - }, - ResourceID: *managementClusterContentResourceID, - } -} - -// managementClusterContentResourceIDFromClusterResourceID returns the resource ID for the -// ManagementClusterContent associated to the given cluster resource ID and maestro bundle internal name. -func (c *readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer) managementClusterContentResourceIDFromClusterResourceID(clusterResourceID *azcorearm.ResourceID, maestroBundleInternalName api.MaestroBundleInternalName) *azcorearm.ResourceID { - return api.Must(azcorearm.ParseResourceID(fmt.Sprintf("%s/%s/%s", clusterResourceID.String(), api.ManagementClusterContentResourceTypeName, maestroBundleInternalName))) -} diff --git a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go index 408470fbd87..42962c9cf55 100644 --- a/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go +++ b/backend/pkg/controllers/read_and_persist_cluster_scoped_maestro_readonly_bundles_content_controller_test.go @@ -18,616 +18,24 @@ import ( "context" "encoding/json" "fmt" - "net/http" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" - workv1 "open-cluster-management.io/api/work/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" - "github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos" - - hsv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" "github.com/Azure/ARO-HCP/backend/pkg/maestro" "github.com/Azure/ARO-HCP/internal/api" "github.com/Azure/ARO-HCP/internal/api/arm" - "github.com/Azure/ARO-HCP/internal/database" "github.com/Azure/ARO-HCP/internal/databasetesting" "github.com/Azure/ARO-HCP/internal/ocm" ) -// errorInjectingMCCCRUD wraps ManagementClusterContentCRUD to allow error injection for testing. -type errorInjectingMCCCRUD struct { - database.ManagementClusterContentCRUD - getResult *api.ManagementClusterContent - getErr error - replaceErr error -} - -func (e *errorInjectingMCCCRUD) Get(ctx context.Context, resourceID string) (*api.ManagementClusterContent, error) { - if e.getErr != nil { - return nil, e.getErr - } - if e.getResult != nil { - return e.getResult, nil - } - return e.ManagementClusterContentCRUD.Get(ctx, resourceID) -} - -func (e *errorInjectingMCCCRUD) Replace(ctx context.Context, obj *api.ManagementClusterContent, opts *azcosmos.ItemOptions) (*api.ManagementClusterContent, error) { - if e.replaceErr != nil { - return nil, e.replaceErr - } - return e.ManagementClusterContentCRUD.Replace(ctx, obj, opts) -} - -var _ database.ManagementClusterContentCRUD = &errorInjectingMCCCRUD{} - -// errorInjectingDBClient wraps MockDBClient to return error-injecting CRUDs. -type errorInjectingDBClient struct { - *databasetesting.MockDBClient - mccCRUD database.ManagementClusterContentCRUD - clustersCRUD database.HCPClusterCRUD - spcCRUD database.ServiceProviderClusterCRUD -} - -func (e *errorInjectingDBClient) ManagementClusterContents(subscriptionID, resourceGroupName, clusterName string) database.ManagementClusterContentCRUD { - if e.mccCRUD != nil { - return e.mccCRUD - } - return e.MockDBClient.ManagementClusterContents(subscriptionID, resourceGroupName, clusterName) -} - -func (e *errorInjectingDBClient) HCPClusters(subscriptionID, resourceGroupName string) database.HCPClusterCRUD { - if e.clustersCRUD != nil { - return e.clustersCRUD - } - return e.MockDBClient.HCPClusters(subscriptionID, resourceGroupName) -} - -func (e *errorInjectingDBClient) ServiceProviderClusters(subscriptionID, resourceGroupName, clusterName string) database.ServiceProviderClusterCRUD { - if e.spcCRUD != nil { - return e.spcCRUD - } - return e.MockDBClient.ServiceProviderClusters(subscriptionID, resourceGroupName, clusterName) -} - -var _ database.DBClient = &errorInjectingDBClient{} - -// errorInjectingHCPClusterCRUD wraps HCPClusterCRUD to allow error injection. -type errorInjectingHCPClusterCRUD struct { - database.HCPClusterCRUD - getResult *api.HCPOpenShiftCluster - getErr error -} - -func (e *errorInjectingHCPClusterCRUD) Get(ctx context.Context, resourceID string) (*api.HCPOpenShiftCluster, error) { - if e.getErr != nil { - return nil, e.getErr - } - if e.getResult != nil { - return e.getResult, nil - } - return e.HCPClusterCRUD.Get(ctx, resourceID) -} - -var _ database.HCPClusterCRUD = &errorInjectingHCPClusterCRUD{} - -// errorInjectingSPCCRUD wraps ServiceProviderClusterCRUD to allow error injection. -type errorInjectingSPCCRUD struct { - database.ServiceProviderClusterCRUD - getErr error -} - -func (e *errorInjectingSPCCRUD) Get(ctx context.Context, resourceID string) (*api.ServiceProviderCluster, error) { - if e.getErr != nil { - return nil, e.getErr - } - return e.ServiceProviderClusterCRUD.Get(ctx, resourceID) -} - -var _ database.ServiceProviderClusterCRUD = &errorInjectingSPCCRUD{} - -func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_buildDegradedCondition(t *testing.T) { - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{} - - cond := syncer.buildDegradedCondition(api.ConditionTrue, "MaestroBundleNotFound", "bundle not found") - assert.Equal(t, "Degraded", cond.Type) - assert.Equal(t, api.ConditionTrue, cond.Status) - assert.Equal(t, "MaestroBundleNotFound", cond.Reason) - assert.Equal(t, "bundle not found", cond.Message) - - condFalse := syncer.buildDegradedCondition(api.ConditionFalse, "", "") - assert.Equal(t, api.ConditionFalse, condFalse.Status) - assert.Empty(t, condFalse.Reason) - assert.Empty(t, condFalse.Message) -} - -func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_buildObjectsFromUnstructuredObj(t *testing.T) { - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{} - - t.Run("single object returns one item", func(t *testing.T) { - obj := &unstructured.Unstructured{} - obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "hypershift.openshift.io", Version: "v1beta1", Kind: "HostedCluster"}) - obj.SetName("test-hc") - obj.SetNamespace("test-ns") - - objs, err := syncer.buildObjectsFromUnstructuredObj(obj) - require.NoError(t, err) - require.Len(t, objs, 1) - assert.Equal(t, obj, objs[0].Object) - }) - - t.Run("list object flattens items", func(t *testing.T) { - // Build an Unstructured that represents a K8s ConfigMapList with two ConfigMap items. - item1 := map[string]interface{}{"kind": "HostedClusterList", "metadata": map[string]interface{}{"name": "cm1"}} - item2 := map[string]interface{}{"kind": "HostedClusterList", "metadata": map[string]interface{}{"name": "cm2"}} - listObj := &unstructured.Unstructured{} - listObj.SetUnstructuredContent(map[string]interface{}{ - "apiVersion": "v1", - "kind": "ConfigMapList", - "items": []interface{}{item1, item2}, - }) - objs, err := syncer.buildObjectsFromUnstructuredObj(listObj) - require.NoError(t, err) - require.Len(t, objs, 2) - - // RawExtension has Object set (not Raw) when coming from buildObjectsFromUnstructuredObj; unmarshal to typed. - require.NotNil(t, objs[0].Object, "Object should be set") - u1 := objs[0].Object.(*unstructured.Unstructured) - cm1 := &hsv1beta1.HostedCluster{} - err = runtime.DefaultUnstructuredConverter.FromUnstructured(u1.UnstructuredContent(), cm1) - require.NoError(t, err) - assert.Equal(t, "cm1", cm1.Name) - - u2 := objs[1].Object.(*unstructured.Unstructured) - cm2 := &hsv1beta1.HostedCluster{} - err = runtime.DefaultUnstructuredConverter.FromUnstructured(u2.UnstructuredContent(), cm2) - require.NoError(t, err) - assert.Equal(t, "cm2", cm2.Name) - }) -} - -// buildTestMaestroBundleWithStatusFeedback builds a ManifestWork with exactly one resource status manifest -// and one status feedback value named "resource" with JsonRaw type. -func buildTestMaestroBundleWithStatusFeedback(name, namespace, rawJSON string) *workv1.ManifestWork { - jsonRaw := rawJSON - return &workv1.ManifestWork{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, - Status: workv1.ManifestWorkStatus{ - ResourceStatus: workv1.ManifestResourceStatus{ - Manifests: []workv1.ManifestCondition{ - { - ResourceMeta: workv1.ManifestResourceMeta{ - Group: "hypershift.openshift.io", - Version: "v1beta1", - Kind: "HostedCluster", - Name: "test-hc", - Namespace: "test-ns", - }, - StatusFeedbacks: workv1.StatusFeedbackResult{ - Values: []workv1.FeedbackValue{ - { - Name: "resource", - Value: workv1.FieldValue{ - Type: workv1.JsonRaw, - JsonRaw: &jsonRaw, - }, - }, - }, - }, - Conditions: []metav1.Condition{}, - }, - }, - }, - }, - } -} - -func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(t *testing.T) { - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{} - validJSON := `{"apiVersion":"v1","kind":"ConfigMap","metadata":{"name":"test"}}` - - tests := []struct { - name string - bundle *workv1.ManifestWork - want string - wantErr bool - errSub string - }{ - { - name: "success - returns raw JSON", - bundle: buildTestMaestroBundleWithStatusFeedback("bundle-1", "ns", validJSON), - want: validJSON, - }, - { - name: "error - zero manifests", - bundle: &workv1.ManifestWork{ - Status: workv1.ManifestWorkStatus{ - ResourceStatus: workv1.ManifestResourceStatus{ - Manifests: []workv1.ManifestCondition{}, - }, - }, - }, - wantErr: true, - errSub: "expected exactly one resource within the Maestro Bundle, got 0", - }, - { - name: "error - two manifests", - bundle: &workv1.ManifestWork{ - Status: workv1.ManifestWorkStatus{ - ResourceStatus: workv1.ManifestResourceStatus{ - Manifests: []workv1.ManifestCondition{ - {ResourceMeta: workv1.ManifestResourceMeta{}, Conditions: []metav1.Condition{}}, - {ResourceMeta: workv1.ManifestResourceMeta{}, Conditions: []metav1.Condition{}}, - }, - }, - }, - }, - wantErr: true, - errSub: "expected exactly one resource within the Maestro Bundle, got 2", - }, - { - name: "error - zero status feedback values", - bundle: &workv1.ManifestWork{ - Status: workv1.ManifestWorkStatus{ - ResourceStatus: workv1.ManifestResourceStatus{ - Manifests: []workv1.ManifestCondition{ - { - ResourceMeta: workv1.ManifestResourceMeta{}, - StatusFeedbacks: workv1.StatusFeedbackResult{Values: []workv1.FeedbackValue{}}, - Conditions: []metav1.Condition{}, - }, - }, - }, - }, - }, - wantErr: true, - errSub: "expected exactly one status feedback value", - }, - { - name: "error - wrong feedback name", - bundle: func() *workv1.ManifestWork { - b := buildTestMaestroBundleWithStatusFeedback("b", "ns", validJSON) - b.Status.ResourceStatus.Manifests[0].StatusFeedbacks.Values[0].Name = "wrong" - return b - }(), - wantErr: true, - errSub: "expected status feedback value name to be 'resource', got wrong", - }, - { - name: "error - wrong feedback type", - bundle: func() *workv1.ManifestWork { - b := buildTestMaestroBundleWithStatusFeedback("b", "ns", validJSON) - b.Status.ResourceStatus.Manifests[0].StatusFeedbacks.Values[0].Value.Type = workv1.String - return b - }(), - wantErr: true, - errSub: "expected status feedback value type to be JsonRaw", - }, - { - name: "error - nil JsonRaw", - bundle: func() *workv1.ManifestWork { - b := buildTestMaestroBundleWithStatusFeedback("b", "ns", validJSON) - b.Status.ResourceStatus.Manifests[0].StatusFeedbacks.Values[0].Value.JsonRaw = nil - return b - }(), - wantErr: true, - errSub: "expected status feedback value JsonRaw to be not nil", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := syncer.getSingleResourceStatusFeedbackRawJSONFromMaestroBundle(tt.bundle) - if tt.wantErr { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.errSub) - } else { - require.NoError(t, err) - assert.Equal(t, tt.want, string(got)) - } - }) - } -} - -func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_managementClusterContentResourceIDFromClusterResourceID(t *testing.T) { - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{} - clusterRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/mycluster")) - - got := syncer.managementClusterContentResourceIDFromClusterResourceID(clusterRID, api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster) - require.NotNil(t, got) - assert.Equal(t, got.ResourceType.Type, api.ManagementClusterContentResourceType.Type) - // Name is the last segment of the resource ID (the management cluster content name) - assert.Equal(t, got.Name, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) -} - -func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_calculateManagementClusterContentFromMaestroBundle(t *testing.T) { - clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) - cluster := &api.HCPOpenShiftCluster{ - TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterResourceID}}, - } - ref := &api.MaestroBundleReference{ - Name: api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster, - MaestroAPIMaestroBundleName: "bundle-name", - } - - hc := hsv1beta1.HostedCluster{TypeMeta: metav1.TypeMeta{APIVersion: "hypershift.openshift.io/v1beta1", Kind: "HostedCluster"}, ObjectMeta: metav1.ObjectMeta{Name: "hc1", Namespace: "ns1"}} - hcJSONBytes, err := json.Marshal(hc) - require.NoError(t, err) - validHCJSON := string(hcJSONBytes) - - tests := []struct { - name string - maestroGet func(*maestro.MockClient) - wantDegraded bool - wantKubeContent bool - wantErr bool - errSub string - }{ - { - name: "bundle not found - desired with degraded condition", - maestroGet: func(m *maestro.MockClient) { - m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "bundle-name")) - }, - wantDegraded: true, - wantKubeContent: false, - }, - { - name: "maestro api maestro bundleget error - returns error", - maestroGet: func(m *maestro.MockClient) { - m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(nil, fmt.Errorf("connection error")) - }, - wantErr: true, - errSub: "failed to get Maestro Bundle", - }, - { - name: "bundle has invalid status feedback - desired with degraded", - maestroGet: func(m *maestro.MockClient) { - // Bundle with no status feedback values - b := &workv1.ManifestWork{ - Status: workv1.ManifestWorkStatus{ - ResourceStatus: workv1.ManifestResourceStatus{ - Manifests: []workv1.ManifestCondition{ - {ResourceMeta: workv1.ManifestResourceMeta{}, StatusFeedbacks: workv1.StatusFeedbackResult{Values: []workv1.FeedbackValue{}}, Conditions: []metav1.Condition{}}, - }, - }, - }, - } - m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - }, - wantDegraded: true, - wantKubeContent: false, - }, - { - name: "success - desired with kube content", - maestroGet: func(m *maestro.MockClient) { - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) - m.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - }, - wantDegraded: false, - wantKubeContent: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - tt.maestroGet(mockMaestro) - - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{} - got, err := syncer.calculateManagementClusterContentFromMaestroBundle(context.Background(), cluster, ref, mockMaestro) - if tt.wantErr { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.errSub) - } else { - require.NoError(t, err) - require.NotNil(t, got) - assert.Equal(t, tt.wantKubeContent, got.Status.KubeContent != nil && len(got.Status.KubeContent.Items) > 0) - hasDegradedTrue := controllerutils.IsConditionTrue(got.Status.Conditions, "Degraded") - assert.Equal(t, tt.wantDegraded, hasDegradedTrue) - } - }) - } -} - -func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_readAndPersistMaestroBundleContent(t *testing.T) { - ctx := context.Background() - clusterResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster")) - cluster := &api.HCPOpenShiftCluster{ - TrackedResource: arm.TrackedResource{Resource: arm.Resource{ID: clusterResourceID}}, - } - ref := &api.MaestroBundleReference{ - Name: api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster, - MaestroAPIMaestroBundleName: "bundle-name", - } - hc := hsv1beta1.HostedCluster{TypeMeta: metav1.TypeMeta{APIVersion: "hypershift.openshift.io/v1beta1", Kind: "HostedCluster"}, ObjectMeta: metav1.ObjectMeta{Name: "hc1", Namespace: "ns1"}} - hcJSONBytes, err := json.Marshal(hc) - require.NoError(t, err) - validHCJSON := string(hcJSONBytes) - - t.Run("creates new ManagementClusterContent when not found", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - - mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.ManagementClusterContents("sub", "rg", "cluster") - - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{cosmosClient: mockDB} - err := syncer.readAndPersistMaestroBundleContent(ctx, cluster, ref, mockMaestro) - require.NoError(t, err) - - // Content should have been created (name = bundle internal name) - got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) - require.NoError(t, err) - require.NotNil(t, got) - require.NotNil(t, got.Status.KubeContent) - require.Len(t, got.Status.KubeContent.Items, 1) - }) - - t.Run("replaces existing when content changed", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - - mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.ManagementClusterContents("sub", "rg", "cluster") - // Pre-create existing content with different payload - existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) - existing := &api.ManagementClusterContent{ - CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, - ResourceID: *existingRID, - Status: api.ManagementClusterContentStatus{KubeContent: &metav1.List{Items: []runtime.RawExtension{}}}, - } - _, err := mccCRUD.Create(ctx, existing, nil) - require.NoError(t, err) - - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{cosmosClient: mockDB} - err = syncer.readAndPersistMaestroBundleContent(ctx, cluster, ref, mockMaestro) - require.NoError(t, err) - - got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) - require.NoError(t, err) - require.NotNil(t, got.Status.KubeContent) - require.Len(t, got.Status.KubeContent.Items, 1) - }) - - t.Run("keeps existing kube content when desired has no content (degraded)", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - // Return bundle that has no valid status feedback so desired has no KubeContent - b := &workv1.ManifestWork{ - Status: workv1.ManifestWorkStatus{ - ResourceStatus: workv1.ManifestResourceStatus{ - Manifests: []workv1.ManifestCondition{ - {ResourceMeta: workv1.ManifestResourceMeta{}, StatusFeedbacks: workv1.StatusFeedbackResult{Values: []workv1.FeedbackValue{}}, Conditions: []metav1.Condition{}}, - }, - }, - }, - } - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - - mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.ManagementClusterContents("sub", "rg", "cluster") - existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) - existingContent := &metav1.List{Items: []runtime.RawExtension{{Raw: []byte(`{}`)}}} - existing := &api.ManagementClusterContent{ - CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, - ResourceID: *existingRID, - Status: api.ManagementClusterContentStatus{KubeContent: existingContent}, - } - _, err := mccCRUD.Create(ctx, existing, nil) - require.NoError(t, err) - - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{cosmosClient: mockDB} - err = syncer.readAndPersistMaestroBundleContent(ctx, cluster, ref, mockMaestro) - require.NoError(t, err) - - got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) - require.NoError(t, err) - // Should have kept existing content - require.NotNil(t, got.Status.KubeContent) - assert.Equal(t, existingContent.Items[0].Raw, got.Status.KubeContent.Items[0].Raw) - }) - - t.Run("no replace occurs when content has not changed", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) - // Get is called once when building desired for pre-create, and once inside readAndPersistMaestroBundleContent. - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil).Times(2) - - mockDB := databasetesting.NewMockDBClient() - mccCRUD := mockDB.ManagementClusterContents("sub", "rg", "cluster") - existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) - // Pre-create content that matches exactly what the syncer would compute (same KubeContent and Degraded=False condition) - // so that DeepEqual(existing, desired) is true and Replace is not called. - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{cosmosClient: mockDB} - desired, err := syncer.calculateManagementClusterContentFromMaestroBundle(ctx, cluster, ref, mockMaestro) - require.NoError(t, err) - require.NotNil(t, desired) - existing := &api.ManagementClusterContent{ - CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, - ResourceID: *existingRID, - Status: desired.Status, - } - _, err = mccCRUD.Create(ctx, existing, nil) - require.NoError(t, err) - - err = syncer.readAndPersistMaestroBundleContent(ctx, cluster, ref, mockMaestro) - require.NoError(t, err) - - // Document should still exist with same content (no Replace was needed) - got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) - require.NoError(t, err) - require.NotNil(t, got.Status.KubeContent) - require.Len(t, got.Status.KubeContent.Items, 1) - }) - - t.Run("error occurs when object has been modified in Cosmos since we retrieved it", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - - existingRID := api.Must(azcorearm.ParseResourceID("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster/managementClusterContents/readonlyHypershiftHostedCluster")) - existingDoc := &api.ManagementClusterContent{ - CosmosMetadata: api.CosmosMetadata{ResourceID: existingRID}, - ResourceID: *existingRID, - Status: api.ManagementClusterContentStatus{KubeContent: &metav1.List{Items: []runtime.RawExtension{{Raw: []byte(validHCJSON)}}}}, - } - - // Use error-injecting wrapper to simulate 412 Precondition Failed on Replace - mockDB := &errorInjectingDBClient{ - MockDBClient: databasetesting.NewMockDBClient(), - mccCRUD: &errorInjectingMCCCRUD{ - getResult: existingDoc, - replaceErr: databasetesting.NewPreconditionFailedError(), - }, - } - - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{cosmosClient: mockDB} - err := syncer.readAndPersistMaestroBundleContent(ctx, cluster, ref, mockMaestro) - require.Error(t, err) - assert.Contains(t, err.Error(), "failed to replace ManagementClusterContent") - assert.True(t, database.IsResponseError(err, http.StatusPreconditionFailed), "expected 412 Precondition Failed") - }) - - t.Run("error occurs when managementClusterContentsDBClient.Get fails", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockMaestro := maestro.NewMockClient(ctrl) - b := buildTestMaestroBundleWithStatusFeedback("bundle-name", "ns", validHCJSON) - mockMaestro.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(b, nil) - - getErr := fmt.Errorf("cosmos connection error") - // Use error-injecting wrapper to simulate Get error - mockDB := &errorInjectingDBClient{ - MockDBClient: databasetesting.NewMockDBClient(), - mccCRUD: &errorInjectingMCCCRUD{ - getErr: getErr, - }, - } - - syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{cosmosClient: mockDB} - err := syncer.readAndPersistMaestroBundleContent(ctx, cluster, ref, mockMaestro) - require.Error(t, err) - assert.Contains(t, err.Error(), "failed to get ManagementClusterContent") - assert.Contains(t, err.Error(), "cosmos connection error") - }) -} - func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_SyncOnce_ClusterNotFound(t *testing.T) { mockDBClient := databasetesting.NewMockDBClient() syncer := &readAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer{ @@ -847,7 +255,7 @@ func TestReadAndPersistClusterScopedMaestroReadonlyBundlesContentSyncer_SyncOnce err = syncer.SyncOnce(ctx, key) require.NoError(t, err) - mccCRUD := mockDBClient.ManagementClusterContents(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName) + mccCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).ManagementClusterContents(key.HCPClusterName) got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) require.NoError(t, err) require.NotNil(t, got) diff --git a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go new file mode 100644 index 00000000000..a1d49e6694f --- /dev/null +++ b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller.go @@ -0,0 +1,145 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package controllers + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" + "github.com/Azure/ARO-HCP/backend/pkg/informers" + "github.com/Azure/ARO-HCP/backend/pkg/listers" + "github.com/Azure/ARO-HCP/backend/pkg/maestro" + "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/database" + "github.com/Azure/ARO-HCP/internal/ocm" + "github.com/Azure/ARO-HCP/internal/utils" +) + +// readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer is a controller that reads the Maestro readonly bundles +// references stored in the ServiceProviderNodePool resource, retrieves the Maestro readonly bundles using those +// references, extracts the content of the Maestro readonly bundles and persists them in Cosmos. +// It is not responsible for creating the Maestro readonly bundles themselves. That is the responsibility of +// the createNodePoolScopedMaestroReadonlyBundlesSyncer controller. +// As of now we support reading the content of the Maestro readonly bundle of the Hypershift's NodePools associated +// to the Cluster. +// This controller assumes that it has full ownership of the ManagementClusterContent resource. +type readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer struct { + cooldownChecker controllerutils.CooldownChecker + + activeOperationLister listers.ActiveOperationLister + + cosmosClient database.DBClient + + clusterServiceClient ocm.ClusterServiceClientSpec + + maestroSourceEnvironmentIdentifier string + maestroClientBuilder maestro.MaestroClientBuilder +} + +var _ controllerutils.NodePoolSyncer = (*readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer)(nil) + +func NewReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentController( + activeOperationLister listers.ActiveOperationLister, + cosmosClient database.DBClient, + clusterServiceClient ocm.ClusterServiceClientSpec, + informers informers.BackendInformers, + maestroSourceEnvironmentIdentifier string, + maestroClientBuilder maestro.MaestroClientBuilder, +) controllerutils.Controller { + + syncer := &readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer{ + cooldownChecker: controllerutils.DefaultActiveOperationPrioritizingCooldown(activeOperationLister), + cosmosClient: cosmosClient, + clusterServiceClient: clusterServiceClient, + activeOperationLister: activeOperationLister, + maestroSourceEnvironmentIdentifier: maestroSourceEnvironmentIdentifier, + maestroClientBuilder: maestroClientBuilder, + } + + controller := controllerutils.NewNodePoolWatchingController( + "ReadAndPersistNodePoolScopedMaestroReadonlyBundlesContent", + cosmosClient, + informers, + 1*time.Minute, + syncer, + ) + + return controller +} + +func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) SyncOnce(ctx context.Context, key controllerutils.HCPNodePoolKey) error { + existingNodePool, err := c.cosmosClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName).Get(ctx, key.HCPNodePoolName) + if database.IsResponseError(err, http.StatusNotFound) { + return nil // nodepool doesn't exist, no work to do + } + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get NodePool: %w", err)) + } + if len(existingNodePool.ServiceProviderProperties.ClusterServiceID.String()) == 0 { + // TODO remove this once we have the information all in cosmos. + return nil + } + + existingServiceProviderNodePool, err := database.GetOrCreateServiceProviderNodePool(ctx, c.cosmosClient, key.GetResourceID()) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get or create ServiceProviderNodePool: %w", err)) + } + + // We return early if there are no Maestro Bundle references to process. + if len(existingServiceProviderNodePool.Status.MaestroReadonlyBundles) == 0 { + return nil + } + + // We get the provision shard (management cluster) the CS cluster is allocated to. + // As of now in CS the shard allocation occurs synchronously during aro-hcp cluster creation call in CS API so + // we are guaranteed to have a shard allocated for the cluster. If this changes in the future + // we would need to change the logic in controllers to check that the retrieved cluster has a + // shard allocated. + csClusterID := existingNodePool.ServiceProviderProperties.ClusterServiceID.ClusterID() + csClusterHREF := ocm.GenerateAROHCPClusterHREF(csClusterID) + csClusterInternalID := api.Must(api.NewInternalID(csClusterHREF)) + clusterProvisionShard, err := c.clusterServiceClient.GetClusterProvisionShard(ctx, csClusterInternalID) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to get Cluster Provision Shard from Cluster Service: %w", err)) + } + // We create a new context with a cancel function so we can cancel the Maestro client when the sync is done. + // This is important to avoid leaking resources when the sync is done. + ctx, cancel := context.WithCancel(ctx) + defer cancel() + maestroClient, err := createMaestroClientFromCSProvisionShard(ctx, c.maestroSourceEnvironmentIdentifier, c.maestroClientBuilder, clusterProvisionShard) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to create Maestro client: %w", err)) + } + + managementClusterContentsDBClient := c.cosmosClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName).ManagementClusterContents(key.HCPNodePoolName) + + var syncErrors []error + for _, maestroBundleReference := range existingServiceProviderNodePool.Status.MaestroReadonlyBundles { + err = readAndPersistMaestroReadonlyBundleContent(ctx, existingNodePool.ID, maestroBundleReference, maestroClient, managementClusterContentsDBClient) + if err != nil { + syncErrors = append(syncErrors, utils.TrackError(fmt.Errorf("failed to read and persist NodePool: %w", err))) + } + + } + + return utils.TrackError(errors.Join(syncErrors...)) +} + +func (c *readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer) CooldownChecker() controllerutils.CooldownChecker { + return c.cooldownChecker +} diff --git a/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go new file mode 100644 index 00000000000..0b542a9fe76 --- /dev/null +++ b/backend/pkg/controllers/read_and_persist_nodepool_scoped_maestro_readonly_bundles_content_controller_test.go @@ -0,0 +1,380 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + + "github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils" + "github.com/Azure/ARO-HCP/backend/pkg/maestro" + "github.com/Azure/ARO-HCP/internal/api" + "github.com/Azure/ARO-HCP/internal/api/arm" + "github.com/Azure/ARO-HCP/internal/database" + "github.com/Azure/ARO-HCP/internal/databasetesting" + "github.com/Azure/ARO-HCP/internal/ocm" +) + +// errorInjectingDBClientForNodePoolReadPersist wraps MockDBClient to return error-injecting CRUDs. +type errorInjectingDBClientForNodePoolReadPersist struct { + *databasetesting.MockDBClient + spnpCRUD database.ServiceProviderNodePoolCRUD +} + +func (e *errorInjectingDBClientForNodePoolReadPersist) ServiceProviderNodePools(subscriptionID, resourceGroupName, clusterName, nodePoolName string) database.ServiceProviderNodePoolCRUD { + if e.spnpCRUD != nil { + return e.spnpCRUD + } + return e.MockDBClient.ServiceProviderNodePools(subscriptionID, resourceGroupName, clusterName, nodePoolName) +} + +var _ database.DBClient = &errorInjectingDBClientForNodePoolReadPersist{} + +// errorInjectingSPNPCRUD wraps ServiceProviderNodePoolCRUD to allow error injection. +type errorInjectingSPNPCRUD struct { + database.ServiceProviderNodePoolCRUD + getErr error +} + +func (e *errorInjectingSPNPCRUD) Get(ctx context.Context, resourceID string) (*api.ServiceProviderNodePool, error) { + if e.getErr != nil { + return nil, e.getErr + } + return e.ServiceProviderNodePoolCRUD.Get(ctx, resourceID) +} + +func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnce_NodePoolNotFound(t *testing.T) { + mockDBClient := databasetesting.NewMockDBClient() + syncer := &readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + // No nodepool in DB -> Get returns NotFound -> SyncOnce returns nil (no work to do) + err := syncer.SyncOnce(context.Background(), key) + assert.NoError(t, err) +} + +func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnce_EmptyClusterServiceID(t *testing.T) { + ctrl := gomock.NewController(t) + ctx := context.Background() + + mockDBClient := databasetesting.NewMockDBClient() + mockClusterService := ocm.NewMockClusterServiceClientSpec(ctrl) + + syncer := &readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + clusterServiceClient: mockClusterService, + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.InternalID{}, + }, + } + nodepoolsCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + {Name: bundleInternalName, MaestroAPIMaestroBundleName: "bundle-name"}, + }, + }, + } + spnpCRUD := mockDBClient.ServiceProviderNodePools(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName, key.HCPNodePoolName) + _, err = spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + + // Cluster service ID not yet populated: skip sync (no OCM / Maestro calls). + err = syncer.SyncOnce(ctx, key) + assert.NoError(t, err) +} + +func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnce_GetServiceProviderNodePoolError(t *testing.T) { + ctx := context.Background() + + baseMockDB := databasetesting.NewMockDBClient() + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + nodepoolsCRUD := baseMockDB.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + expectedError := fmt.Errorf("database error") + mockDBClient := &errorInjectingDBClientForNodePoolReadPersist{ + MockDBClient: baseMockDB, + spnpCRUD: &errorInjectingSPNPCRUD{ + getErr: expectedError, + }, + } + + syncer := &readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + } + + err = syncer.SyncOnce(ctx, key) + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to get or create ServiceProviderNodePool") +} + +func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnce_NoMaestroReadonlyBundlesRefs(t *testing.T) { + ctx := context.Background() + mockDBClient := databasetesting.NewMockDBClient() + syncer := &readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + nodepoolsCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + // SPNP with no bundle references -> SyncOnce returns nil (nothing to process) + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + } + spnpCRUD := mockDBClient.ServiceProviderNodePools(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName, key.HCPNodePoolName) + _, err = spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + + err = syncer.SyncOnce(ctx, key) + assert.NoError(t, err) +} + +func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnce_GetProvisionShardError(t *testing.T) { + ctrl := gomock.NewController(t) + ctx := context.Background() + + mockDBClient := databasetesting.NewMockDBClient() + mockClusterService := ocm.NewMockClusterServiceClientSpec(ctrl) + + syncer := &readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + clusterServiceClient: mockClusterService, + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + nodepoolsCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + bundleInternalName := api.MaestroBundleInternalNameReadonlyHypershiftNodePool + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + {Name: bundleInternalName, MaestroAPIMaestroBundleName: "bundle-name"}, + }, + }, + } + spnpCRUD := mockDBClient.ServiceProviderNodePools(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName, key.HCPNodePoolName) + _, err = spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + + mockClusterService.EXPECT(). + GetClusterProvisionShard(gomock.Any(), nodepool.ServiceProviderProperties.ClusterServiceID). + Return(nil, fmt.Errorf("provision shard error")) + + err = syncer.SyncOnce(ctx, key) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to get Cluster Provision Shard") +} + +func TestReadAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer_SyncOnce_ReadAndPersistFlow(t *testing.T) { + ctrl := gomock.NewController(t) + ctx := context.Background() + + mockDBClient := databasetesting.NewMockDBClient() + mockClusterService := ocm.NewMockClusterServiceClientSpec(ctrl) + mockMaestroBuilder := maestro.NewMockMaestroClientBuilder(ctrl) + mockMaestroClient := maestro.NewMockClient(ctrl) + + syncer := &readAndPersistNodePoolScopedMaestroReadonlyBundlesContentSyncer{ + cooldownChecker: &alwaysSyncCooldownChecker{}, + cosmosClient: mockDBClient, + clusterServiceClient: mockClusterService, + maestroClientBuilder: mockMaestroBuilder, + maestroSourceEnvironmentIdentifier: "test-env", + } + + key := controllerutils.HCPNodePoolKey{ + SubscriptionID: "test-sub", + ResourceGroupName: "test-rg", + HCPClusterName: "test-cluster", + HCPNodePoolName: "test-nodepool", + } + + nodepoolResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool")) + nodepool := &api.HCPOpenShiftClusterNodePool{ + TrackedResource: arm.TrackedResource{ + Resource: arm.Resource{ + ID: nodepoolResourceID, + Name: "test-nodepool", + }, + }, + ServiceProviderProperties: api.HCPOpenShiftClusterNodePoolServiceProviderProperties{ + ClusterServiceID: api.Must(api.NewInternalID("/api/aro_hcp/v1alpha1/clusters/11111111111111111111111111111111")), + }, + } + nodepoolsCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName) + _, err := nodepoolsCRUD.Create(ctx, nodepool, nil) + require.NoError(t, err) + + spnpResourceID := api.Must(azcorearm.ParseResourceID("/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/test-cluster/nodePools/test-nodepool/serviceProviderNodePools/default")) + spnp := &api.ServiceProviderNodePool{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: spnpResourceID}, + ResourceID: *spnpResourceID, + Status: api.ServiceProviderNodePoolStatus{ + MaestroReadonlyBundles: api.MaestroBundleReferenceList{ + {Name: api.MaestroBundleInternalNameReadonlyHypershiftNodePool, MaestroAPIMaestroBundleName: "bundle-name"}, + }, + }, + } + spnpCRUD := mockDBClient.ServiceProviderNodePools(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName, key.HCPNodePoolName) + _, err = spnpCRUD.Create(ctx, spnp, nil) + require.NoError(t, err) + + provisionShard := buildTestProvisionShard("test-consumer") + mockClusterService.EXPECT(). + GetClusterProvisionShard(gomock.Any(), nodepool.ServiceProviderProperties.ClusterServiceID). + Return(provisionShard, nil) + + restEndpoint := provisionShard.MaestroConfig().RestApiConfig().Url() + grpcEndpoint := provisionShard.MaestroConfig().GrpcApiConfig().Url() + consumerName := provisionShard.MaestroConfig().ConsumerName() + sourceID := maestro.GenerateMaestroSourceID("test-env", provisionShard.ID()) + mockMaestroBuilder.EXPECT(). + NewClient(gomock.Any(), restEndpoint, grpcEndpoint, consumerName, sourceID). + Return(mockMaestroClient, nil) + + validNPJSON := `{"apiVersion":"hypershift.openshift.io/v1beta1","kind":"NodePool","metadata":{"name":"np1","namespace":"ns1"}}` + bundle := buildTestMaestroBundleWithStatusFeedback("bundle-name", "test-consumer", validNPJSON) + mockMaestroClient.EXPECT().Get(gomock.Any(), "bundle-name", gomock.Any()).Return(bundle, nil) + + err = syncer.SyncOnce(ctx, key) + require.NoError(t, err) + + mccCRUD := mockDBClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).NodePools(key.HCPClusterName).ManagementClusterContents(key.HCPNodePoolName) + got, err := mccCRUD.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftNodePool)) + require.NoError(t, err) + require.NotNil(t, got) + require.NotNil(t, got.Status.KubeContent) + require.Len(t, got.Status.KubeContent.Items, 1) + + var u unstructured.Unstructured + err = json.Unmarshal(got.Status.KubeContent.Items[0].Raw, &u) + require.NoError(t, err) + assert.Equal(t, "NodePool", u.GetKind()) + assert.Equal(t, "np1", u.GetName()) +} diff --git a/backend/pkg/controllers/upgradecontrollers/control_plane_active_version_controller.go b/backend/pkg/controllers/upgradecontrollers/control_plane_active_version_controller.go index 72449152077..8417d4e8053 100644 --- a/backend/pkg/controllers/upgradecontrollers/control_plane_active_version_controller.go +++ b/backend/pkg/controllers/upgradecontrollers/control_plane_active_version_controller.go @@ -85,7 +85,7 @@ func (c *controlPlaneActiveVersionSyncer) SyncOnce(ctx context.Context, key cont return utils.TrackError(fmt.Errorf("failed to get Cluster: %w", err)) } - managementClusterContentClient := c.cosmosClient.ManagementClusterContents(key.SubscriptionID, key.ResourceGroupName, key.HCPClusterName) + managementClusterContentClient := c.cosmosClient.HCPClusters(key.SubscriptionID, key.ResourceGroupName).ManagementClusterContents(key.HCPClusterName) managementClusterContent, err := managementClusterContentClient.Get(ctx, string(api.MaestroBundleInternalNameReadonlyHypershiftHostedCluster)) if database.IsResponseError(err, http.StatusNotFound) { return nil diff --git a/backend/pkg/controllers/upgradecontrollers/control_plane_active_version_controller_test.go b/backend/pkg/controllers/upgradecontrollers/control_plane_active_version_controller_test.go index 3064fcb7a64..3f247b4d6d3 100644 --- a/backend/pkg/controllers/upgradecontrollers/control_plane_active_version_controller_test.go +++ b/backend/pkg/controllers/upgradecontrollers/control_plane_active_version_controller_test.go @@ -309,6 +309,6 @@ func createManagementClusterContentWithKubeContentItems(t *testing.T, ctx contex }, }, } - _, err := mockDB.ManagementClusterContents(testSubscriptionID, testResourceGroupName, testClusterName).Create(ctx, managementClusterContent, nil) + _, err := mockDB.HCPClusters(testSubscriptionID, testResourceGroupName).ManagementClusterContents(testClusterName).Create(ctx, managementClusterContent, nil) require.NoError(t, err) } diff --git a/backend/pkg/informers/informers.go b/backend/pkg/informers/informers.go index 36a0046b16d..d98cd646bf8 100644 --- a/backend/pkg/informers/informers.go +++ b/backend/pkg/informers/informers.go @@ -340,13 +340,13 @@ func NewServiceProviderClusterInformerWithRelistDuration(lister database.GlobalL } // NewManagementClusterContentInformer creates an unstarted SharedIndexInformer for management cluster contents -// with a cluster index using the default relist duration. +// with cluster and node pool indexes using the default relist duration. func NewManagementClusterContentInformer(lister database.GlobalLister[api.ManagementClusterContent]) cache.SharedIndexInformer { return NewManagementClusterContentInformerWithRelistDuration(lister, ManagementClusterContentRelistDuration) } // NewManagementClusterContentInformerWithRelistDuration creates an unstarted SharedIndexInformer for management cluster contents -// with a cluster index and a configurable relist duration. +// with cluster and node pool indexes and a configurable relist duration. func NewManagementClusterContentInformerWithRelistDuration(lister database.GlobalLister[api.ManagementClusterContent], relistDuration time.Duration) cache.SharedIndexInformer { lw := &cache.ListWatch{ ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { @@ -381,7 +381,8 @@ func NewManagementClusterContentInformerWithRelistDuration(lister database.Globa cache.SharedIndexInformerOptions{ ResyncPeriod: 1 * time.Hour, // this is only a default. Shorter resyncs can be added when registering handlers. Indexers: cache.Indexers{ - listers.ByCluster: clusterResourceIDIndexFunc, + listers.ByCluster: clusterResourceIDIndexFunc, + listers.ByNodePool: nodePoolResourceIDIndexFunc, }, }, ) @@ -668,8 +669,8 @@ func activeOperationClusterIndexFunc(obj interface{}) ([]string, error) { return clusterResourceIDFromResourceID(op.ExternalID) } -// nodePoolResourceIDIndexFunc indexes service provider node pools by their parent node pool -// resource ID, derived from the embedded CosmosMetadata. +// nodePoolResourceIDIndexFunc indexes objects by the node pool resource ID of their nearest +// nodePool ancestor in the ARM path (Cosmos metadata resource ID). func nodePoolResourceIDIndexFunc(obj interface{}) ([]string, error) { switch castObj := obj.(type) { case arm.CosmosMetadataAccessor: diff --git a/backend/pkg/informers/types.go b/backend/pkg/informers/types.go index eca9492a230..449ade8c4ea 100644 --- a/backend/pkg/informers/types.go +++ b/backend/pkg/informers/types.go @@ -36,6 +36,8 @@ type BackendInformers interface { ServiceProviderClusters() (cache.SharedIndexInformer, listers.ServiceProviderClusterLister) ServiceProviderNodePools() (cache.SharedIndexInformer, listers.ServiceProviderNodePoolLister) Controllers() (cache.SharedIndexInformer, listers.ControllerLister) + // ManagementClusterContents is the single shared informer for all managementClusterContents documents belonging + // to different resource types. ManagementClusterContents() (cache.SharedIndexInformer, listers.ManagementClusterContentLister) BillingDocs() (cache.SharedIndexInformer, listers.BillingLister) diff --git a/backend/pkg/listers/management_cluster_content_lister.go b/backend/pkg/listers/management_cluster_content_lister.go index e2b8e115827..02a98227f0b 100644 --- a/backend/pkg/listers/management_cluster_content_lister.go +++ b/backend/pkg/listers/management_cluster_content_lister.go @@ -22,10 +22,11 @@ import ( "github.com/Azure/ARO-HCP/internal/api" ) -// ManagementClusterContentLister lists ManagementClusterContent from an informer's indexer. +// ManagementClusterContentLister lists ManagementClusterContent from the shared informer indexer. type ManagementClusterContentLister interface { List(ctx context.Context) ([]*api.ManagementClusterContent, error) ListForCluster(ctx context.Context, subscriptionID, resourceGroupName, clusterName string) ([]*api.ManagementClusterContent, error) + ListForNodePool(ctx context.Context, subscriptionName, resourceGroupName, clusterName, nodePoolName string) ([]*api.ManagementClusterContent, error) } // managementClusterContentLister implements ManagementClusterContentLister backed by a SharedIndexInformer. @@ -48,3 +49,8 @@ func (l *managementClusterContentLister) ListForCluster(ctx context.Context, sub key := api.ToClusterResourceIDString(subscriptionID, resourceGroupName, clusterName) return listFromIndex[api.ManagementClusterContent](l.indexer, ByCluster, key) } + +func (l *managementClusterContentLister) ListForNodePool(ctx context.Context, subscriptionName, resourceGroupName, clusterName, nodePoolName string) ([]*api.ManagementClusterContent, error) { + key := api.ToNodePoolResourceIDString(subscriptionName, resourceGroupName, clusterName, nodePoolName) + return listFromIndex[api.ManagementClusterContent](l.indexer, ByNodePool, key) +} diff --git a/backend/pkg/listertesting/db_listers.go b/backend/pkg/listertesting/db_listers.go index 41048b4558b..c91ecb16bd3 100644 --- a/backend/pkg/listertesting/db_listers.go +++ b/backend/pkg/listertesting/db_listers.go @@ -248,6 +248,46 @@ func (l *DBControllerLister) listWithPrefix(ctx context.Context, prefix string) return result, nil } +// DBManagementClusterContentLister implements listers.ManagementClusterContentLister backed by a database.DBClient. +type DBManagementClusterContentLister struct { + DBClient database.DBClient +} + +var _ listers.ManagementClusterContentLister = &DBManagementClusterContentLister{} + +func (l *DBManagementClusterContentLister) List(ctx context.Context) ([]*api.ManagementClusterContent, error) { + iter, err := l.DBClient.GlobalListers().ManagementClusterContents().List(ctx, nil) + if err != nil { + return nil, err + } + return collectFromIterator(ctx, iter) +} + +func (l *DBManagementClusterContentLister) ListForCluster(ctx context.Context, subscriptionID, resourceGroupName, clusterName string) ([]*api.ManagementClusterContent, error) { + prefix := api.ToClusterResourceIDString(subscriptionID, resourceGroupName, clusterName) + return l.listMCCWithPrefix(ctx, prefix) +} + +func (l *DBManagementClusterContentLister) ListForNodePool(ctx context.Context, subscriptionName, resourceGroupName, clusterName, nodePoolName string) ([]*api.ManagementClusterContent, error) { + prefix := api.ToNodePoolResourceIDString(subscriptionName, resourceGroupName, clusterName, nodePoolName) + return l.listMCCWithPrefix(ctx, prefix) +} + +func (l *DBManagementClusterContentLister) listMCCWithPrefix(ctx context.Context, prefix string) ([]*api.ManagementClusterContent, error) { + all, err := l.List(ctx) + if err != nil { + return nil, err + } + var result []*api.ManagementClusterContent + for _, mcc := range all { + rid := mcc.GetResourceID() + if rid != nil && strings.HasPrefix(strings.ToLower(rid.String()), strings.ToLower(prefix)) { + result = append(result, mcc) + } + } + return result, nil +} + // DBSubscriptionLister implements listers.SubscriptionLister backed by a database.DBClient. type DBSubscriptionLister struct { DBClient database.DBClient diff --git a/backend/pkg/listertesting/db_listers_test.go b/backend/pkg/listertesting/db_listers_test.go index a9f47f97aec..451a8545c94 100644 --- a/backend/pkg/listertesting/db_listers_test.go +++ b/backend/pkg/listertesting/db_listers_test.go @@ -355,6 +355,61 @@ func TestDBControllerLister(t *testing.T) { }) } +func TestDBManagementClusterContentLister(t *testing.T) { + ctx := context.Background() + + cluster1 := newTestCluster(testSubscriptionID, testResourceGroupName, testClusterName) + cluster2 := newTestCluster(testSubscriptionID, testResourceGroupName, testClusterName2) + np := newTestNodePool(testSubscriptionID, testResourceGroupName, testClusterName, testNodePoolName) + + mccCluster1 := newTestClusterScopedManagementClusterContent(testSubscriptionID, testResourceGroupName, testClusterName, "mcc-under-cluster") + mccCluster2 := newTestClusterScopedManagementClusterContent(testSubscriptionID, testResourceGroupName, testClusterName2, "mcc-under-cluster2") + mccNP := newTestNodePoolScopedManagementClusterContent(testSubscriptionID, testResourceGroupName, testClusterName, testNodePoolName, "mcc-under-np") + + mockDB, err := databasetesting.NewMockDBClientWithResources(ctx, []any{ + cluster1, cluster2, np, + mccCluster1, mccCluster2, mccNP, + }) + require.NoError(t, err) + + lister := &DBManagementClusterContentLister{DBClient: mockDB} + + t.Run("List returns all management cluster contents", func(t *testing.T) { + result, err := lister.List(ctx) + require.NoError(t, err) + assert.Len(t, result, 3) + }) + + t.Run("ListForCluster returns cluster-scoped and node-pool-scoped MCC for that cluster", func(t *testing.T) { + result, err := lister.ListForCluster(ctx, testSubscriptionID, testResourceGroupName, testClusterName) + require.NoError(t, err) + assert.Len(t, result, 2) + names := []string{result[0].GetResourceID().Name, result[1].GetResourceID().Name} + assert.Contains(t, names, "mcc-under-cluster") + assert.Contains(t, names, "mcc-under-np") + }) + + t.Run("ListForCluster returns only MCC for other cluster", func(t *testing.T) { + result, err := lister.ListForCluster(ctx, testSubscriptionID, testResourceGroupName, testClusterName2) + require.NoError(t, err) + require.Len(t, result, 1) + assert.Equal(t, "mcc-under-cluster2", result[0].GetResourceID().Name) + }) + + t.Run("ListForNodePool returns only node-pool-scoped MCC", func(t *testing.T) { + result, err := lister.ListForNodePool(ctx, testSubscriptionID, testResourceGroupName, testClusterName, testNodePoolName) + require.NoError(t, err) + require.Len(t, result, 1) + assert.Equal(t, "mcc-under-np", result[0].GetResourceID().Name) + }) + + t.Run("ListForNodePool returns empty for non-existent node pool", func(t *testing.T) { + result, err := lister.ListForNodePool(ctx, testSubscriptionID, testResourceGroupName, testClusterName, "non-existent") + require.NoError(t, err) + assert.Empty(t, result) + }) +} + func TestDBClusterListerWithEmptyDB(t *testing.T) { ctx := context.Background() mockDB := databasetesting.NewMockDBClient() diff --git a/backend/pkg/listertesting/slice_listers_test.go b/backend/pkg/listertesting/slice_listers_test.go index e7dcd9c4cb9..4052a273a9b 100644 --- a/backend/pkg/listertesting/slice_listers_test.go +++ b/backend/pkg/listertesting/slice_listers_test.go @@ -459,3 +459,30 @@ func newTestSubscription(subscriptionID string) *arm.Subscription { ResourceID: resourceID, } } + +func newTestClusterScopedManagementClusterContent(subscriptionID, resourceGroupName, clusterName, mccName string) *api.ManagementClusterContent { + resourceID := api.Must(azcorearm.ParseResourceID( + "/subscriptions/" + subscriptionID + + "/resourceGroups/" + resourceGroupName + + "/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/" + clusterName + + "/managementClusterContents/" + mccName, + )) + return &api.ManagementClusterContent{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: resourceID}, + ResourceID: *resourceID, + } +} + +func newTestNodePoolScopedManagementClusterContent(subscriptionID, resourceGroupName, clusterName, nodePoolName, mccName string) *api.ManagementClusterContent { + resourceID := api.Must(azcorearm.ParseResourceID( + "/subscriptions/" + subscriptionID + + "/resourceGroups/" + resourceGroupName + + "/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/" + clusterName + + "/nodePools/" + nodePoolName + + "/managementClusterContents/" + mccName, + )) + return &api.ManagementClusterContent{ + CosmosMetadata: arm.CosmosMetadata{ResourceID: resourceID}, + ResourceID: *resourceID, + } +} diff --git a/backend/pkg/maestro/maestro_api_maestro_bundle_name_generator.go b/backend/pkg/maestro/maestro_api_maestro_bundle_name_generator.go new file mode 100644 index 00000000000..2d3dcc2890e --- /dev/null +++ b/backend/pkg/maestro/maestro_api_maestro_bundle_name_generator.go @@ -0,0 +1,71 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maestro + +import ( + "fmt" + + "github.com/google/uuid" + + "github.com/Azure/ARO-HCP/internal/utils" +) + +// MaestroAPIMaestroBundleNameGenerator is an interface that defines a method to generate a new Maestro API Maestro Bundle name. +// The generated name must be globally unique within a given Maestro Consumer Name and Maestro Source ID. +// It can be used to generate a new Maestro API Maestro Bundle name for a new Maestro Bundle reference. +type MaestroAPIMaestroBundleNameGenerator interface { + // NewMaestroAPIMaestroBundleName generates a new Maestro API Maestro Bundle name. + // The generated name must be globally unique within a given Maestro Consumer Name and Maestro Source ID. + NewMaestroAPIMaestroBundleName() (string, error) +} + +// NewMaestroAPIMaestroBundleNameGenerator creates a new Maestro API Maestro Bundle name generator. +// The generator generates a new Maestro API Maestro Bundle name whose value is a UUIDv4. +func NewMaestroAPIMaestroBundleNameGenerator() MaestroAPIMaestroBundleNameGenerator { + return &maestroAPIMaestroBundleNameGenerator{ + uuidV4Generator: uuid.NewRandom, + } +} + +type maestroAPIMaestroBundleNameGenerator struct { + uuidV4Generator func() (uuid.UUID, error) +} + +// generateNewMaestroAPIMaestroBundleName generates a new Maestro API Maestro Bundle name. +// Used to generate a new Maestro API Maestro Bundle name for a new Maestro Bundle reference. +// The generated name is a UUIDv4. +func (c *maestroAPIMaestroBundleNameGenerator) NewMaestroAPIMaestroBundleName() (string, error) { + newUUIDForMaestroAPIMaestroBundleName, err := c.uuidV4Generator() + if err != nil { + return "", utils.TrackError(fmt.Errorf("failed to generate UUIDv4 for Maestro API Maestro Bundle name: %w", err)) + } + return newUUIDForMaestroAPIMaestroBundleName.String(), nil +} + +type alwaysSameNameMaestroAPIMaestroBundleNameGenerator struct { + maestroAPIMaestroBundleName string +} + +// NewAlwaysSameNameMaestroAPIMaestroBundleNameGenerator creates a new Maestro API Maestro Bundle name generator that always returns the same name. +// This is useful for testing purposes. +func NewAlwaysSameNameMaestroAPIMaestroBundleNameGenerator(maestroAPIMaestroBundleName string) MaestroAPIMaestroBundleNameGenerator { + return &alwaysSameNameMaestroAPIMaestroBundleNameGenerator{ + maestroAPIMaestroBundleName: maestroAPIMaestroBundleName, + } +} + +func (c *alwaysSameNameMaestroAPIMaestroBundleNameGenerator) NewMaestroAPIMaestroBundleName() (string, error) { + return c.maestroAPIMaestroBundleName, nil +} diff --git a/backend/pkg/maestro/maestro_api_maestro_bundle_name_generator_test.go b/backend/pkg/maestro/maestro_api_maestro_bundle_name_generator_test.go new file mode 100644 index 00000000000..67ac94f2cf0 --- /dev/null +++ b/backend/pkg/maestro/maestro_api_maestro_bundle_name_generator_test.go @@ -0,0 +1,45 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maestro + +import ( + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMaestroAPIMaestroBundleNameGenerator_NewMaestroAPIMaestroBundleName(t *testing.T) { + generator := NewMaestroAPIMaestroBundleNameGenerator() + + // Test successful generation + name1, err := generator.NewMaestroAPIMaestroBundleName() + require.NoError(t, err) + assert.NotEmpty(t, name1) + + // Verify it's a valid UUID + _, err = uuid.Parse(name1) + assert.NoError(t, err, "Generated name should be a valid UUID") + + // Test that multiple calls generate different UUIDs + name2, err := generator.NewMaestroAPIMaestroBundleName() + require.NoError(t, err) + assert.NotEqual(t, name1, name2, "Multiple calls should generate different UUIDs") + + // Verify second name is also a valid UUID + _, err = uuid.Parse(name2) + assert.NoError(t, err, "Second generated name should also be a valid UUID") +} diff --git a/backend/pkg/maestro/maestro_bundle.go b/backend/pkg/maestro/maestro_bundle.go new file mode 100644 index 00000000000..31855ee91f8 --- /dev/null +++ b/backend/pkg/maestro/maestro_bundle.go @@ -0,0 +1,79 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maestro + +import ( + "context" + "fmt" + + workv1 "open-cluster-management.io/api/work/v1" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/Azure/ARO-HCP/internal/utils" +) + +// GetOrCreateMaestroBundle gets (or creates if it does not exist) a Maestro Bundle for a given Maestro Bundle namespaced name. +func GetOrCreateMaestroBundle(ctx context.Context, maestroClient Client, maestroBundle *workv1.ManifestWork) (*workv1.ManifestWork, error) { + logger := utils.LoggerFromContext(ctx) + existingMaestroBundle, err := maestroClient.Get(ctx, maestroBundle.Name, metav1.GetOptions{}) + if err == nil { + logger.Info(fmt.Sprintf("retrieved maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) + return existingMaestroBundle, nil + } + if !k8serrors.IsNotFound(err) { + logger.Error(err, "failed to get Maestro Bundle and it is not already exists error") + return nil, utils.TrackError(fmt.Errorf("failed to get Maestro Bundle: %w", err)) + } + + logger.Info(fmt.Sprintf("attempting to create maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) + existingMaestroBundle, err = maestroClient.Create(ctx, maestroBundle, metav1.CreateOptions{}) + if err == nil { + logger.Info(fmt.Sprintf("created maestro bundle name %s with resource name %s", maestroBundle.Name, maestroBundle.Spec.ManifestConfigs[0].ResourceIdentifier.Name)) + return existingMaestroBundle, nil + } + if !k8serrors.IsAlreadyExists(err) { + logger.Error(err, "failed to create Maestro Bundle and it is not already exists error") + return nil, utils.TrackError(fmt.Errorf("failed to create Maestro Bundle: %w", err)) + } + logger.Error(err, "failed to create Maestro Bundle because it returned already exists error. Attempting to get it again") + existingMaestroBundle, err = maestroClient.Get(ctx, maestroBundle.Name, metav1.GetOptions{}) + return existingMaestroBundle, err +} + +// ForEachMaestroBundle lists all Maestro Bundles across all pages matching opts using the Maestro client client and calls fn +// for each Maestro Bundle. +// Pagination can be controlled by setting the Limit attribute in opts. +// fn is called for each bundle in page order. If fn returns an error, iteration stops and the error is returned. +func ForEachMaestroBundle(ctx context.Context, client Client, opts metav1.ListOptions, fn func(*workv1.ManifestWork) error) error { + for { + bundles, err := client.List(ctx, opts) + if err != nil { + return utils.TrackError(fmt.Errorf("failed to list Maestro Bundles: %w", err)) + } + for i := range bundles.Items { + if err := fn(&bundles.Items[i]); err != nil { + return err + } + } + token := bundles.GetContinue() + if token == "" { + break + } + opts.Continue = token + } + return nil +} diff --git a/backend/pkg/maestro/maestro_bundle_test.go b/backend/pkg/maestro/maestro_bundle_test.go new file mode 100644 index 00000000000..3b38b23cc1f --- /dev/null +++ b/backend/pkg/maestro/maestro_bundle_test.go @@ -0,0 +1,237 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maestro + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + gomock "go.uber.org/mock/gomock" + workv1 "open-cluster-management.io/api/work/v1" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func TestGetOrCreateMaestroBundle(t *testing.T) { + desiredBundle := &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-maestro-api-maestro-bundle-name", + Namespace: "test-maestro-consumer", + }, + Spec: workv1.ManifestWorkSpec{ + ManifestConfigs: []workv1.ManifestConfigOption{ + { + ResourceIdentifier: workv1.ResourceIdentifier{ + Name: "hostedcluster-name", + Namespace: "ocm-testenv-11111111111111111111111111111111", + }, + }, + }, + }, + } + + tests := []struct { + name string + setupMock func(*MockClient, *workv1.ManifestWork) + wantBundle *workv1.ManifestWork + wantErr bool + errSubstr string + }{ + { + name: "returns existing bundle if it already exists", + setupMock: func(m *MockClient, want *workv1.ManifestWork) { + m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(want, nil) + }, + wantBundle: &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-maestro-api-maestro-bundle-name", Namespace: "test-maestro-consumer", UID: "existing-uid", + }, + }, + }, + { + name: "creates new bundle if it does not exist", + setupMock: func(m *MockClient, want *workv1.ManifestWork) { + m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "not-found")) + m.EXPECT().Create(gomock.Any(), desiredBundle, gomock.Any()).Return(want, nil) + }, + wantBundle: &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-maestro-api-maestro-bundle-name", Namespace: "test-maestro-consumer", UID: "new-uid", + }, + }, + }, + { + name: "returns existing bundle when internal call to create returns AlreadyExists and then the following get succeeds", + setupMock: func(m *MockClient, want *workv1.ManifestWork) { + m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "not-found")) + m.EXPECT().Create(gomock.Any(), desiredBundle, gomock.Any()).Return(nil, k8serrors.NewAlreadyExists(schema.GroupResource{}, "test-maestro-api-maestro-bundle-name")) + m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(want, nil) + }, + wantBundle: &workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-maestro-api-maestro-bundle-name", Namespace: "test-maestro-consumer", UID: "existing-uid", + }, + }, + }, + { + name: "returns error if it fails to get the bundle", + setupMock: func(m *MockClient, _ *workv1.ManifestWork) { + m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(nil, fmt.Errorf("connection error")) + }, + wantErr: true, + errSubstr: "failed to get Maestro Bundle", + }, + { + name: "returns error if it fails to create the bundle", + setupMock: func(m *MockClient, _ *workv1.ManifestWork) { + m.EXPECT().Get(gomock.Any(), "test-maestro-api-maestro-bundle-name", gomock.Any()).Return(nil, k8serrors.NewNotFound(schema.GroupResource{}, "test-maestro-api-maestro-bundle-name")) + m.EXPECT().Create(gomock.Any(), desiredBundle, gomock.Any()).Return(nil, fmt.Errorf("maestro API error")) + }, + wantErr: true, + errSubstr: "failed to create Maestro Bundle: maestro API error", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := NewMockClient(ctrl) + tt.setupMock(mockMaestro, tt.wantBundle) + + result, err := GetOrCreateMaestroBundle(context.Background(), mockMaestro, desiredBundle) + + if tt.wantErr { + require.Error(t, err) + assert.Nil(t, result) + assert.Contains(t, err.Error(), tt.errSubstr) + } else { + require.NoError(t, err) + assert.Equal(t, tt.wantBundle, result) + } + }) + } +} + +func TestForEachMaestroBundle(t *testing.T) { + ctx := context.Background() + callbackErr := errors.New("stop here") + + tests := []struct { + name string + setupMock func(*MockClient) + listOpts metav1.ListOptions + errAfterName string + wantNames []string + wantErr bool + errContains []string + errIs error + }{ + { + name: "empty list", + setupMock: func(m *MockClient) { + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{ + Items: []workv1.ManifestWork{}, + }, nil) + }, + wantNames: []string{}, + }, + { + name: "invokes fn for each bundle in order on one page", + setupMock: func(m *MockClient) { + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{ + Items: []workv1.ManifestWork{ + {ObjectMeta: metav1.ObjectMeta{Name: "a"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "b"}}, + }, + }, nil) + }, + wantNames: []string{"a", "b"}, + }, + { + name: "paginates using continue token", + setupMock: func(m *MockClient) { + m.EXPECT().List(gomock.Any(), metav1.ListOptions{Limit: 10}).Return(&workv1.ManifestWorkList{ + ListMeta: metav1.ListMeta{Continue: "next-page"}, + Items: []workv1.ManifestWork{{ObjectMeta: metav1.ObjectMeta{Name: "first"}}}, + }, nil) + m.EXPECT().List(gomock.Any(), metav1.ListOptions{Limit: 10, Continue: "next-page"}).Return(&workv1.ManifestWorkList{ + Items: []workv1.ManifestWork{{ObjectMeta: metav1.ObjectMeta{Name: "second"}}}, + }, nil) + }, + listOpts: metav1.ListOptions{Limit: 10}, + wantNames: []string{"first", "second"}, + }, + { + name: "returns wrapped error when list fails", + setupMock: func(m *MockClient) { + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, fmt.Errorf("upstream failure")) + }, + wantNames: []string{}, + wantErr: true, + errContains: []string{"failed to list Maestro Bundles", "upstream failure"}, + }, + { + name: "returns callback error and stops iteration", + setupMock: func(m *MockClient) { + m.EXPECT().List(gomock.Any(), gomock.Any()).Return(&workv1.ManifestWorkList{ + Items: []workv1.ManifestWork{ + {ObjectMeta: metav1.ObjectMeta{Name: "a"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "b"}}, + }, + }, nil) + }, + errAfterName: "a", + wantNames: []string{"a"}, + wantErr: true, + errIs: callbackErr, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockMaestro := NewMockClient(ctrl) + tt.setupMock(mockMaestro) + + names := make([]string, 0) + err := ForEachMaestroBundle(ctx, mockMaestro, tt.listOpts, func(mw *workv1.ManifestWork) error { + names = append(names, mw.Name) + if tt.errAfterName != "" && mw.Name == tt.errAfterName { + return callbackErr + } + return nil + }) + + assert.Equal(t, tt.wantNames, names) + if tt.wantErr { + require.Error(t, err) + if tt.errIs != nil { + require.ErrorIs(t, err, tt.errIs) + } + for _, sub := range tt.errContains { + assert.Contains(t, err.Error(), sub) + } + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/frontend/pkg/frontend/frontend_test.go b/frontend/pkg/frontend/frontend_test.go index 46ff678776d..7868a66b1db 100644 --- a/frontend/pkg/frontend/frontend_test.go +++ b/frontend/pkg/frontend/frontend_test.go @@ -54,7 +54,7 @@ func newClusterResourceID(t *testing.T) *azcorearm.ResourceID { } func newClusterInternalID(t *testing.T) ocm.InternalID { - internalID, err := api.NewInternalID(ocm.GenerateClusterHREF("myCluster")) + internalID, err := api.NewInternalID(ocm.GenerateOCMCommercialClusterHREF("myCluster")) require.NoError(t, err) return internalID } @@ -636,7 +636,7 @@ func TestRequestAdminCredential(t *testing.T) { mockCSClient.EXPECT(). PostBreakGlassCredential(gomock.Any(), clusterInternalID). Return(cmv1.NewBreakGlassCredential(). - HREF(ocm.GenerateBreakGlassCredentialHREF(clusterInternalID.String(), "0")).Build()) + HREF(ocm.GenerateOCMCommercialBreakGlassCredentialHREF(clusterInternalID.String(), "0")).Build()) } subs := map[string]*arm.Subscription{ diff --git a/frontend/pkg/frontend/helpers_test.go b/frontend/pkg/frontend/helpers_test.go index cc305f9f077..660bf8e0299 100644 --- a/frontend/pkg/frontend/helpers_test.go +++ b/frontend/pkg/frontend/helpers_test.go @@ -121,7 +121,7 @@ func TestCheckForProvisioningStateConflict(t *testing.T) { // Pre-populate the parent cluster in the database for nested resources (node pool, external auth) if tt.parentConflict != nil { parentResourceID := resourceID.Parent - clusterInternalID := api.Must(api.NewInternalID(ocm.GenerateClusterHREF("testCluster"))) + clusterInternalID := api.Must(api.NewInternalID(ocm.GenerateOCMCommercialClusterHREF("testCluster"))) parentCluster := &api.HCPOpenShiftCluster{ TrackedResource: arm.TrackedResource{ Resource: arm.Resource{ @@ -168,7 +168,7 @@ func TestCheckForProvisioningStateConflict(t *testing.T) { parentResourceID := resourceID.Parent if parentResourceID.ResourceType.Namespace == resourceID.ResourceType.Namespace { // Pre-populate the parent cluster with the test provisioning state - clusterInternalID := api.Must(api.NewInternalID(ocm.GenerateClusterHREF("testCluster"))) + clusterInternalID := api.Must(api.NewInternalID(ocm.GenerateOCMCommercialClusterHREF("testCluster"))) parentCluster := &api.HCPOpenShiftCluster{ TrackedResource: arm.TrackedResource{ Resource: arm.Resource{ diff --git a/internal/api/registry.go b/internal/api/registry.go index dca72a11760..15c501f94f1 100644 --- a/internal/api/registry.go +++ b/internal/api/registry.go @@ -81,20 +81,23 @@ const ( ) var ( - OperationStatusResourceType = azcorearm.NewResourceType(ProviderNamespace, OperationStatusResourceTypeName) - ClusterResourceType = azcorearm.NewResourceType(ProviderNamespace, ClusterResourceTypeName) - ServiceProviderClusterResourceType = azcorearm.NewResourceType(ProviderNamespace, ClusterResourceTypeName+"/"+ServiceProviderClusterResourceTypeName) - NodePoolResourceType = azcorearm.NewResourceType(ProviderNamespace, ClusterResourceTypeName+"/"+NodePoolResourceTypeName) - ServiceProviderNodePoolResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, NodePoolResourceTypeName, ServiceProviderNodePoolResourceTypeName)) - ExternalAuthResourceType = azcorearm.NewResourceType(ProviderNamespace, ClusterResourceTypeName+"/"+ExternalAuthResourceTypeName) - PreflightResourceType = azcorearm.NewResourceType(ProviderNamespace, "deployments/preflight") - VersionResourceType = azcorearm.NewResourceType(ProviderNamespace, "locations/"+VersionResourceTypeName) - ClusterControllerResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, ControllerResourceTypeName)) - NodePoolControllerResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, NodePoolResourceTypeName, ControllerResourceTypeName)) - ExternalAuthControllerResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, ExternalAuthResourceTypeName, ControllerResourceTypeName)) - RequestAdminCredentialActionType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, RequestAdminCredentialActionTypeName)) - RevokeAdminCredentialsActionType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, RevokeAdminCredentialsActionTypeName)) - ManagementClusterContentResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, ManagementClusterContentResourceTypeName)) + OperationStatusResourceType = azcorearm.NewResourceType(ProviderNamespace, OperationStatusResourceTypeName) + ClusterResourceType = azcorearm.NewResourceType(ProviderNamespace, ClusterResourceTypeName) + ServiceProviderClusterResourceType = azcorearm.NewResourceType(ProviderNamespace, ClusterResourceTypeName+"/"+ServiceProviderClusterResourceTypeName) + NodePoolResourceType = azcorearm.NewResourceType(ProviderNamespace, ClusterResourceTypeName+"/"+NodePoolResourceTypeName) + ServiceProviderNodePoolResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, NodePoolResourceTypeName, ServiceProviderNodePoolResourceTypeName)) + ExternalAuthResourceType = azcorearm.NewResourceType(ProviderNamespace, ClusterResourceTypeName+"/"+ExternalAuthResourceTypeName) + PreflightResourceType = azcorearm.NewResourceType(ProviderNamespace, "deployments/preflight") + VersionResourceType = azcorearm.NewResourceType(ProviderNamespace, "locations/"+VersionResourceTypeName) + ClusterControllerResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, ControllerResourceTypeName)) + NodePoolControllerResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, NodePoolResourceTypeName, ControllerResourceTypeName)) + ExternalAuthControllerResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, ExternalAuthResourceTypeName, ControllerResourceTypeName)) + RequestAdminCredentialActionType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, RequestAdminCredentialActionTypeName)) + RevokeAdminCredentialsActionType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, RevokeAdminCredentialsActionTypeName)) + // ClusterScopedManagementClusterContentResourceType is managementClusterContents nested directly under a Cluster + ClusterScopedManagementClusterContentResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, ManagementClusterContentResourceTypeName)) + // NodePoolScopedManagementClusterContentResourceType is managementClusterContents nested under a Node Pool + NodePoolScopedManagementClusterContentResourceType = azcorearm.NewResourceType(ProviderNamespace, filepath.Join(ClusterResourceTypeName, NodePoolResourceTypeName, ManagementClusterContentResourceTypeName)) ) type VersionedResource interface { diff --git a/internal/api/types_management_cluster_content.go b/internal/api/types_management_cluster_content.go index b83784c3582..b1d028ac69c 100644 --- a/internal/api/types_management_cluster_content.go +++ b/internal/api/types_management_cluster_content.go @@ -23,7 +23,7 @@ import ( // ManagementClusterContent represents K8s resources in the Management Cluster // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type ManagementClusterContent struct { - // CosmosMetadata ResourceID is nested under the cluster so that association and cleanup work as expected + // CosmosMetadata is nested under the corresponding resource type so that association and cleanup work as expected CosmosMetadata `json:"cosmosMetadata"` // resourceID exists to match cosmosMetadata.resourceID until we're able to transition all types to use cosmosMetadata, diff --git a/internal/api/types_serviceprovider_nodepool.go b/internal/api/types_serviceprovider_nodepool.go index bd8756b7212..64cbc941e74 100644 --- a/internal/api/types_serviceprovider_nodepool.go +++ b/internal/api/types_serviceprovider_nodepool.go @@ -102,6 +102,12 @@ type ServiceProviderNodePoolStatus struct { // } // } NodePoolVersion ServiceProviderNodePoolStatusVersion `json:"nodePoolVersion,omitempty"` + + // MaestroReadonlyBundles contains a list of Maestro readonly bundles references. + // These bundles are used to retrieve particular K8s resources from the Management Cluster. + // The reference contains a mapping between the logical name we give to the Maestro bundle internally + // and the Maestro Bundle Name and ID at the Maestro API level. + MaestroReadonlyBundles MaestroBundleReferenceList `json:"maestroReadonlyBundles,omitempty"` } // ServiceProviderNodePoolStatusVersion contains the actual version information. @@ -111,6 +117,12 @@ type ServiceProviderNodePoolStatusVersion struct { ActiveVersions []HCPNodePoolActiveVersion `json:"activeVersions,omitempty"` } +const ( + // MaestroBundleInternalNameReadonlyHypershiftNodePool is the internal name of the Maestro Bundle that represents + // the NodePool's Hypershift's NodePool K8s resource. + MaestroBundleInternalNameReadonlyHypershiftNodePool MaestroBundleInternalName = "readonlyHypershiftNodePool" +) + // HCPNodePoolActiveVersion represents a single version active in the nodepool. type HCPNodePoolActiveVersion struct { // Version is the full version in x.y.z format (e.g., "4.19.2") diff --git a/internal/api/zz_generated.deepcopy.go b/internal/api/zz_generated.deepcopy.go index 833b45d7a60..bf8f9266d1d 100644 --- a/internal/api/zz_generated.deepcopy.go +++ b/internal/api/zz_generated.deepcopy.go @@ -1581,6 +1581,17 @@ func (in *ServiceProviderNodePoolStatus) DeepCopyInto(out *ServiceProviderNodePo } } in.NodePoolVersion.DeepCopyInto(&out.NodePoolVersion) + if in.MaestroReadonlyBundles != nil { + in, out := &in.MaestroReadonlyBundles, &out.MaestroReadonlyBundles + *out = make(MaestroBundleReferenceList, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(MaestroBundleReference) + **out = **in + } + } + } return } diff --git a/internal/database/crud_hcpcluster.go b/internal/database/crud_hcpcluster.go index 1047512b81f..03430cec658 100644 --- a/internal/database/crud_hcpcluster.go +++ b/internal/database/crud_hcpcluster.go @@ -108,6 +108,7 @@ func (d *operationCRUD) ListActiveOperations(options *DBClientListActiveOperatio type HCPClusterCRUD interface { ResourceCRUD[api.HCPOpenShiftCluster] ControllerContainer + ManagementClusterContentContainer ExternalAuth(hcpClusterID string) ExternalAuthsCRUD NodePools(hcpClusterID string) NodePoolsCRUD @@ -133,6 +134,7 @@ func NewHCPClusterCRUD(containerClient *azcosmos.ContainerClient, subscriptionID type NodePoolsCRUD interface { ResourceCRUD[api.HCPOpenShiftClusterNodePool] ControllerContainer + ManagementClusterContentContainer } type ExternalAuthsCRUD interface { @@ -193,6 +195,22 @@ func (h *hcpClusterCRUD) Controllers(hcpClusterName string) ResourceCRUD[api.Con return NewControllerCRUD(h.containerClient, parentResourceID, api.ClusterControllerResourceType) } +func (h *hcpClusterCRUD) ManagementClusterContents(hcpClusterName string) ManagementClusterContentCRUD { + parentResourceID := api.Must(azcorearm.ParseResourceID( + path.Join( + h.parentResourceID.String(), + "providers", + h.resourceType.Namespace, + h.resourceType.Type, + hcpClusterName))) + + return NewCosmosResourceCRUD[api.ManagementClusterContent, GenericDocument[api.ManagementClusterContent]]( + h.containerClient, + parentResourceID, + api.ClusterScopedManagementClusterContentResourceType, + ) +} + type externalAuthCRUD struct { *nestedCosmosResourceCRUD[api.HCPOpenShiftClusterExternalAuth, ExternalAuth] } @@ -223,6 +241,21 @@ func (h *nodePoolsCRUD) Controllers(nodePoolName string) ResourceCRUD[api.Contro return NewControllerCRUD(h.containerClient, parentResourceID, api.NodePoolControllerResourceType) } +func (h *nodePoolsCRUD) ManagementClusterContents(nodePoolName string) ManagementClusterContentCRUD { + parentResourceID := api.Must(azcorearm.ParseResourceID( + path.Join( + h.parentResourceID.String(), + h.resourceType.Types[len(h.resourceType.Types)-1], + nodePoolName, + ))) + + return NewCosmosResourceCRUD[api.ManagementClusterContent, GenericDocument[api.ManagementClusterContent]]( + h.containerClient, + parentResourceID, + api.NodePoolScopedManagementClusterContentResourceType, + ) +} + func NewControllerCRUD( containerClient *azcosmos.ContainerClient, parentResourceID *azcorearm.ResourceID, resourceType azcorearm.ResourceType) ResourceCRUD[api.Controller] { diff --git a/internal/database/crud_management_cluster_content.go b/internal/database/crud_management_cluster_content.go index 229f6893d09..ac8c3a80790 100644 --- a/internal/database/crud_management_cluster_content.go +++ b/internal/database/crud_management_cluster_content.go @@ -21,3 +21,7 @@ import ( type ManagementClusterContentCRUD interface { ResourceCRUD[api.ManagementClusterContent] } + +type ManagementClusterContentContainer interface { + ManagementClusterContents(resourceName string) ManagementClusterContentCRUD +} diff --git a/internal/database/database.go b/internal/database/database.go index e7b6c10943e..3773e051d68 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -119,8 +119,6 @@ type DBClient interface { GlobalListers() GlobalListers ServiceProviderNodePools(subscriptionID, resourceGroupName, clusterName, nodePoolName string) ServiceProviderNodePoolCRUD - - ManagementClusterContents(subscriptionID, resourceGroupName, clusterName string) ManagementClusterContentCRUD } var _ DBClient = &cosmosDBClient{} @@ -201,12 +199,6 @@ func (d *cosmosDBClient) ServiceProviderNodePools(subscriptionID, resourceGroupN d.resources, nodePoolResourceID, api.ServiceProviderNodePoolResourceType) } -func (d *cosmosDBClient) ManagementClusterContents(subscriptionID, resourceGroupName, clusterName string) ManagementClusterContentCRUD { - clusterResourceID := NewClusterResourceID(subscriptionID, resourceGroupName, clusterName) - return NewCosmosResourceCRUD[api.ManagementClusterContent, GenericDocument[api.ManagementClusterContent]]( - d.resources, clusterResourceID, api.ManagementClusterContentResourceType) -} - func (d *cosmosDBClient) UntypedCRUD(parentResourceID azcorearm.ResourceID) (UntypedResourceCRUD, error) { return NewUntypedCRUD(d.resources, parentResourceID), nil } diff --git a/internal/database/global_lister.go b/internal/database/global_lister.go index 936a6691843..f29601bff9c 100644 --- a/internal/database/global_lister.go +++ b/internal/database/global_lister.go @@ -43,6 +43,9 @@ type GlobalListers interface { ServiceProviderClusters() GlobalLister[api.ServiceProviderCluster] ServiceProviderNodePools() GlobalLister[api.ServiceProviderNodePool] Controllers() GlobalLister[api.Controller] + // ManagementClusterContents lists ManagementClusterContent documents across + // partitions for every Cosmos resource type where managementClusterContents + // is nested as a direct child resource. Those types are registered on the lister implementation. ManagementClusterContents() GlobalLister[api.ManagementClusterContent] Operations() GlobalLister[api.Operation] ActiveOperations() GlobalLister[api.Operation] @@ -118,9 +121,12 @@ func (g *cosmosGlobalListers) Controllers() GlobalLister[api.Controller] { } func (g *cosmosGlobalListers) ManagementClusterContents() GlobalLister[api.ManagementClusterContent] { - return &cosmosGlobalLister[api.ManagementClusterContent, GenericDocument[api.ManagementClusterContent]]{ + return &cosmosManagementClusterContentGlobalLister{ containerClient: g.resources, - resourceType: api.ManagementClusterContentResourceType, + managementClusterContentResourceTypes: []azcorearm.ResourceType{ + api.ClusterScopedManagementClusterContentResourceType, + api.NodePoolScopedManagementClusterContentResourceType, + }, } } @@ -247,3 +253,37 @@ func (l *cosmosBillingGlobalLister) List(ctx context.Context, options *DBClientL } return newQueryBillingIterator(pager), nil } + +// cosmosManagementClusterContentGlobalLister lists managementClusterContents whether nested under a +// cluster or under a node pool. +type cosmosManagementClusterContentGlobalLister struct { + containerClient *azcosmos.ContainerClient + managementClusterContentResourceTypes []azcorearm.ResourceType +} + +func (l *cosmosManagementClusterContentGlobalLister) List(ctx context.Context, options *DBClientListResourceDocsOptions) (DBClientIterator[api.ManagementClusterContent], error) { + var resourceTypeConditions []string + for _, resourceType := range l.managementClusterContentResourceTypes { + resourceTypeConditions = append(resourceTypeConditions, fmt.Sprintf("STRINGEQUALS(c.resourceType, %q, true)", resourceType.String())) + } + whereClause := strings.Join(resourceTypeConditions, " OR ") + query := fmt.Sprintf("SELECT * FROM c WHERE %s", whereClause) + + queryOptions := azcosmos.QueryOptions{ + PageSizeHint: -1, + } + if options != nil { + if options.PageSizeHint != nil { + queryOptions.PageSizeHint = max(*options.PageSizeHint, -1) + } + queryOptions.ContinuationToken = options.ContinuationToken + } + + partitionKey := azcosmos.NewPartitionKey() + pager := l.containerClient.NewQueryItemsPager(query, partitionKey, &queryOptions) + + if options != nil && ptr.Deref(options.PageSizeHint, -1) > 0 { + return newQueryResourcesSinglePageIterator[api.ManagementClusterContent, GenericDocument[api.ManagementClusterContent]](pager), nil + } + return newQueryResourcesIterator[api.ManagementClusterContent, GenericDocument[api.ManagementClusterContent]](pager), nil +} diff --git a/internal/database/iterators.go b/internal/database/iterators.go index f11f6851b22..143186a9e97 100644 --- a/internal/database/iterators.go +++ b/internal/database/iterators.go @@ -157,3 +157,43 @@ func (iter *queryBillingIterator) GetContinuationToken() string { func (iter *queryBillingIterator) GetError() error { return iter.err } + +// ListAll accumulates all pages from a DB GlobalLister into a slice. +// It calls listFn with a page size hint of pageSize and follows continuation tokens +// until all items have been collected. +// Any failure to list or iterate causes an early return because the caller needs the complete +// set of data to make correct decisions. +// listFn is expected to accept a opts *DBClientListResourceDocsOptions that supports +// paging via the PageSizeHint attribute. +// ListAll assists client code in breaking large list queries into multiple smaller chunks of pageSize or smaller. This +// helps reduce the load on the database at the cost of more round trips. +// pageSize determines the maximum number of items to be retrieved at once. A negative value will set a dynamic +// page size controlled by Cosmos. +func ListAll[InternalAPIType any]( + ctx context.Context, + pageSize int32, + listFn func(ctx context.Context, opts *DBClientListResourceDocsOptions) (DBClientIterator[InternalAPIType], error), +) ([]*InternalAPIType, error) { + opts := &DBClientListResourceDocsOptions{ + PageSizeHint: &pageSize, + } + all := make([]*InternalAPIType, 0) + for { + iterator, err := listFn(ctx, opts) + if err != nil { + return nil, fmt.Errorf("failed to list: %w", err) + } + for _, item := range iterator.Items(ctx) { + all = append(all, item) + } + if err := iterator.GetError(); err != nil { + return nil, fmt.Errorf("failed iterating: %w", err) + } + token := iterator.GetContinuationToken() + if token == "" { + break + } + opts.ContinuationToken = &token + } + return all, nil +} diff --git a/internal/database/iterators_test.go b/internal/database/iterators_test.go new file mode 100644 index 00000000000..c0973a51a39 --- /dev/null +++ b/internal/database/iterators_test.go @@ -0,0 +1,238 @@ +// Copyright 2025 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeIteratorEntry holds one item yielded by a fakeIterator. +type fakeIteratorEntry[T any] struct { + id string + item *T +} + +// fakeIterator is a test implementation of DBClientIterator[T] that yields a +// fixed set of items, optionally returning a continuation token and/or an error. +type fakeIterator[T any] struct { + entries []fakeIteratorEntry[T] + token string + err error +} + +func (f *fakeIterator[T]) Items(_ context.Context) DBClientIteratorItem[T] { + return func(yield func(string, *T) bool) { + for _, e := range f.entries { + if !yield(e.id, e.item) { + return + } + } + } +} + +func (f *fakeIterator[T]) GetContinuationToken() string { return f.token } +func (f *fakeIterator[T]) GetError() error { return f.err } + +func TestListAll(t *testing.T) { + // testListItem is a minimal type used as the ListAll type parameter in tests. + type testListItem struct{ value string } + + type listFn = func(context.Context, *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) + + type testCase struct { + name string + pageSize int32 + listFn listFn + wantItems []string // nil means expect empty/nil result + wantErr string // non-empty means expect an error containing this substring + verify func(*testing.T) // optional: extra assertions beyond result/error + } + + // --- Stateful setup: state variables defined here are captured by the closures below. --- + + // multi-page: verify the continuation token is forwarded on the second call + var mpCallCount int + var mpReceivedTokens []*string + mpListFn := func(_ context.Context, opts *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) { + mpCallCount++ + mpReceivedTokens = append(mpReceivedTokens, opts.ContinuationToken) + if mpCallCount == 1 { + return &fakeIterator[testListItem]{ + entries: []fakeIteratorEntry[testListItem]{ + {id: "p1-0", item: &testListItem{value: "p1-a"}}, + {id: "p1-1", item: &testListItem{value: "p1-b"}}, + }, + token: "tok-1", + }, nil + } + return &fakeIterator[testListItem]{ + entries: []fakeIteratorEntry[testListItem]{ + {id: "p2-0", item: &testListItem{value: "p2-a"}}, + }, + }, nil + } + + // three-page: verify listFn is called exactly three times + var threeCallCount int + threeListFn := func(_ context.Context, _ *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) { + threeCallCount++ + switch threeCallCount { + case 1: + return &fakeIterator[testListItem]{ + entries: []fakeIteratorEntry[testListItem]{{id: "0", item: &testListItem{value: "first"}}}, + token: "tok-a", + }, nil + case 2: + return &fakeIterator[testListItem]{ + entries: []fakeIteratorEntry[testListItem]{{id: "1", item: &testListItem{value: "second"}}}, + token: "tok-b", + }, nil + default: + return &fakeIterator[testListItem]{ + entries: []fakeIteratorEntry[testListItem]{{id: "2", item: &testListItem{value: "third"}}}, + }, nil + } + } + + // error on second page: verify listFn is called exactly twice + var secondPageCallCount int + secondPageListFn := func(_ context.Context, _ *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) { + secondPageCallCount++ + if secondPageCallCount == 1 { + return &fakeIterator[testListItem]{ + entries: []fakeIteratorEntry[testListItem]{{id: "id-0", item: &testListItem{value: "first"}}}, + token: "tok", + }, nil + } + return nil, fmt.Errorf("second page error") + } + + // page-size hint: capture the PageSizeHint seen by listFn + var capturedPageSize *int32 + pageSizeListFn := func(_ context.Context, opts *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) { + capturedPageSize = opts.PageSizeHint + return &fakeIterator[testListItem]{}, nil + } + + // --- Test table --- + + tests := []testCase{ + { + name: "empty result", + pageSize: 10, + listFn: func(_ context.Context, _ *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) { + return &fakeIterator[testListItem]{}, nil + }, + }, + { + name: "single page returns all items", + pageSize: 10, + wantItems: []string{"a", "b", "c"}, + listFn: func(_ context.Context, _ *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) { + return &fakeIterator[testListItem]{ + entries: []fakeIteratorEntry[testListItem]{ + {id: "id-0", item: &testListItem{value: "a"}}, + {id: "id-1", item: &testListItem{value: "b"}}, + {id: "id-2", item: &testListItem{value: "c"}}, + }, + }, nil + }, + }, + { + name: "multi-page follows continuation token", + pageSize: 5, + listFn: mpListFn, + wantItems: []string{"p1-a", "p1-b", "p2-a"}, + verify: func(t *testing.T) { + assert.Equal(t, 2, mpCallCount, "expected exactly two listFn calls") + assert.Nil(t, mpReceivedTokens[0], "first call should have no continuation token") + require.NotNil(t, mpReceivedTokens[1], "second call should carry the continuation token from page 1") + assert.Equal(t, "tok-1", *mpReceivedTokens[1]) + }, + }, + { + name: "three pages accumulates all items", + pageSize: 1, + listFn: threeListFn, + wantItems: []string{"first", "second", "third"}, + verify: func(t *testing.T) { + assert.Equal(t, 3, threeCallCount) + }, + }, + { + name: "pageSize is forwarded as PageSizeHint", + pageSize: 42, + listFn: pageSizeListFn, + verify: func(t *testing.T) { + require.NotNil(t, capturedPageSize) + assert.Equal(t, int32(42), *capturedPageSize) + }, + }, + { + name: "listFn error on first call", + pageSize: 10, + listFn: func(_ context.Context, _ *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) { + return nil, fmt.Errorf("connection refused") + }, + wantErr: "failed to list: connection refused", + }, + { + name: "listFn error on second page", + pageSize: 10, + listFn: secondPageListFn, + wantErr: "failed to list: second page error", + verify: func(t *testing.T) { + assert.Equal(t, 2, secondPageCallCount) + }, + }, + { + name: "iterator error", + pageSize: 10, + listFn: func(_ context.Context, _ *DBClientListResourceDocsOptions) (DBClientIterator[testListItem], error) { + return &fakeIterator[testListItem]{err: fmt.Errorf("unmarshal error")}, nil + }, + wantErr: "failed iterating: unmarshal error", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := ListAll(context.Background(), tt.pageSize, tt.listFn) + + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + assert.Nil(t, result) + } else { + require.NoError(t, err) + assert.NotNil(t, result) + var gotValues []string + for _, item := range result { + gotValues = append(gotValues, item.value) + } + assert.Equal(t, tt.wantItems, gotValues) + } + + if tt.verify != nil { + tt.verify(t) + } + }) + } +} diff --git a/internal/databasetesting/mock_crud.go b/internal/databasetesting/mock_crud.go index f7cec9d5354..d004412f029 100644 --- a/internal/databasetesting/mock_crud.go +++ b/internal/databasetesting/mock_crud.go @@ -511,6 +511,18 @@ func (m *mockHCPClusterCRUD) Controllers(hcpClusterName string) database.Resourc return newMockResourceCRUD[api.Controller, database.GenericDocument[api.Controller]](m.client, parentResourceID, api.ClusterControllerResourceType) } +func (m *mockHCPClusterCRUD) ManagementClusterContents(hcpClusterName string) database.ManagementClusterContentCRUD { + parentResourceID := api.Must(azcorearm.ParseResourceID( + path.Join( + m.parentResourceID.String(), + "providers", + m.resourceType.Namespace, + m.resourceType.Type, + hcpClusterName))) + + return newMockManagementClusterContentCRUD(m.client, parentResourceID, api.ClusterScopedManagementClusterContentResourceType) +} + var _ database.HCPClusterCRUD = &mockHCPClusterCRUD{} // mockNodePoolsCRUD implements database.NodePoolsCRUD. @@ -529,6 +541,17 @@ func (m *mockNodePoolsCRUD) Controllers(nodePoolName string) database.ResourceCR return newMockResourceCRUD[api.Controller, database.GenericDocument[api.Controller]](m.client, parentResourceID, api.NodePoolControllerResourceType) } +func (m *mockNodePoolsCRUD) ManagementClusterContents(nodePoolName string) database.ManagementClusterContentCRUD { + parentResourceID := api.Must(azcorearm.ParseResourceID( + path.Join( + m.parentResourceID.String(), + m.resourceType.Types[len(m.resourceType.Types)-1], + nodePoolName, + ))) + + return newMockManagementClusterContentCRUD(m.client, parentResourceID, api.NodePoolScopedManagementClusterContentResourceType) +} + var _ database.NodePoolsCRUD = &mockNodePoolsCRUD{} // mockExternalAuthCRUD implements database.ExternalAuthsCRUD. @@ -689,10 +712,10 @@ type mockManagementClusterContentCRUD struct { *mockResourceCRUD[api.ManagementClusterContent, database.GenericDocument[api.ManagementClusterContent]] } -func newMockManagementClusterContentCRUD(client *MockDBClient, parentResourceID *azcorearm.ResourceID) *mockManagementClusterContentCRUD { +func newMockManagementClusterContentCRUD(client *MockDBClient, parentResourceID *azcorearm.ResourceID, resourceType azcorearm.ResourceType) *mockManagementClusterContentCRUD { return &mockManagementClusterContentCRUD{ mockResourceCRUD: newMockResourceCRUD[api.ManagementClusterContent, database.GenericDocument[api.ManagementClusterContent]]( - client, parentResourceID, api.ManagementClusterContentResourceType), + client, parentResourceID, resourceType), } } diff --git a/internal/databasetesting/mock_dbclient.go b/internal/databasetesting/mock_dbclient.go index de4b902bf1e..4389837b1ea 100644 --- a/internal/databasetesting/mock_dbclient.go +++ b/internal/databasetesting/mock_dbclient.go @@ -148,12 +148,6 @@ func (m *MockDBClient) ServiceProviderNodePools(subscriptionID, resourceGroupNam return newMockServiceProviderNodePoolCRUD(m, nodePoolResourceID) } -// ManagementClusterContents returns a CRUD interface for management cluster content resources. -func (m *MockDBClient) ManagementClusterContents(subscriptionID, resourceGroupName, clusterName string) database.ManagementClusterContentCRUD { - clusterResourceID := database.NewClusterResourceID(subscriptionID, resourceGroupName, clusterName) - return newMockManagementClusterContentCRUD(m, clusterResourceID) -} - // LoadFromDirectory loads cosmos-record context data from a directory. // It reads all JSON files that match the pattern for "load" directories. func (m *MockDBClient) LoadFromDirectory(dirPath string) error { diff --git a/internal/databasetesting/mock_global_lister.go b/internal/databasetesting/mock_global_lister.go index 30a7d83df68..1ead4b74450 100644 --- a/internal/databasetesting/mock_global_lister.go +++ b/internal/databasetesting/mock_global_lister.go @@ -84,9 +84,12 @@ func (g *mockGlobalListers) Controllers() database.GlobalLister[api.Controller] } func (g *mockGlobalListers) ManagementClusterContents() database.GlobalLister[api.ManagementClusterContent] { - return &mockTypedGlobalLister[api.ManagementClusterContent, database.GenericDocument[api.ManagementClusterContent]]{ - client: g.client, - resourceType: api.ManagementClusterContentResourceType, + return &mockManagementClusterContentGlobalLister{ + client: g.client, + resourceTypes: []azcorearm.ResourceType{ + api.ClusterScopedManagementClusterContentResourceType, + api.NodePoolScopedManagementClusterContentResourceType, + }, } } @@ -283,3 +286,49 @@ func (l *mockBillingGlobalLister) List(ctx context.Context, options *database.DB return newMockIterator(ids, items), nil } + +// mockManagementClusterContentGlobalLister lists management cluster content for cluster-scoped and node-pool-scoped documents. +type mockManagementClusterContentGlobalLister struct { + client *MockDBClient + resourceTypes []azcorearm.ResourceType +} + +func (l *mockManagementClusterContentGlobalLister) List(ctx context.Context, options *database.DBClientListResourceDocsOptions) (database.DBClientIterator[api.ManagementClusterContent], error) { + allDocs := l.client.GetAllDocuments() + + var ids []string + var items []*api.ManagementClusterContent + + for _, data := range allDocs { + var typedDoc database.TypedDocument + if err := json.Unmarshal(data, &typedDoc); err != nil { + continue + } + + resourceTypeMatches := false + for _, resourceType := range l.resourceTypes { + if strings.EqualFold(typedDoc.ResourceType, resourceType.String()) { + resourceTypeMatches = true + break + } + } + if !resourceTypeMatches { + continue + } + + var cosmosObj database.GenericDocument[api.ManagementClusterContent] + if err := json.Unmarshal(data, &cosmosObj); err != nil { + continue + } + + internalObj, err := database.CosmosGenericToInternal(&cosmosObj) + if err != nil { + continue + } + + ids = append(ids, typedDoc.ID) + items = append(items, internalObj) + } + + return newMockIterator(ids, items), nil +} diff --git a/internal/databasetesting/mock_init.go b/internal/databasetesting/mock_init.go index 88a5a277e55..670fe0ca485 100644 --- a/internal/databasetesting/mock_init.go +++ b/internal/databasetesting/mock_init.go @@ -32,6 +32,7 @@ import ( // - *api.ServiceProviderCluster // - *arm.Subscription // - *api.Controller +// - *api.ManagementClusterContent // // Returns an error if any resource cannot be created or if an unsupported type is encountered. func NewMockDBClientWithResources(ctx context.Context, resources []any) (*MockDBClient, error) { @@ -63,6 +64,8 @@ func (m *MockDBClient) addResource(ctx context.Context, resource any) error { return m.addSubscription(ctx, r) case *api.Controller: return m.addController(ctx, r) + case *api.ManagementClusterContent: + return m.addManagementClusterContent(ctx, r) default: return fmt.Errorf("unsupported resource type: %T", resource) } @@ -172,3 +175,32 @@ func (m *MockDBClient) addController(ctx context.Context, controller *api.Contro } return fmt.Errorf("unsupported parent resource type: %s", parentType) } + +func (m *MockDBClient) addManagementClusterContent(ctx context.Context, mcc *api.ManagementClusterContent) error { + resourceID := mcc.GetResourceID() + if resourceID == nil { + return fmt.Errorf("management cluster content is missing resource ID") + } + if resourceID.Parent == nil { + return fmt.Errorf("management cluster content is missing parent ID") + } + parentType := resourceID.Parent.ResourceType + switch { + case armhelpers.ResourceTypeEqual(parentType, api.ClusterResourceType): + clusterName := resourceID.Parent.Name + mccCRUD := m.HCPClusters(resourceID.SubscriptionID, resourceID.ResourceGroupName).ManagementClusterContents(clusterName) + _, err := mccCRUD.Create(ctx, mcc, nil) + return err + case armhelpers.ResourceTypeEqual(parentType, api.NodePoolResourceType): + if resourceID.Parent.Parent == nil { + return fmt.Errorf("node pool management cluster content is missing grandparent cluster ID") + } + clusterName := resourceID.Parent.Parent.Name + nodePoolName := resourceID.Parent.Name + mccCRUD := m.HCPClusters(resourceID.SubscriptionID, resourceID.ResourceGroupName).NodePools(clusterName).ManagementClusterContents(nodePoolName) + _, err := mccCRUD.Create(ctx, mcc, nil) + return err + default: + return fmt.Errorf("unsupported parent resource type for management cluster content: %s", parentType) + } +} diff --git a/internal/ocm/internalid.go b/internal/ocm/internalid.go index dc73c87752b..4d4145057a6 100644 --- a/internal/ocm/internalid.go +++ b/internal/ocm/internalid.go @@ -42,22 +42,38 @@ var ( aroHcpV1Alpha1ProvisionShardPattern = path.Join(aroHcpV1Alpha1ClusterPattern, provisionShardKey, "*") ) -func GenerateClusterHREF(clusterName string) string { +func GenerateOCMCommercialClusterHREF(clusterName string) string { return path.Join(v1Pattern, clusterKey, clusterName) } -func GenerateNodePoolHREF(clusterPath string, nodePoolName string) string { +func GenerateAROHCPClusterHREF(clusterName string) string { + return path.Join(aroHcpV1Alpha1Pattern, clusterKey, clusterName) +} + +func GenerateOCMCommercialNodePoolHREF(clusterPath string, nodePoolName string) string { return path.Join(clusterPath, nodePoolKey, nodePoolName) } -func GenerateExternalAuthHREF(clusterPath string, externalAuthName string) string { +func GenerateAROHCPNodePoolHREF(clusterPath string, nodePoolName string) string { + return path.Join(aroHcpV1Alpha1Pattern, clusterKey, clusterPath, nodePoolKey, nodePoolName) +} + +func GenerateOCMCommercialExternalAuthHREF(clusterPath string, externalAuthName string) string { return path.Join(clusterPath, externalAuthKey, externalAuthName) } -func GenerateBreakGlassCredentialHREF(clusterPath string, credentialName string) string { +func GenerateAROHCPExternalAuthHREF(clusterPath string, externalAuthName string) string { + return path.Join(aroHcpV1Alpha1Pattern, clusterKey, clusterPath, externalAuthKey, externalAuthName) +} + +func GenerateOCMCommercialBreakGlassCredentialHREF(clusterPath string, credentialName string) string { return path.Join(clusterPath, "break_glass_credentials", credentialName) } +func GenerateAROHCPBreakGlassCredentialHREF(clusterPath string, credentialName string) string { + return path.Join(aroHcpV1Alpha1Pattern, clusterKey, clusterPath, "break_glass_credentials", credentialName) +} + type InternalID = api.InternalID // getClusterClient returns a v1 ClusterClient from the InternalID.