feat: move and enable asynchronous CS update interactions

frontend/pkg/frontend/cluster.go

@@ -610,8 +610,6 @@ func (f *Frontend) patchHCPCluster(writer http.ResponseWriter, request *http.Req
 }
 
 func (f *Frontend) updateHCPClusterInCosmos(ctx context.Context, writer http.ResponseWriter, request *http.Request, httpStatusCode int, newInternalCluster, oldInternalCluster *api.HCPOpenShiftCluster) error {
-	logger := utils.LoggerFromContext(ctx)
-
 	subscription, err := f.resourcesDBClient.Subscriptions().Get(ctx, oldInternalCluster.ID.SubscriptionID)
 	if err != nil {
 		return utils.TrackError(err)
@@ -656,37 +654,11 @@ func (f *Frontend) updateHCPClusterInCosmos(ctx context.Context, writer http.Res
 	}
 	completeClusterIdentity(newInternalCluster, existingUserAssignedIdentities)
 
-	var tenantID string
-	if subscription.Properties != nil && subscription.Properties.TenantId != nil {
-		tenantID = *subscription.Properties.TenantId
-	}
-
-	if oldInternalCluster.ServiceProviderProperties.ClusterServiceID != nil {
-		oldClusterServiceCluster, err := f.clusterServiceClient.GetCluster(ctx, *oldInternalCluster.ServiceProviderProperties.ClusterServiceID)
-		if err != nil {
-			return utils.TrackError(err)
-		}
-		newClusterServiceClusterBuilder, newClusterServiceAutoscalerBuilder, err := ocm.BuildCSCluster(oldInternalCluster.ID, tenantID, newInternalCluster, nil, oldClusterServiceCluster, admissionContext.ServiceProviderCluster)
-		if err != nil {
-			return utils.TrackError(err)
-		}
-
-		logger.Info(fmt.Sprintf("updating resource %s", oldInternalCluster.ID))
-		_, err = f.clusterServiceClient.UpdateClusterAutoscaler(ctx, *oldInternalCluster.ServiceProviderProperties.ClusterServiceID, newClusterServiceAutoscalerBuilder)
-		if err != nil {
-			return utils.TrackError(err)
-		}
-		_, err = f.clusterServiceClient.UpdateCluster(ctx, *oldInternalCluster.ServiceProviderProperties.ClusterServiceID, newClusterServiceClusterBuilder)
-		if err != nil {
-			return utils.TrackError(err)
-		}
-	}
-
 	transaction := f.resourcesDBClient.NewTransaction(oldInternalCluster.ID.SubscriptionID)
 	clusterUpdateOperation := database.NewOperation(
 		database.OperationRequestUpdate,
 		oldInternalCluster.ID,
-		ptr.Deref(oldInternalCluster.ServiceProviderProperties.ClusterServiceID, api.InternalID{}),
+		api.InternalID{},
 		f.azureLocation,
 		request.Header.Get(arm.HeaderNameHomeTenantID),
 		request.Header.Get(arm.HeaderNameClientObjectID),

frontend/pkg/frontend/external_auth.go

@@ -470,21 +470,7 @@ func (f *Frontend) updateExternalAuthInCosmos(ctx context.Context, writer http.R
 		return utils.TrackError(err)
 	}
 
-	// Temporary check until creation and update interaction with CS is moved to the backend: If an update arrives after the externalauth
-	// has been created in Cosmos but before it exists in CS, or before its ClusterServiceID has been persisted in Cosmos, return an error.
-	if oldInternalExternalAuth.ServiceProviderProperties.ClusterServiceID == nil || len(oldInternalExternalAuth.ServiceProviderProperties.ClusterServiceID.String()) == 0 {
-		return utils.TrackError(fmt.Errorf("serviceProviderProperties.clusterServiceID is required to update an external auth"))
-	}
-
-	csExternalAuthBuilder, err := ocm.BuildCSExternalAuth(ctx, newInternalExternalAuth, true)
-	if err != nil {
-		return utils.TrackError(err)
-	}
 	logger.Info(fmt.Sprintf("updating resource %s", oldInternalExternalAuth.ID))
-	_, err = f.clusterServiceClient.UpdateExternalAuth(ctx, *oldInternalExternalAuth.ServiceProviderProperties.ClusterServiceID, csExternalAuthBuilder)
-	if err != nil {
-		return utils.TrackError(err)
-	}
 
 	transaction := f.resourcesDBClient.NewTransaction(oldInternalExternalAuth.ID.SubscriptionID)
 

frontend/pkg/frontend/node_pool.go

@@ -35,7 +35,6 @@ import (
 	"github.com/Azure/ARO-HCP/internal/api/arm"
 	"github.com/Azure/ARO-HCP/internal/conversion"
 	"github.com/Azure/ARO-HCP/internal/database"
-	"github.com/Azure/ARO-HCP/internal/ocm"
 	"github.com/Azure/ARO-HCP/internal/utils"
 	"github.com/Azure/ARO-HCP/internal/validation"
 )
@@ -577,37 +576,17 @@ func (f *Frontend) updateNodePoolInCosmos(ctx context.Context, writer http.Respo
 		return utils.TrackError(err)
 	}
 
-	// Temporary check until creation and update interaction with CS is moved to the backend: If an update arrives after the node pool
-	// has been created in Cosmos but before it exists in CS, or before its ClusterServiceID has been persisted in Cosmos, return an error.
-	if oldInternalNodePool.ServiceProviderProperties.ClusterServiceID == nil || len(oldInternalNodePool.ServiceProviderProperties.ClusterServiceID.String()) == 0 {
-		return utils.TrackError(fmt.Errorf("serviceProviderProperties.clusterServiceID is required to update a node pool"))
-	}
-
-	csNodePoolBuilder, err := ocm.BuildCSNodePool(ctx, newInternalNodePool, true)
-	if err != nil {
-		return utils.TrackError(err)
-	}
 	logger.Info(fmt.Sprintf("updating resource %s", oldInternalNodePool.ID))
-	_, err = f.clusterServiceClient.UpdateNodePool(ctx, *oldInternalNodePool.ServiceProviderProperties.ClusterServiceID, csNodePoolBuilder)
-	if err != nil {
-		return utils.TrackError(err)
-	}
 
 	// The cosmos representation the new desired version
 	// The controllers will take care of handle the upgrade
 
 	transaction := f.resourcesDBClient.NewTransaction(oldInternalNodePool.ID.SubscriptionID)
 
-	// Always create an operation, even for version-only changes. This provides consistent
-	// ARM API behavior and avoids the complexity of detecting what changed.
-	// For version-only updates, CS receives a no-op PATCH
-	// (version is excluded in BuildCSNodePool for updates) and stays ready, so the
-	// operation resolves on the next poll cycle (~10s). The actual version upgrade is
-	// handled transparently by backend controllers using the desired version stored above.
 	nodePoolUpdateOperation := database.NewOperation(
 		database.OperationRequestUpdate,
 		newInternalNodePool.ID,
-		ptr.Deref(newInternalNodePool.ServiceProviderProperties.ClusterServiceID, api.InternalID{}),
+		api.InternalID{},
 		f.azureLocation,
 		request.Header.Get(arm.HeaderNameHomeTenantID),
 		request.Header.Get(arm.HeaderNameClientObjectID),

test-integration/frontend/artifacts/FrontendCRUD/Cluster/experimental-features/00-httpCreate-subscription-with-afec/subscription.json

@@ -5,7 +5,8 @@
 				"name": "Microsoft.RedHatOpenShift/ExperimentalReleaseFeatures",
 				"state": "Registered"
 			}
-		]
+		],
+		"tenantId": "00000000-0000-0000-0000-000000000000"
 	},
 	"registrationDate": "2025-12-19T19:53:15+00:00",
 	"resourceId": "/subscriptions/6b690bec-0c16-4ecb-8f67-781caf40bba7",

test-integration/frontend/artifacts/FrontendCRUD/Cluster/experimental-features/01-httpCreate-cluster/feature-flag-test.json

@@ -43,7 +43,7 @@
 		},
 		"version": {
 			"channelGroup": "stable",
-			"id": "4.19"
+			"id": "4.20"
 		}
 	},
 	"tags": {

test-integration/frontend/artifacts/FrontendCRUD/Cluster/experimental-features/04-httpReplace-subscription-keep-afec/subscription.json

@@ -5,7 +5,8 @@
 				"name": "Microsoft.RedHatOpenShift/ExperimentalReleaseFeatures",
 				"state": "Registered"
 			}
-		]
+		],
+		"tenantId": "00000000-0000-0000-0000-000000000000"
 	},
 	"registrationDate": "2025-12-19T19:53:15+00:00",
 	"resourceId": "/subscriptions/6b690bec-0c16-4ecb-8f67-781caf40bba7",

test-integration/frontend/artifacts/FrontendCRUD/Cluster/experimental-features/05-httpReplace-cluster/feature-flag-test.json

@@ -43,7 +43,7 @@
 		},
 		"version": {
 			"channelGroup": "stable",
-			"id": "4.19"
+			"id": "4.20"
 		}
 	},
 	"tags": {},

test-integration/frontend/artifacts/FrontendCRUD/Cluster/experimental-features/06-syncClusterClusterServiceUpdateDispatch-propagate-replace-to-cs/00-key.json

@@ -0,0 +1,4 @@
+{
+	"id": "d189de99-ebfa-5c9f-846c-6656f3718661",
+	"resourceID": "/subscriptions/6b690bec-0c16-4ecb-8f67-781caf40bba7/resourceGroups/resourceGroupName/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/feature-flag-test"
+}

test-integration/frontend/artifacts/FrontendCRUD/NodePool/experimental-features/00-httpCreate-subscription-with-afec/subscription.json

@@ -5,7 +5,8 @@
 				"name": "Microsoft.RedHatOpenShift/ExperimentalReleaseFeatures",
 				"state": "Registered"
 			}
-		]
+		],
+		"tenantId": "00000000-0000-0000-0000-000000000000"
 	},
 	"registrationDate": "2025-12-19T19:53:15+00:00",
 	"resourceId": "/subscriptions/a1b2c3d4-0000-0000-0000-afec00000000",

test-integration/utils/databasemutationhelpers/resource_crud_test_util.go

@@ -267,6 +267,8 @@ func NewUntypedStep(indexString, stepType, stepName string, testDir fs.FS, path
 		return newSetClusterServiceIDStep(stepID, stepDir)
 	case "clusterServiceCompare":
 		return newClusterServiceCompareStep(stepID, stepDir)
+	case "syncClusterClusterServiceUpdateDispatch":
+		return newSyncClusterClusterServiceUpdateDispatchStep(stepID, stepDir)
 	case "migrateCosmos":
 		return newMigrateCosmosStep(stepID, stepDir)
 	case "kubernetesLoad":
@@ -364,6 +366,9 @@ func NewStep[InternalAPIType any, InternalAPITypePointer arm.CosmosMetadataAcces
 	case "clusterServiceCompare":
 		return newClusterServiceCompareStep(stepID, stepDir)
 
+	case "syncClusterClusterServiceUpdateDispatch":
+		return newSyncClusterClusterServiceUpdateDispatchStep(stepID, stepDir)
+
 	case "migrateCosmos":
 		return newMigrateCosmosStep(stepID, stepDir)
 

test-integration/utils/databasemutationhelpers/step_sync_cluster_cluster_service_update_dispatch.go

@@ -0,0 +1,93 @@
+// Copyright 2026 Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package databasemutationhelpers
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
+
+	"github.com/Azure/ARO-HCP/backend/pkg/controllers/clusterupdate"
+	"github.com/Azure/ARO-HCP/backend/pkg/controllers/controllerutils"
+	"github.com/Azure/ARO-HCP/backend/pkg/listertesting"
+	"github.com/Azure/ARO-HCP/internal/api"
+	"github.com/Azure/ARO-HCP/internal/utils"
+	"github.com/Azure/ARO-HCP/test-integration/utils/integrationutils"
+)
+
+type syncClusterClusterServiceUpdateDispatchStep struct {
+	stepID StepID
+	key    controllerutils.HCPClusterKey
+}
+
+func newSyncClusterClusterServiceUpdateDispatchStep(stepID StepID, stepDir fs.FS) (*syncClusterClusterServiceUpdateDispatchStep, error) {
+	keyBytes, err := fs.ReadFile(stepDir, "00-key.json")
+	if err != nil {
+		return nil, fmt.Errorf("failed to read 00-key.json: %w", err)
+	}
+	var resourceKey ResourceKey
+	if err := json.Unmarshal(keyBytes, &resourceKey); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal 00-key.json: %w", err)
+	}
+
+	resourceID, err := azcorearm.ParseResourceID(resourceKey.ResourceID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse resource ID %q: %w", resourceKey.ResourceID, err)
+	}
+
+	return &syncClusterClusterServiceUpdateDispatchStep{
+		stepID: stepID,
+		key: controllerutils.HCPClusterKey{
+			SubscriptionID:    resourceID.SubscriptionID,
+			ResourceGroupName: resourceID.ResourceGroupName,
+			HCPClusterName:    resourceID.Name,
+		},
+	}, nil
+}
+
+var _ IntegrationTestStep = &syncClusterClusterServiceUpdateDispatchStep{}
+
+func (s *syncClusterClusterServiceUpdateDispatchStep) StepID() StepID {
+	return s.stepID
+}
+
+func (s *syncClusterClusterServiceUpdateDispatchStep) RunTest(ctx context.Context, t *testing.T, stepInput StepInput) {
+	require.NotNil(t, stepInput.ClusterServiceMockInfo, "syncClusterClusterServiceUpdateDispatch requires a ClusterServiceMock")
+
+	ctx = utils.ContextWithLogger(ctx, integrationutils.DefaultLogger(t))
+
+	cluster, err := stepInput.ResourcesDBClient.HCPClusters(s.key.SubscriptionID, s.key.ResourceGroupName).Get(ctx, s.key.HCPClusterName)
+	require.NoError(t, err)
+
+	clusterLister := &listertesting.SliceClusterLister{Clusters: []*api.HCPOpenShiftCluster{cluster}}
+	activeOperationLister := &listertesting.SliceActiveOperationLister{}
+	subscriptionLister := &listertesting.DBSubscriptionLister{ResourcesDBClient: stepInput.ResourcesDBClient}
+	syncer := clusterupdate.NewClusterClusterServiceUpdateDispatchSyncer(
+		stepInput.ResourcesDBClient,
+		stepInput.ClusterServiceMockInfo.MockClusterServiceClient,
+		activeOperationLister,
+		clusterLister,
+		subscriptionLister,
+	)
+
+	err = syncer.SyncOnce(ctx, s.key)
+	require.NoError(t, err)
+}