diff --git a/docs/cosmos-resource-deletion.md b/docs/cosmos-resource-deletion.md new file mode 100644 index 00000000000..e1e6a6f554b --- /dev/null +++ b/docs/cosmos-resource-deletion.md @@ -0,0 +1,132 @@ +# Cosmos Resource Deletion via Soft-Delete + +## Problem + +Cosmos DB's "latest version" change feed mode does not surface hard deletions. +When a resource is deleted via `DeleteItem`, the change feed never sees that event, which means +controllers watching through `ChangeFeedListWatcher` cannot react to deletions until the next +full relist (up to 30 minutes later). This breaks correctness guarantees for controllers that +need to act promptly on resource removal. + +## Solution: DeletionTimestamp + TTL + +Instead of hard-deleting documents with `DeleteItem`, we implement a two-phase soft-delete: + +1. **Mark phase** (immediate): Set `DeletionTimestamp` on the document and set the Cosmos TTL + to 30 seconds. This is a Replace operation, which the change feed **does** surface. +2. **Cleanup phase** (automatic): After the TTL expires, Cosmos DB automatically removes the + document from storage. + +### Document Schema Change + +`TypedDocument` gains a new field: + +```go +type TypedDocument struct { + BaseDocument + PartitionKey string `json:"partitionKey"` + ResourceID *azcorearm.ResourceID `json:"resourceID"` + ResourceType string `json:"resourceType"` + DeletionTimestamp *metav1.Time `json:"deletionTimestamp,omitempty"` + Properties json.RawMessage `json:"properties"` +} +``` + +When `DeletionTimestamp` is nil (or absent from the JSON), the document is live. When set, the +document is logically deleted and awaiting Cosmos TTL cleanup. + +### CRUD Behavior Changes + +#### Delete + +When `Delete` is called, the CRUD helper: + +1. Reads the document from Cosmos (`ReadItem`). +2. Sets `deletionTimestamp` to the current time. +3. Sets `ttl` to 30 (seconds after the write timestamp `_ts`). +4. Increments `properties.cosmosMetadata.instanceVersion` so the change feed recognizes the update. +5. Replaces the document with an etag precondition (optimistic concurrency). + +If the document is already not found, the delete is a no-op (idempotent). + +#### Get / GetByID + +After reading a document, the CRUD layer checks for `DeletionTimestamp`. If it is non-nil, +the method returns a 404 Not Found error as if the document does not exist. This ensures +callers never see a logically-deleted document through the typed CRUD layer. + +#### List (including global listers) + +All list queries gain an additional WHERE clause: + +```sql +AND (NOT IS_DEFINED(c.deletionTimestamp)) +``` + +Since `deletionTimestamp` uses `omitempty` in its JSON tag, live documents have no +`deletionTimestamp` field in their JSON at all. The `IS_DEFINED` check is both correct +and index-friendly. This filter applies to: + +- Per-partition scoped lists (`nestedCosmosResourceCRUD.List`) +- Cross-partition global listers (`cosmosGlobalLister.List`) +- Active operations lister (`cosmosActiveOperationsGlobalLister.List`) + +### Change Feed Behavior + +The `processDocument` method in `ChangeFeedWatcher` checks `DeletionTimestamp` on every +document surfaced by the change feed: + +- **DeletionTimestamp is non-nil AND the resource was previously seen** (in the list snapshot + or a prior change feed event): Deliver a `watch.Deleted` event with the current document + content. +- **DeletionTimestamp is non-nil AND the resource was NOT previously seen**: Skip silently. + This handles the case where a resource was created and deleted between relists. + +This means controllers and informers receive timely Delete notifications without waiting for +a relist cycle. + +### Mock Implementation + +The in-memory mock (`MockResourcesDBClient`) mirrors production behavior: + +- `Delete` performs a read-modify-write that sets `DeletionTimestamp` and `TTL`, then calls + `StoreDocument` (which records to the mock change feed). +- `GetDocument` returns the raw document regardless of deletion status (the CRUD layer does + the filtering). +- `ListDocuments` filters out documents with `DeletionTimestamp` set. +- The mock change feed sees the soft-delete as a normal mutation (since it goes through + `StoreDocument`), so changefeed-based tests work identically. + +### Timeline + +``` +T+0s Delete() called + ├─ ReadItem (get current doc + etag) + ├─ Set deletionTimestamp = now, ttl = 30 + ├─ Increment instanceVersion + └─ ReplaceItem (conditional on etag) + +T+0-1s Change feed surfaces the updated document + └─ ChangeFeedWatcher delivers watch.Deleted event + +T+30s Cosmos TTL expires + └─ Document physically removed from storage +``` + +## Integration Tests + +The following test scenarios verify the end-to-end behavior: + +1. **Changefeed delivers Delete event**: Create a cluster, start list/watch, delete the + cluster, verify a `watch.Deleted` event arrives with the document content. + +2. **Changefeed + lister reports not-found**: Create a cluster, start an informer, delete + the cluster, verify the informer delivers a Delete event and the lister returns not-found. + +3. **Changefeed + relist has no extra records**: Create a cluster, delete it, then perform a + fresh list/watch cycle. Verify no events are delivered for the deleted cluster (it should + not appear in either the list result or as a change feed event). + +4. **Cosmos TTL cleanup**: Create a cluster, delete it (sets 30s TTL), wait for TTL to expire, + then verify the document is physically absent from Cosmos. This test runs only against the + real Cosmos emulator. diff --git a/internal/database/crud_hcpcluster.go b/internal/database/crud_hcpcluster.go index dea4a525710..e35461326f8 100644 --- a/internal/database/crud_hcpcluster.go +++ b/internal/database/crud_hcpcluster.go @@ -69,6 +69,7 @@ func (d *operationCRUD) ListActiveOperations(options *ResourcesDBClientListActiv query := fmt.Sprintf( "SELECT * FROM c WHERE STRINGEQUALS(c.resourceType, %q, true) "+ "AND LENGTH(c.resourceID) > 0 "+ + "AND (NOT IS_DEFINED(c.deletionTimestamp)) "+ "AND NOT ARRAYCONTAINS([%q, %q, %q], c.properties.status)", api.OperationStatusResourceType.String(), arm.ProvisioningStateSucceeded, diff --git a/internal/database/crud_helpers.go b/internal/database/crud_helpers.go index 659b0d7cda2..108e1d93931 100644 --- a/internal/database/crud_helpers.go +++ b/internal/database/crud_helpers.go @@ -20,7 +20,10 @@ import ( "errors" "fmt" "strings" + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/utils/ptr" azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" @@ -30,6 +33,8 @@ import ( "github.com/Azure/ARO-HCP/internal/utils" ) +const SoftDeleteTTLSeconds = 30 + func OldResourceIDToCosmosID(resourceID *azcorearm.ResourceID) (string, error) { if resourceID == nil { return "", errors.New("resource ID is nil") @@ -76,9 +81,21 @@ func getByItemID[InternalAPIType, CosmosAPIType any](ctx context.Context, contai return nil, utils.TrackError(err) } + if IsSoftDeleted(responseItem.Value) { + return nil, NewNotFoundError() + } + return responseItemToInternalObj[InternalAPIType, CosmosAPIType](ctx, cosmosID, responseItem) } +func IsSoftDeleted(data []byte) bool { + var doc TypedDocument + if err := json.Unmarshal(data, &doc); err != nil { + return false + } + return doc.DeletionTimestamp != nil +} + func get[InternalAPIType, CosmosAPIType any](ctx context.Context, containerClient *azcosmos.ContainerClient, partitionKeyString string, completeResourceID *azcorearm.ResourceID) (*InternalAPIType, error) { // try to see if the cosmosID we've passed is also the exact resource ID. If so, then return the value we got. newExactCosmosID, err := arm.ResourceIDToCosmosID(completeResourceID) @@ -101,9 +118,9 @@ func list[InternalAPIType, CosmosAPIType any](ctx context.Context, containerClie PageSizeHint: -1, } if prefix == nil { - query = "SELECT * FROM c WHERE LENGTH(c.resourceID) > 0" + query = "SELECT * FROM c WHERE LENGTH(c.resourceID) > 0 AND (NOT IS_DEFINED(c.deletionTimestamp))" } else { - query = "SELECT * FROM c WHERE STARTSWITH(c.resourceID, @prefix, true)" + query = "SELECT * FROM c WHERE STARTSWITH(c.resourceID, @prefix, true) AND (NOT IS_DEFINED(c.deletionTimestamp))" queryOptions = azcosmos.QueryOptions{ PageSizeHint: -1, QueryParameters: []azcosmos.QueryParameter{ @@ -347,20 +364,75 @@ func deleteResource(ctx context.Context, containerClient *azcosmos.ContainerClie return utils.TrackError(err) } - _, err = containerClient.DeleteItem(ctx, azcosmos.NewPartitionKeyString(partitionKeyString), cosmosID, nil) + return softDeleteByCosmosID(ctx, containerClient, partitionKeyString, cosmosID) +} + +func deleteByCosmosID(ctx context.Context, containerClient *azcosmos.ContainerClient, partitionKeyString, cosmosID string) error { + return softDeleteByCosmosID(ctx, containerClient, partitionKeyString, cosmosID) +} + +func softDeleteByCosmosID(ctx context.Context, containerClient *azcosmos.ContainerClient, partitionKeyString, cosmosID string) error { + pk := azcosmos.NewPartitionKeyString(partitionKeyString) + + responseItem, err := containerClient.ReadItem(ctx, pk, cosmosID, nil) if IsNotFoundError(err) { return nil } if err != nil { return utils.TrackError(err) } - return nil -} -func deleteByCosmosID(ctx context.Context, containerClient *azcosmos.ContainerClient, partitionKeyString, cosmosID string) error { - _, err := containerClient.DeleteItem(ctx, azcosmos.NewPartitionKeyString(partitionKeyString), cosmosID, nil) + var doc GenericDocument[map[string]interface{}] + if err := json.Unmarshal(responseItem.Value, &doc); err != nil { + return fmt.Errorf("failed to unmarshal document for soft delete: %w", err) + } + + if doc.DeletionTimestamp != nil { + return nil + } + + SetSoftDeleteFields(&doc, time.Now()) + + modified, err := json.Marshal(doc) + if err != nil { + return fmt.Errorf("failed to marshal soft-deleted document: %w", err) + } + + opts := &azcosmos.ItemOptions{ + IfMatchEtag: &responseItem.ETag, + } + _, err = containerClient.ReplaceItem(ctx, pk, cosmosID, modified, opts) if err != nil { return utils.TrackError(err) } return nil } + +// SetSoftDeleteFields marks a document as soft-deleted and increments its instanceVersion. +// We use map[string]interface{} instead of unstructured.Unstructured because +// Unstructured.UnmarshalJSON requires kind/apiVersion fields and fails with +// "Object 'Kind' is missing in ''" on Cosmos documents. +func SetSoftDeleteFields(doc *GenericDocument[map[string]interface{}], now time.Time) { + if doc.DeletionTimestamp != nil { + return + } + + ts := metav1.NewTime(now) + doc.DeletionTimestamp = &ts + doc.TimeToLive = SoftDeleteTTLSeconds + + // allow deleting legacy content + if doc.Content == nil { + return + } + + fv, found, err := unstructured.NestedFloat64(doc.Content, "cosmosMetadata", "instanceVersion") + current := int64(1) + if err == nil && found { + current = int64(fv) + if current < 1 { + current = 1 + } + } + _ = unstructured.SetNestedField(doc.Content, current+1, "cosmosMetadata", "instanceVersion") +} diff --git a/internal/database/global_lister.go b/internal/database/global_lister.go index 99b37bc16af..82e35d3fa32 100644 --- a/internal/database/global_lister.go +++ b/internal/database/global_lister.go @@ -150,6 +150,7 @@ func (l *cosmosActiveOperationsGlobalLister) List(ctx context.Context, options * query := fmt.Sprintf( "SELECT * FROM c WHERE STRINGEQUALS(c.resourceType, %q, true) "+ "AND LENGTH(c.resourceID) > 0 "+ + "AND (NOT IS_DEFINED(c.deletionTimestamp)) "+ "AND NOT ARRAYCONTAINS([%q, %q, %q], c.properties.status)", api.OperationStatusResourceType.String(), arm.ProvisioningStateSucceeded, @@ -192,7 +193,7 @@ func (l *cosmosGlobalLister[InternalAPIType, CosmosAPIType]) List(ctx context.Co resourceTypeConditions = append(resourceTypeConditions, fmt.Sprintf("STRINGEQUALS(c.resourceType, %q, true)", resourceType.String())) } whereClause := strings.Join(resourceTypeConditions, " OR ") - query := fmt.Sprintf("SELECT * FROM c WHERE LENGTH(c.resourceID) > 0 AND (%s)", whereClause) + query := fmt.Sprintf("SELECT * FROM c WHERE LENGTH(c.resourceID) > 0 AND (NOT IS_DEFINED(c.deletionTimestamp)) AND (%s)", whereClause) queryOptions := azcosmos.QueryOptions{ PageSizeHint: -1, diff --git a/internal/database/informers/changefeed_list_watch.go b/internal/database/informers/changefeed_list_watch.go index 8c3c8e24db4..c3f0a5dcd42 100644 --- a/internal/database/informers/changefeed_list_watch.go +++ b/internal/database/informers/changefeed_list_watch.go @@ -404,10 +404,16 @@ func (c *ChangeFeedWatcher[InternalAPIType, InternalAPITypePointer, CosmosAPITyp } objDeleted := false - if c.shouldDeliverItemFn != nil && !c.shouldDeliverItemFn(internalObj) { + if objAsTypedDocument.DeletionTimestamp != nil { + if objPreviouslySeen { + objDeleted = true + } else { + logger.Info("skipping soft-deleted document not previously seen", "content", cosmosObj) + return nil + } + } else if c.shouldDeliverItemFn != nil && !c.shouldDeliverItemFn(internalObj) { if objPreviouslySeen { objDeleted = true - // we need to deliver a delete, so fall through } else { logger.Info("should not deliver document", "content", cosmosObj) return nil diff --git a/internal/database/types_typeddocument.go b/internal/database/types_typeddocument.go index b8005925aad..ddca805caa6 100644 --- a/internal/database/types_typeddocument.go +++ b/internal/database/types_typeddocument.go @@ -17,6 +17,8 @@ package database import ( "encoding/json" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + azcorearm "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" ) @@ -26,8 +28,9 @@ import ( // implements the DocumentProperties interface. type TypedDocument struct { BaseDocument - PartitionKey string `json:"partitionKey"` - ResourceID *azcorearm.ResourceID `json:"resourceID"` - ResourceType string `json:"resourceType"` - Properties json.RawMessage `json:"properties"` + PartitionKey string `json:"partitionKey"` + ResourceID *azcorearm.ResourceID `json:"resourceID"` + ResourceType string `json:"resourceType"` + DeletionTimestamp *metav1.Time `json:"deletionTimestamp,omitempty"` + Properties json.RawMessage `json:"properties"` } diff --git a/internal/databasetesting/mock_resources_changefeed.go b/internal/databasetesting/mock_resources_changefeed.go index c7df02e2881..7c2c6d24556 100644 --- a/internal/databasetesting/mock_resources_changefeed.go +++ b/internal/databasetesting/mock_resources_changefeed.go @@ -27,9 +27,10 @@ import ( // mockChangeFeed is an in-memory log of mutations to the Resources // container. Every successful StoreDocument call (Create or Replace) // appends a snapshot of the stored document; reads return everything -// past the consumer's continuation position. Deletes are intentionally -// NOT recorded — that mirrors "latest version" change feed mode in -// production Cosmos DB, which does not surface deletions. +// past the consumer's continuation position. Hard deletes are +// intentionally NOT recorded — that mirrors "latest version" change +// feed mode in production Cosmos DB. Soft deletes (which are modeled +// as Replace operations) ARE recorded via StoreDocument. type mockChangeFeed struct { mu sync.Mutex events []json.RawMessage diff --git a/internal/databasetesting/mock_resources_crud.go b/internal/databasetesting/mock_resources_crud.go index 4d23cdbe50b..0d363bc65a6 100644 --- a/internal/databasetesting/mock_resources_crud.go +++ b/internal/databasetesting/mock_resources_crud.go @@ -21,6 +21,7 @@ import ( "net/http" "path" "strings" + "time" "github.com/google/uuid" @@ -173,6 +174,10 @@ func (m *mockResourceCRUD[InternalAPIType, InternalAPITypePointer, CosmosAPIType return nil, database.NewNotFoundError() } + if database.IsSoftDeleted(data) { + return nil, database.NewNotFoundError() + } + var cosmosObj CosmosAPIType if err := json.Unmarshal(data, &cosmosObj); err != nil { return nil, fmt.Errorf("failed to unmarshal document: %w", err) @@ -303,7 +308,37 @@ func (m *mockResourceCRUD[InternalAPIType, InternalAPITypePointer, CosmosAPIType if err != nil { return err } - m.client.DeleteDocument(cosmosUID) + + return mockSoftDelete(m.client, cosmosUID) +} + +func mockSoftDelete(store mockDocumentStore, cosmosID string) error { + data, ok := store.GetDocument(cosmosID) + if !ok { + return nil + } + + var doc database.GenericDocument[map[string]interface{}] + if err := json.Unmarshal(data, &doc); err != nil { + return fmt.Errorf("failed to unmarshal document for soft delete: %w", err) + } + + if doc.DeletionTimestamp != nil { + return nil + } + + database.SetSoftDeleteFields(&doc, time.Now()) + + modified, err := json.Marshal(doc) + if err != nil { + return fmt.Errorf("failed to marshal soft-deleted document: %w", err) + } + + dataWithETag, _, err := injectETag(json.RawMessage(modified)) + if err != nil { + return fmt.Errorf("failed to inject etag: %w", err) + } + store.StoreDocument(cosmosID, dataWithETag) return nil } @@ -537,12 +572,14 @@ func (m *mockOperationCRUD) ListActiveOperations(options *database.ResourcesDBCl continue } - // Mirror the production query, which requires IS_DEFINED(c.resourceID); - // documents without a resourceID are never returned by list. if typedDoc.ResourceID == nil { continue } + if typedDoc.DeletionTimestamp != nil { + continue + } + var cosmosObj database.GenericDocument[api.Operation] if err := json.Unmarshal(data, &cosmosObj); err != nil { continue @@ -689,6 +726,9 @@ func (m *mockUntypedCRUD) Get(ctx context.Context, resourceID *azcorearm.Resourc data, ok := m.client.GetDocument(newCosmosID) if ok { + if database.IsSoftDeleted(data) { + return nil, database.NewNotFoundError() + } var typedDoc database.TypedDocument if err := json.Unmarshal(data, &typedDoc); err != nil { return nil, fmt.Errorf("failed to unmarshal document: %w", err) @@ -729,6 +769,10 @@ func (m *mockUntypedCRUD) listInternal(ctx context.Context, opts *database.DBCli continue } + if typedDoc.DeletionTimestamp != nil { + continue + } + // For non-recursive, check slash count if nonRecursive { slashCount := strings.Count(typedDoc.ResourceID.String(), "/") @@ -754,13 +798,11 @@ func (m *mockUntypedCRUD) Delete(ctx context.Context, resourceID *azcorearm.Reso if err != nil { return err } - m.client.DeleteDocument(cosmosUID) - return nil + return mockSoftDelete(m.client, cosmosUID) } func (m *mockUntypedCRUD) DeleteByCosmosID(ctx context.Context, partitionKey, cosmosID string) error { - m.client.DeleteDocument(cosmosID) - return nil + return mockSoftDelete(m.client, cosmosID) } func (m *mockUntypedCRUD) Child(resourceType azcorearm.ResourceType, resourceName string) (database.UntypedResourceCRUD, error) { diff --git a/internal/databasetesting/mock_resources_db_client.go b/internal/databasetesting/mock_resources_db_client.go index 10a8f8b46ab..57cc61d0d50 100644 --- a/internal/databasetesting/mock_resources_db_client.go +++ b/internal/databasetesting/mock_resources_db_client.go @@ -122,8 +122,9 @@ func (m *MockResourcesDBClient) ServiceProviderNodePools(subscriptionID, resourc // GetChangeFeed reads the in-memory change-feed log. Each // successful StoreDocument call records a snapshot of the document; // reads return everything past the position encoded in -// options.Continuation. Deletes are not recorded, which mirrors -// "latest version" mode in real Cosmos DB. +// options.Continuation. Hard deletes are not recorded, which mirrors +// "latest version" mode in real Cosmos DB. Soft deletes go through +// StoreDocument and are therefore recorded. func (m *MockResourcesDBClient) GetChangeFeed(ctx context.Context, options *azcosmos.ChangeFeedOptions) (azcosmos.ChangeFeedResponse, error) { var continuation string if options != nil && options.Continuation != nil { @@ -245,6 +246,7 @@ func (m *MockResourcesDBClient) DeleteDocument(cosmosID string) { } // ListDocuments returns all documents matching the given resource type and prefix. +// Soft-deleted documents (those with a DeletionTimestamp set) are excluded. func (m *MockResourcesDBClient) ListDocuments(resourceType *azcorearm.ResourceType, prefix string) []json.RawMessage { m.mu.RLock() defer m.mu.RUnlock() @@ -262,6 +264,10 @@ func (m *MockResourcesDBClient) ListDocuments(resourceType *azcorearm.ResourceTy continue } + if typedDoc.DeletionTimestamp != nil { + continue + } + // Check resource type match if specified if resourceType != nil { if !strings.EqualFold(typedDoc.ResourceType, resourceType.String()) { diff --git a/internal/databasetesting/mock_resources_global_lister.go b/internal/databasetesting/mock_resources_global_lister.go index f62f7a2b969..faf59e98638 100644 --- a/internal/databasetesting/mock_resources_global_lister.go +++ b/internal/databasetesting/mock_resources_global_lister.go @@ -155,12 +155,14 @@ func (l *mockActiveOperationsGlobalLister) List(ctx context.Context, options *da continue } - // Mirror the production query, which requires IS_DEFINED(c.resourceID); - // documents without a resourceID are never returned by list. if typedDoc.ResourceID == nil { continue } + if typedDoc.DeletionTimestamp != nil { + continue + } + var cosmosObj database.GenericDocument[api.Operation] if err := json.Unmarshal(data, &cosmosObj); err != nil { continue @@ -222,6 +224,10 @@ func (l *mockGlobalLister[InternalAPIType, CosmosAPIType]) List(ctx context.Cont continue } + if typedDoc.DeletionTimestamp != nil { + continue + } + var cosmosObj CosmosAPIType if err := json.Unmarshal(data, &cosmosObj); err != nil { continue diff --git a/test-integration/backend/changefeed/changefeed_test.go b/test-integration/backend/changefeed/changefeed_test.go index ef3f4b12cd2..f16d730cd56 100644 --- a/test-integration/backend/changefeed/changefeed_test.go +++ b/test-integration/backend/changefeed/changefeed_test.go @@ -155,28 +155,51 @@ func TestChangeFeedListWatcher(t *testing.T) { }, }, { - name: "deleted item produces no watch event", + name: "soft-deleted item produces Deleted watch event", run: func(t *testing.T, env *changefeedTestEnv) { - clusterRID := env.uniqueClusterResourceID("deleted") + clusterRID := env.uniqueClusterResourceID("soft-deleted") env.createCluster(t, clusterRID) _, watcher := env.startListAndWatch(t) - // Drain any straggling event the list captured (the - // cluster was created before the list, so it should - // already be in the resourceIDToInstanceVersion map - // — but the change feed may still surface it once; - // the watcher's instanceVersion gate should drop it). - // We give the change feed a short moment to do that - // before issuing the delete. drainBriefly(watcher, 1*time.Second) env.deleteCluster(t, clusterRID) - // The "latest version" change feed mode does not - // surface deletes. Wait the full event deadline; any - // delivered event is a failure of the contract. - assertNoEvent(t, watcher, silenceDeadline) + evt := waitForDeleteEvent(t, watcher, clusterRID.String(), eventDeadline) + gotResourceID, _ := metadataOf(t, evt.Object) + require.Truef(t, strings.EqualFold(gotResourceID, clusterRID.String()), + "event resourceID %q != deleted %q", gotResourceID, clusterRID.String()) + }, + }, + { + name: "soft-deleted item not in list returns not-found on Get", + run: func(t *testing.T, env *changefeedTestEnv) { + clusterRID := env.uniqueClusterResourceID("get-after-delete") + env.createCluster(t, clusterRID) + + env.deleteCluster(t, clusterRID) + + _, err := env.resourcesDBClient.HCPClusters(clusterRID.SubscriptionID, clusterRID.ResourceGroupName). + Get(env.ctx, clusterRID.Name) + require.Truef(t, database.IsNotFoundError(err), + "expected not-found error after soft-delete, got: %v", err) + }, + }, + { + name: "relist after soft-delete does not include deleted item in list", + run: func(t *testing.T, env *changefeedTestEnv) { + clusterRID := env.uniqueClusterResourceID("relist-after-delete") + env.createCluster(t, clusterRID) + + env.deleteCluster(t, clusterRID) + + listed, _ := env.startListAndWatch(t) + + for _, rid := range listed { + require.Falsef(t, strings.EqualFold(rid, clusterRID.String()), + "deleted cluster %q must not appear in the list", clusterRID.String()) + } }, }, { @@ -642,6 +665,25 @@ func metadataOf(t *testing.T, obj any) (string, int64) { return rid.String(), accessor.GetInstanceVersion() } +func waitForDeleteEvent(t *testing.T, watcher *clusterChangeFeedWatcher, resourceID string, timeout time.Duration) watch.Event { + t.Helper() + deadline := time.After(timeout) + for { + select { + case evt, ok := <-watcher.ResultChan(): + require.True(t, ok, "watcher result channel closed before delivering a Deleted event") + if evt.Type == watch.Deleted { + rid, _ := metadataOf(t, evt.Object) + if strings.EqualFold(rid, resourceID) { + return evt + } + } + case <-deadline: + t.Fatalf("no Deleted event for %s received within %s", resourceID, timeout) + } + } +} + func waitForEvent(t *testing.T, watcher *clusterChangeFeedWatcher, timeout time.Duration) watch.Event { t.Helper() select { @@ -829,6 +871,64 @@ func TestActiveOperationInformer(t *testing.T) { }) } +// TestListActiveOperationsExcludesSoftDeleted verifies that +// ListActiveOperations does not return operations that have been soft-deleted. +func TestListActiveOperationsExcludesSoftDeleted(t *testing.T) { + integrationutils.WithAndWithoutCosmos(t, func(t *testing.T, withMock bool) { + ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second) + ctx = utils.ContextWithLogger(ctx, integrationutils.DefaultLogger(t)) + + var ( + storage integrationutils.StorageIntegrationTestInfo + err error + ) + if withMock { + storage, err = integrationutils.NewMockCosmosFromTestingEnv(ctx, t) + } else { + storage, err = integrationutils.NewCosmosFromTestingEnv(ctx, t) + } + require.NoError(t, err, "create test storage") + defer func() { + cancel() + cleanupCtx := utils.ContextWithLogger(context.Background(), integrationutils.DefaultLogger(t)) + storage.Cleanup(cleanupCtx) + }() + + resourcesDBClient := storage.ResourcesDBClient() + + clusterRID := api.Must(azcorearm.ParseResourceID(fmt.Sprintf( + "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-active-op-%d", + testSubscriptionID, testResourceGroup, time.Now().UnixNano()))) + opRID := api.Must(azcorearm.ParseResourceID(fmt.Sprintf( + "/subscriptions/%s/providers/Microsoft.RedHatOpenShift/hcpOperationStatuses/list-active-op-%d", + testSubscriptionID, time.Now().UnixNano()))) + + op := newOperationFixture(opRID, clusterRID) + _, err = resourcesDBClient.Operations(opRID.SubscriptionID).Create(ctx, op, nil) + require.NoError(t, err, "Create operation") + + iter := resourcesDBClient.Operations(opRID.SubscriptionID).ListActiveOperations(nil) + found := false + for _, item := range iter.Items(ctx) { + if strings.EqualFold(item.GetResourceID().String(), opRID.String()) { + found = true + } + } + require.NoError(t, iter.GetError(), "ListActiveOperations before delete") + require.True(t, found, "active operation must appear in ListActiveOperations before delete") + + err = resourcesDBClient.Operations(opRID.SubscriptionID).Delete(ctx, opRID.Name) + require.NoError(t, err, "Delete operation") + + iter = resourcesDBClient.Operations(opRID.SubscriptionID).ListActiveOperations(nil) + for _, item := range iter.Items(ctx) { + require.Falsef(t, strings.EqualFold(item.GetResourceID().String(), opRID.String()), + "soft-deleted operation %q must not appear in ListActiveOperations", opRID.String()) + } + require.NoError(t, iter.GetError(), "ListActiveOperations after delete") + }) +} + func waitForChannelEvent(t *testing.T, ch <-chan watch.Event, timeout time.Duration) watch.Event { t.Helper() select {