From 7859e5a5cdadc5ec81537d4e855bcc14d766bae1 Mon Sep 17 00:00:00 2001
From: Ben Broderick Phillips <bebroder@microsoft.com>
Date: Fri, 28 Jun 2024 15:44:57 -0400
Subject: [PATCH] Add continue on error to live test resource cleanup

---
 eng/scripts/live-test-resource-cleanup.ps1 | 59 +++++++++++++---------
 1 file changed, 36 insertions(+), 23 deletions(-)

diff --git a/eng/scripts/live-test-resource-cleanup.ps1 b/eng/scripts/live-test-resource-cleanup.ps1
index 9b2765061da..55f846bb492 100644
--- a/eng/scripts/live-test-resource-cleanup.ps1
+++ b/eng/scripts/live-test-resource-cleanup.ps1
@@ -398,42 +398,51 @@ function DeleteOrUpdateResourceGroups() {
     }
   }
 
-  DeleteAndPurgeGroups $toDelete
+  $hasError = DeleteAndPurgeGroups $toDelete
 
   foreach ($rg in $toClean) {
     DeleteArmDeployments $rg
   }
+
+  if ($hasError) {
+    throw "Encountered errors removing some resource groups"
+  }
 }
 
 function DeleteAndPurgeGroups([array]$toDelete) {
+  $hasError = $false
   # Get purgeable resources already in a deleted state.
   $purgeableResources = @(Get-PurgeableResources)
 
   if ($toDelete) {
     Write-Host "Total Resource Groups To Delete: $($toDelete.Count)"
   }
-  foreach ($rg in $toDelete)
-  {
-    $deleteAfter = GetTag $rg "DeleteAfter"
-    if ($Force -or $PSCmdlet.ShouldProcess("$($rg.ResourceGroupName) [DeleteAfter (UTC): $deleteAfter]", "Delete Group")) {
-      # Add purgeable resources that will be deleted with the resource group to the collection.
-      $purgeableResourcesFromRG = @(Get-PurgeableGroupResources $rg.ResourceGroupName)
+  foreach ($rg in $toDelete) {
+    try {
+      $deleteAfter = GetTag $rg "DeleteAfter"
+      if ($Force -or $PSCmdlet.ShouldProcess("$($rg.ResourceGroupName) [DeleteAfter (UTC): $deleteAfter]", "Delete Group")) {
+        # Add purgeable resources that will be deleted with the resource group to the collection.
+        $purgeableResourcesFromRG = @(Get-PurgeableGroupResources $rg.ResourceGroupName)
 
-      if ($purgeableResourcesFromRG) {
-        $purgeableResources += $purgeableResourcesFromRG
-        Write-Verbose "Found $($purgeableResourcesFromRG.Count) potentially purgeable resources in resource group $($rg.ResourceGroupName)"
-      }
+        if ($purgeableResourcesFromRG) {
+          $purgeableResources += $purgeableResourcesFromRG
+          Write-Verbose "Found $($purgeableResourcesFromRG.Count) potentially purgeable resources in resource group $($rg.ResourceGroupName)"
+        }
 
-      Write-Verbose "Deleting group: $($rg.ResourceGroupName)"
-      Write-Verbose "  tags $($rg.Tags | ConvertTo-Json -Compress)"
+        Write-Verbose "Deleting group: $($rg.ResourceGroupName)"
+        Write-Verbose "  tags $($rg.Tags | ConvertTo-Json -Compress)"
 
-      # For storage tests specifically, if they are aborted then blobs with immutability policies
-      # can be left around which prevent deletion.
-      if ($rg.Tags?.ContainsKey('ServiceDirectory') -and $rg.Tags.ServiceDirectory -like '*storage*') {
-        & $PSScriptRoot/Remove-WormStorageAccounts.ps1 -GroupPrefix $rg.ResourceGroupName
-      } else {
-        Write-Host ($rg | Remove-AzResourceGroup -Force -AsJob).Name
+        # For storage tests specifically, if they are aborted then blobs with immutability policies
+        # can be left around which prevent deletion.
+        if ($rg.Tags?.ContainsKey('ServiceDirectory') -and $rg.Tags.ServiceDirectory -like '*storage*') {
+          & $PSScriptRoot/Remove-WormStorageAccounts.ps1 -GroupPrefix $rg.ResourceGroupName
+        } else {
+          Write-Host ($rg | Remove-AzResourceGroup -Force -AsJob).Name
+        }
       }
+    } catch {
+      Write-Error $_
+      $hasError = $true
     }
   }
 
@@ -449,6 +458,8 @@ function DeleteAndPurgeGroups([array]$toDelete) {
       $failedResources | Sort-Object AzsdkResourceType, AzsdkName | Format-Table -Property @{l='Type'; e={$_.AzsdkResourceType}}, @{l='Name'; e={$_.AzsdkName}}
     }
   }
+
+  return $hasError
 }
 
 function Login() {
@@ -488,8 +499,10 @@ if ($SubscriptionId -and ($originalSubscription -ne $SubscriptionId)) {
   Select-AzSubscription -Subscription $SubscriptionId -Confirm:$false -WhatIf:$false
 }
 
-DeleteOrUpdateResourceGroups
-
-if ($SubscriptionId -and ($originalSubscription -ne $SubscriptionId)) {
-  Select-AzSubscription -Subscription $originalSubscription -Confirm:$false -WhatIf:$false
+try {
+  DeleteOrUpdateResourceGroups
+} finally {
+  if ($SubscriptionId -and ($originalSubscription -ne $SubscriptionId)) {
+    Select-AzSubscription -Subscription $originalSubscription -Confirm:$false -WhatIf:$false
+  }
 }