diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1
index 49616b64b33..9d940e72316 100644
--- a/eng/common/TestResources/New-TestResources.ps1
+++ b/eng/common/TestResources/New-TestResources.ps1
@@ -771,7 +771,6 @@ try {
                     -TemplateParameterObject $templateFileParameters `
                     -Force:$Force
         }
-
         if ($deployment.ProvisioningState -ne 'Succeeded') {
             Write-Host "Deployment '$($deployment.DeploymentName)' has state '$($deployment.ProvisioningState)' with CorrelationId '$($deployment.CorrelationId)'. Exiting..."
             Write-Host @'
@@ -803,6 +802,9 @@ try {
             Write-Verbose "Removing compiled bicep file $($templateFile.jsonFilePath)"
             Remove-Item $templateFile.jsonFilePath
         }
+
+        Write-Host "Deleting ARM deployment as it may contain secrets. Deployed resources will not be affected."
+        $null = $deployment | Remove-AzResourceGroupDeployment
     }
 
 } finally {
diff --git a/eng/scripts/Remove-WormStorageAccounts.ps1 b/eng/scripts/Remove-WormStorageAccounts.ps1
index 98ffd0ca46e..6d37365dc39 100644
--- a/eng/scripts/Remove-WormStorageAccounts.ps1
+++ b/eng/scripts/Remove-WormStorageAccounts.ps1
@@ -41,7 +41,12 @@ foreach ($group in $groups) {
                 Write-Error $_
                 throw
             }
-            $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlob -Force
+            # Sometimes we get a 404 blob not found but can still delete containers,
+            # and sometimes we must delete the blob if there's a legal hold.
+            # Try to remove the blob, but keep running regardless.
+            try {
+                $ctx | Get-AzStorageContainer | Get-AzStorageBlob | Remove-AzStorageBlob -Force
+            } catch {}
             # Use AzRm cmdlet as deletion will only work through ARM with the immutability policies defined on the blobs
             $ctx | Get-AzStorageContainer | % { Remove-AzRmStorageContainer -Name $_.Name -StorageAccountName $ctx.StorageAccountName -ResourceGroupName $group.ResourceGroupName -Force }
             Remove-AzStorageAccount -StorageAccountName $account.StorageAccountName -ResourceGroupName $account.ResourceGroupName -Force
diff --git a/eng/scripts/live-test-resource-cleanup.ps1 b/eng/scripts/live-test-resource-cleanup.ps1
index 2f6a0a8669a..bdbe25ca313 100644
--- a/eng/scripts/live-test-resource-cleanup.ps1
+++ b/eng/scripts/live-test-resource-cleanup.ps1
@@ -52,6 +52,9 @@ param (
   [Parameter()]
   [switch] $DeleteNonCompliantGroups,
 
+  [Parameter()]
+  [switch] $DeleteArmDeployments,
+
   [Parameter()]
   [int] $DeleteAfterHours = 24,
 
@@ -269,7 +272,7 @@ function FindOrCreateDeleteAfterTag {
     [object]$ResourceGroup
   )
 
-  if (!$ResourceGroup) {
+  if (!$DeleteNonCompliantGroups -or !$ResourceGroup) {
       return
   }
 
@@ -326,6 +329,14 @@ function HasDeleteLock([object]$ResourceGroup) {
   return $false
 }
 
+function DeleteArmDeployments([object]$ResourceGroup) {
+  if (!$DeleteArmDeployments) {
+      return
+  }
+  Write-Host "Deleting ARM deployments for group $($ResourceGroup.ResourceGroupName) as they may contain secrets. Deployed resources will not be affected."
+  $null = Get-AzResourceGroupDeployment -ResourceGroupName $ResourceGroup.ResourceGroupName | Remove-AzResourceGroupDeployment
+}
+
 function DeleteOrUpdateResourceGroups() {
   [CmdletBinding(SupportsShouldProcess = $true, ConfirmImpact = 'Medium')]
   param()
@@ -338,6 +349,7 @@ function DeleteOrUpdateResourceGroups() {
   [Array]$allGroups = Retry { Get-AzResourceGroup }
   $toDelete = @()
   $toUpdate = @()
+  $toClean = @()
   Write-Host "Total Resource Groups: $($allGroups.Count)"
 
   foreach ($rg in $allGroups) {
@@ -351,31 +363,25 @@ function DeleteOrUpdateResourceGroups() {
       }
       continue
     }
-    if (!$DeleteNonCompliantGroups) {
-      continue
-    }
-    if (HasDoNotDeleteTag $rg) {
+    if ((IsChildResource $rg) -or (HasDeleteLock $rg)) {
       continue
     }
-    if (IsChildResource $rg) {
-      continue
-    }
-    if (HasValidAliasInName $rg) {
-      continue
-    }
-    if (HasValidOwnerTag $rg) {
-      continue
-    }
-    if (HasDeleteLock $rg) {
+    if ((HasDoNotDeleteTag $rg) -or (HasValidAliasInName $rg) -or (HasValidOwnerTag $rg)) {
+      $toClean += $rg
       continue
     }
     $toUpdate += $rg
   }
 
+
   foreach ($rg in $toUpdate) {
     FindOrCreateDeleteAfterTag $rg
   }
 
+  foreach ($rg in $toClean) {
+    DeleteArmDeployments $rg
+  }
+
   # Get purgeable resources already in a deleted state.
   $purgeableResources = @(Get-PurgeableResources)