diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1
index fa524d987eef..2b6c7c5c4871 100644
--- a/eng/common/TestResources/Remove-TestResources.ps1
+++ b/eng/common/TestResources/Remove-TestResources.ps1
@@ -61,6 +61,21 @@ if (!$PSBoundParameters.ContainsKey('ErrorAction')) {
     $ErrorActionPreference = 'Stop'
 }
 
+# Support actions to invoke on exit.
+$exitActions = @({
+    if ($exitActions.Count -gt 1) {
+        Write-Verbose 'Running registered exit actions.'
+    }
+})
+
+trap {
+    # Like using try..finally in PowerShell, but without keeping track of more braces or tabbing content.
+    $exitActions.Invoke()
+}
+
+# Source helpers to purge resources.
+. "$PSScriptRoot\..\scripts\Helpers\Resource-Helpers.ps1"
+
 function Log($Message) {
     Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
 }
@@ -86,18 +101,6 @@ function Retry([scriptblock] $Action, [int] $Attempts = 5) {
     }
 }
 
-# Support actions to invoke on exit.
-$exitActions = @({
-    if ($exitActions.Count -gt 1) {
-        Write-Verbose 'Running registered exit actions.'
-    }
-})
-
-trap {
-    # Like using try..finally in PowerShell, but without keeping track of more braces or tabbing content.
-    $exitActions.Invoke()
-}
-
 if ($ProvisionerApplicationId) {
     $null = Disable-AzContextAutosave -Scope Process
 
@@ -213,16 +216,23 @@ $verifyDeleteScript = {
     }
 }
 
+# Get any resources that can be purged after the resource group is deleted coerced into a collection even if empty.
+$purgeableResources = @(Get-PurgeableGroupResources $ResourceGroupName)
+
 Log "Deleting resource group '$ResourceGroupName'"
-if ($Force) {
+if ($Force -and !$purgeableResources) {
     Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force -AsJob
+    Write-Verbose "Running background job to delete resource group '$ResourceGroupName'"
+
     Retry $verifyDeleteScript 3
-    Write-Verbose "Requested async deletion of resource group '$ResourceGroupName'"
 } else {
     # Don't swallow interactive confirmation when Force is false
     Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force
 }
 
+# Now purge the resources that should have been deleted with the resource group.
+Remove-PurgeableResources $purgeableResources
+
 $exitActions.Invoke()
 
 <#
diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1
new file mode 100644
index 000000000000..e4aca5544e05
--- /dev/null
+++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1
@@ -0,0 +1,101 @@
+# Add 'AzsdkResourceType' member to outputs since actual output types have changed over the years.
+
+function Get-PurgeableGroupResources {
+    param (
+        [Parameter(Mandatory=$true, Position=0)]
+        [string] $ResourceGroupName
+    )
+
+    # Get any Key Vaults that will be deleted so they can be purged later if soft delete is enabled.
+    Get-AzKeyVault @PSBoundParameters | ForEach-Object {
+        # Enumerating vaults from a resource group does not return all properties we required.
+        Get-AzKeyVault -VaultName $_.VaultName | Where-Object { $_.EnableSoftDelete } `
+            | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru
+    }
+
+    # Get any Managed HSMs in the resource group, for which soft delete cannot be disabled.
+    Get-AzKeyVaultManagedHsm @PSBoundParameters `
+        | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Managed HSM' -PassThru
+}
+
+function Get-PurgeableResources {
+    $subscriptionId = (Get-AzContext).Subscription.Id
+
+    # Get deleted Key Vaults for the current subscription.
+    Get-AzKeyVault -InRemovedState `
+        | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru
+
+    # Get deleted Managed HSMs for the current subscription.
+    $response = Invoke-AzRestMethod -Method GET -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/deletedManagedHSMs?api-version=2021-04-01-preview" -ErrorAction Ignore
+    if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300 -and $response.Content) {
+        $content = $response.Content | ConvertFrom-Json
+        foreach ($r in $content.value) {
+            [pscustomobject] @{
+                AzsdkResourceType = 'Managed HSM'
+                Id = $r.id
+                Name = $r.name
+                Location = $r.properties.location
+                DeletionDate = $r.properties.deletionDate -as [DateTime]
+                ScheduledPurgeDate = $r.properties.scheduledPurgeDate -as [DateTime]
+                EnablePurgeProtection = $r.properties.purgeProtectionEnabled
+            }
+        }
+    }
+}
+
+# A filter differs from a function by teating body as -process {} instead of -end {}.
+# This allows you to pipe a collection and process each item in the collection.
+filter Remove-PurgeableResources {
+    param (
+        [Parameter(Position=0, ValueFromPipeline=$true)]
+        [object[]] $Resource
+    )
+
+    if (!$Resource) {
+        return
+    }
+
+    $subscriptionId = (Get-AzContext).Subscription.Id
+
+    foreach ($r in $Resource) {
+        switch ($r.AzsdkResourceType) {
+            'Key Vault' {
+                Log "Attempting to purge $($r.AzsdkResourceType) '$($r.VaultName)'"
+                if ($r.EnablePurgeProtection) {
+                    # We will try anyway but will ignore errors
+                    Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
+                }
+            
+                Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue
+            }
+
+            'Managed HSM' {
+                Log "Attempting to purge $($r.AzsdkResourceType) '$($r.Name)'"
+                if ($r.EnablePurgeProtection) {
+                    # We will try anyway but will ignore errors
+                    Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"
+                }
+            
+                $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore
+                if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) {
+                    Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged."
+                } elseif ($response.Content) {
+                    $content = $response.Content | ConvertFrom-Json
+                    if ($content.error) {
+                        $err = $content.error
+                        Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)"
+                    }
+                }
+            }
+
+            default {
+                Write-Warning "Cannot purge resource type $($r.AzsdkResourceType). Add support to https://github.com/Azure/azure-sdk-tools/blob/main/eng/common/scripts/Helpers/Resource-Helpers.ps1."
+            }
+        }
+    }
+}
+
+# The Log function can be overridden by the sourcing script.
+function Log($Message) {
+    Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
+}
diff --git a/sdk/keyvault/remove-test-resources-pre.ps1 b/sdk/keyvault/remove-test-resources-pre.ps1
deleted file mode 100644
index 70b4515ebd69..000000000000
--- a/sdk/keyvault/remove-test-resources-pre.ps1
+++ /dev/null
@@ -1,82 +0,0 @@
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License.
-
-# IMPORTANT: Do not invoke this file directly. Please instead run eng/common/TestResources/Remove-TestResources.ps1 from the repository root.
-
-#Requires -Version 6.0
-#Requires -PSEdition Core
-
-# Use same parameter names as declared in eng/common/TestResources/Remove-TestResources.ps1 (assume validation therein).
-[CmdletBinding()]
-param (
-    [Parameter()]
-    [string] $ResourceGroupName,
-
-    [Parameter()]
-    [string] $TenantId,
-
-    [Parameter()]
-    [string] $ProvisionerApplicationId,
-
-    [Parameter()]
-    [string] $ProvisionerApplicationSecret,
-
-    [Parameter()]
-    [string] $SubscriptionId,
-
-    # Captures any arguments from eng/common/Remove-TestResources.ps1 not declared here (no parameter errors).
-    [Parameter(ValueFromRemainingArguments = $true)]
-    $RemainingArguments
-)
-
-# By default stop for any error.
-if (!$PSBoundParameters.ContainsKey('ErrorAction')) {
-    $ErrorActionPreference = 'Stop'
-}
-
-function Log($Message) {
-    Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
-}
-
-function PurgeKeyVault($Vault) {
-  Log "Deleting Key Vault named '$($Vault.VaultName)'"
-  Remove-AzKeyVault -Name "$($Vault.VaultName)" -ResourceGroupName "$($Vault.ResourceGroupName)" -Location "$($Vault.Location)" -Force
-
-  Log "Purging Key Vault named '$($Vault.VaultName)'"
-  Remove-AzKeyVault -Name "$($Vault.VaultName)" -Location "$($Vault.Location)" -InRemovedState -Force
-
-  Log "'$($Vault.VaultName)' successfully deleted and purged."
-}
-
-function PurgeManagedHsm($ManagedHsm) {
-  Log "Deleting Managed HSM named '$($ManagedHsm.Name)'"
-  az keyvault delete --resource-group "$ResourceGroupName" --hsm-name "$($ManagedHsm.Name)"
-
-  Log "Purging Managed HSM named '$($ManagedHsm.Name)'"
-  az keyvault purge --hsm-name "$($ManagedHsm.Name)" --location "$($ManagedHsm.Location)"
-
-  Log "$($ManagedHsm.Name) successfully deleted and purged."
-}
-
-Log "Permanently deleting all Key Vaults in resource group $ResourceGroupName"
-Get-AzKeyVault -ResourceGroupName $ResourceGroupName | ForEach-Object { PurgeKeyVault($_) }
-
-# TODO: Use Az module when available; for now, assumes Azure CLI is installed and in $Env:PATH.
-if ($ProvisionerApplicationId -and $ProvisionerApplicationSecret -and $TenantId) {
-  Log "Logging '$ProvisionerApplicationId' into the Azure CLI"
-  az login --service-principal --tenant "$tenantId" --username "$ProvisionerApplicationId" --password="$ProvisionerApplicationSecret" --output none
-  Log "Setting the subscription for the logged in user"
-  az account set --subscription "$SubscriptionId"
-} else {
-  Log "No credentials provided; skipping Azure CLI login and assuming current user is logged in and is using the correct subscription."
-}
-
-Log "Permanently deleting all Managed HSMs in resource group $ResourceGroupName"
-Get-AzKeyVaultManagedHsm -ResourceGroupName "$ResourceGroupName" | ForEach-Object { PurgeManagedHsm($_) }
-
-Log "Successfully deleted and purged all Key Vaults and Managed HSMs."
-
-if ($ProvisionerApplicationId) {
-  Log "Logging out of Azure CLI"
-  az logout --username "$ProvisionerApplicationId"
-}