From 01a2bfc0c0483487fd14ad987a11b2968c5572bd Mon Sep 17 00:00:00 2001 From: Heath Stewart Date: Wed, 11 Aug 2021 10:55:37 -0700 Subject: [PATCH 1/8] Purge Key Vaults after deleting resource group --- .../TestResources/Remove-TestResources.ps1 | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1 index fa524d987eef..88dd28b42e70 100644 --- a/eng/common/TestResources/Remove-TestResources.ps1 +++ b/eng/common/TestResources/Remove-TestResources.ps1 @@ -213,16 +213,38 @@ $verifyDeleteScript = { } } +# Get any Key Vaults that will be deleted so they can be purged later if soft delete is enabled. +$vaults = Get-AzKeyVault -ResourceGroupName "$ResourceGroupName" | ForEach-Object { + # Enumerating vaults from a resource group does not return all properties we required. + Get-AzKeyVault -VaultName $_.VaultName | Where-Object { $_.EnableSoftDelete } +} + +# You may add additional resource checks that require the resource group to be deleted first before purging here. +$purgeRequired = !!$vaults + Log "Deleting resource group '$ResourceGroupName'" -if ($Force) { +if ($Force -and !$purgeRequired) { Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force -AsJob + Write-Verbose "Running background job to delete resource group '$ResourceGroupName'" + Retry $verifyDeleteScript 3 - Write-Verbose "Requested async deletion of resource group '$ResourceGroupName'" } else { # Don't swallow interactive confirmation when Force is false Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force } +# Purge any soft deleted vaults since there is now a limit per-subscription. +foreach ($vault in $vaults) { + Log "Attempting to purge Key Vault '$($vault.VaultName)'" + + if ($vault.EnablePurgeProtection) { + # We will try anyway but will ignore errors + Write-Warning "Key Vault '$($vault.VaultName)' has purge protection enabled and may not be purged for $($vault.SoftDeleteRetentionInDays) days" + } + + Remove-AzKeyVault -VaultName $vault.VaultName -Location $vault.Location -InRemovedState -Force -ErrorAction Ignore +} + $exitActions.Invoke() <# From 5f548dc8f2171bea9cf58b4e6cd99f6e85b0bccd Mon Sep 17 00:00:00 2001 From: Heath Stewart Date: Wed, 11 Aug 2021 16:16:31 -0700 Subject: [PATCH 2/8] Resolve PR feedback --- eng/common/TestResources/Remove-TestResources.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1 index 88dd28b42e70..d2dc1f4ed2f8 100644 --- a/eng/common/TestResources/Remove-TestResources.ps1 +++ b/eng/common/TestResources/Remove-TestResources.ps1 @@ -220,7 +220,7 @@ $vaults = Get-AzKeyVault -ResourceGroupName "$ResourceGroupName" | ForEach-Objec } # You may add additional resource checks that require the resource group to be deleted first before purging here. -$purgeRequired = !!$vaults +$purgeRequired = $vaults -or $false Log "Deleting resource group '$ResourceGroupName'" if ($Force -and !$purgeRequired) { From f4d683d087e7e688ec69f53d4c46edfc99ae6fb0 Mon Sep 17 00:00:00 2001 From: Heath Stewart Date: Thu, 12 Aug 2021 14:14:48 -0700 Subject: [PATCH 3/8] Purge Key Vaults and Managed HSMs --- .../TestResources/Remove-TestResources.ps1 | 52 ++++++-------- .../scripts/Helpers/Resource-Helpers.ps1 | 71 +++++++++++++++++++ 2 files changed, 91 insertions(+), 32 deletions(-) create mode 100644 eng/common/scripts/Helpers/Resource-Helpers.ps1 diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1 index d2dc1f4ed2f8..dbf7bb0e837f 100644 --- a/eng/common/TestResources/Remove-TestResources.ps1 +++ b/eng/common/TestResources/Remove-TestResources.ps1 @@ -61,6 +61,21 @@ if (!$PSBoundParameters.ContainsKey('ErrorAction')) { $ErrorActionPreference = 'Stop' } +# Support actions to invoke on exit. +$exitActions = @({ + if ($exitActions.Count -gt 1) { + Write-Verbose 'Running registered exit actions.' + } +}) + +trap { + # Like using try..finally in PowerShell, but without keeping track of more braces or tabbing content. + $exitActions.Invoke() +} + +# Source helpers to purge resources. +. "$PSScriptRoot\..\scripts\Helpers\Resource-Helpers.ps1" + function Log($Message) { Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message) } @@ -86,18 +101,6 @@ function Retry([scriptblock] $Action, [int] $Attempts = 5) { } } -# Support actions to invoke on exit. -$exitActions = @({ - if ($exitActions.Count -gt 1) { - Write-Verbose 'Running registered exit actions.' - } -}) - -trap { - # Like using try..finally in PowerShell, but without keeping track of more braces or tabbing content. - $exitActions.Invoke() -} - if ($ProvisionerApplicationId) { $null = Disable-AzContextAutosave -Scope Process @@ -213,17 +216,11 @@ $verifyDeleteScript = { } } -# Get any Key Vaults that will be deleted so they can be purged later if soft delete is enabled. -$vaults = Get-AzKeyVault -ResourceGroupName "$ResourceGroupName" | ForEach-Object { - # Enumerating vaults from a resource group does not return all properties we required. - Get-AzKeyVault -VaultName $_.VaultName | Where-Object { $_.EnableSoftDelete } -} - -# You may add additional resource checks that require the resource group to be deleted first before purging here. -$purgeRequired = $vaults -or $false +# Get any resources that can be purged after the resource group is deleted. +$purgeableResources = Get-PurgeableResources $ResourceGroupName Log "Deleting resource group '$ResourceGroupName'" -if ($Force -and !$purgeRequired) { +if ($Force -and !$purgeableResources) { Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force -AsJob Write-Verbose "Running background job to delete resource group '$ResourceGroupName'" @@ -233,17 +230,8 @@ if ($Force -and !$purgeRequired) { Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force } -# Purge any soft deleted vaults since there is now a limit per-subscription. -foreach ($vault in $vaults) { - Log "Attempting to purge Key Vault '$($vault.VaultName)'" - - if ($vault.EnablePurgeProtection) { - # We will try anyway but will ignore errors - Write-Warning "Key Vault '$($vault.VaultName)' has purge protection enabled and may not be purged for $($vault.SoftDeleteRetentionInDays) days" - } - - Remove-AzKeyVault -VaultName $vault.VaultName -Location $vault.Location -InRemovedState -Force -ErrorAction Ignore -} +# Now purge the resources that should have been deleted with the resource group. +Remove-PurgeableResources $purgeableResources $exitActions.Invoke() diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 new file mode 100644 index 000000000000..9b5d425bc050 --- /dev/null +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -0,0 +1,71 @@ +# Add 'AzsdkResourceType' member to outputs since actual output types have changed over the years. + +function Get-PurgeableResources { + param ( + [Parameter(Position=0)] + [ValidateNotNullOrEmpty()] + [string] $ResourceGroupName + ) + + # Get any Key Vaults that will be deleted so they can be purged later if soft delete is enabled. + Get-AzKeyVault @PSBoundParameters | ForEach-Object { + # Enumerating vaults from a resource group does not return all properties we required. + Get-AzKeyVault -VaultName $_.VaultName | Where-Object { $_.EnableSoftDelete } ` + | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru + } + + # Get any Managed HSMs in the resource group, for which soft delete cannot be disabled. + Get-AzKeyVaultManagedHsm @PSBoundParameters ` + | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Managed HSM' -PassThru +} + +function Remove-PurgeableResources { + param ( + [Parameter(Mandatory=$true, Position=0, ValueFromPipeline=$true)] + [psobject[]] $Resource + ) + + $subscriptionId = (Get-AzContext).Subscription.Id + + foreach ($r in $Resource) { + Log "Attempting to purge $($r.AzsdkResourceType) '$($r.VaultName)'" + + switch ($r.AzsdkResourceType) { + 'Key Vault' { + if ($r.EnablePurgeProtection) { + # We will try anyway but will ignore errors + Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days" + } + + Remove-AzKeyVault -VaultName $r.VaultName -Location $r.Location -InRemovedState -Force -ErrorAction Continue + } + + 'Managed HSM' { + if ($r.EnablePurgeProtection) { + # We will try anyway but will ignore errors + Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days" + } + + $response = Invoke-AzRestMethod -Method POST -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/locations/$($r.Location)/deletedManagedHSMs/$($r.Name)/purge?api-version=2021-04-01-preview" -ErrorAction Ignore + if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300) { + Write-Warning "Successfully requested that Managed HSM '$($r.Name)' be purged, but may take a few minutes before it is actually purged." + } elseif ($response.Content) { + $content = $response.Content | ConvertFrom-Json + if ($content.error) { + $err = $content.error + Write-Warning "Failed to deleted Managed HSM '$($r.Name)': ($($err.code)) $($err.message)" + } + } + } + + default { + Write-Warning "Cannot purge resource type $($r.AzsdkResourceType). Add support to https://github.com/Azure/azure-sdk-tools/blob/main/eng/common/scripts/Helpers/Resource-Helpers.ps1." + } + } + } +} + +# The Log function can be overridden by the sourcing script. +function Log($Message) { + Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message) +} From 3b93e1ba6076663e1f5efa2e73c3ca68d8e4977a Mon Sep 17 00:00:00 2001 From: Heath Stewart Date: Thu, 12 Aug 2021 15:25:16 -0700 Subject: [PATCH 4/8] Clean up all purgeable KVs and MHSMs --- .../TestResources/Remove-TestResources.ps1 | 2 +- .../scripts/Helpers/Resource-Helpers.ps1 | 30 +++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1 index dbf7bb0e837f..d2f64a03576c 100644 --- a/eng/common/TestResources/Remove-TestResources.ps1 +++ b/eng/common/TestResources/Remove-TestResources.ps1 @@ -217,7 +217,7 @@ $verifyDeleteScript = { } # Get any resources that can be purged after the resource group is deleted. -$purgeableResources = Get-PurgeableResources $ResourceGroupName +$purgeableResources = Get-PurgeableGroupResources $ResourceGroupName Log "Deleting resource group '$ResourceGroupName'" if ($Force -and !$purgeableResources) { diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index 9b5d425bc050..9ad6abd781b9 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -1,9 +1,8 @@ # Add 'AzsdkResourceType' member to outputs since actual output types have changed over the years. -function Get-PurgeableResources { +function Get-PurgeableGroupResources { param ( - [Parameter(Position=0)] - [ValidateNotNullOrEmpty()] + [Parameter(Mandatory=$true, Position=0)] [string] $ResourceGroupName ) @@ -19,6 +18,31 @@ function Get-PurgeableResources { | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Managed HSM' -PassThru } +function Get-PurgeableResources { + $subscriptionId = (Get-AzContext).Subscription.Id + + # Get deleted Key Vaults for the current subscription. + Get-AzKeyVault -InRemovedState ` + | Add-Member -MemberType NoteProperty -Name AzsdkResourceType -Value 'Key Vault' -PassThru + + # Get deleted Managed HSMs for the current subscription. + $response = Invoke-AzRestMethod -Method GET -Path "/subscriptions/$subscriptionId/providers/Microsoft.KeyVault/deletedManagedHSMs?api-version=2021-04-01-preview" -ErrorAction Ignore + if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300 -and $response.Content) { + $content = $response.Content | ConvertFrom-Json + foreach ($r in $content.value) { + [pscustomobject] @{ + AzsdkResourceType = 'Managed HSM' + Id = $r. + Name = $r.name + Location = $r.properties.location + DeletionDate = $r.properties.deletionDate -as [DateTime] + ScheduledPurgeDate = $r.properties.scheduledPurgeDate -as [DateTime] + EnablePurgeProtection = $r.properties.purgeProtectionEnabled + } + } + } +} + function Remove-PurgeableResources { param ( [Parameter(Mandatory=$true, Position=0, ValueFromPipeline=$true)] From dc12660371f431182f71f9b72b4423876c9d5ffb Mon Sep 17 00:00:00 2001 From: Heath Stewart Date: Thu, 12 Aug 2021 16:02:45 -0700 Subject: [PATCH 5/8] Ignore null/empty purgeable resources --- eng/common/TestResources/Remove-TestResources.ps1 | 4 ++-- eng/common/scripts/Helpers/Resource-Helpers.ps1 | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/eng/common/TestResources/Remove-TestResources.ps1 b/eng/common/TestResources/Remove-TestResources.ps1 index d2f64a03576c..2b6c7c5c4871 100644 --- a/eng/common/TestResources/Remove-TestResources.ps1 +++ b/eng/common/TestResources/Remove-TestResources.ps1 @@ -216,8 +216,8 @@ $verifyDeleteScript = { } } -# Get any resources that can be purged after the resource group is deleted. -$purgeableResources = Get-PurgeableGroupResources $ResourceGroupName +# Get any resources that can be purged after the resource group is deleted coerced into a collection even if empty. +$purgeableResources = @(Get-PurgeableGroupResources $ResourceGroupName) Log "Deleting resource group '$ResourceGroupName'" if ($Force -and !$purgeableResources) { diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index 9ad6abd781b9..ebea638d8052 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -49,6 +49,10 @@ function Remove-PurgeableResources { [psobject[]] $Resource ) + if (!$Resource) { + return + } + $subscriptionId = (Get-AzContext).Subscription.Id foreach ($r in $Resource) { From 90bec4f44db8a814b696c7f88e9389efbfc7a40b Mon Sep 17 00:00:00 2001 From: Heath Stewart Date: Thu, 12 Aug 2021 16:08:58 -0700 Subject: [PATCH 6/8] Resolve PR feedback --- eng/common/scripts/Helpers/Resource-Helpers.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index ebea638d8052..b0fdab6158f9 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -45,7 +45,7 @@ function Get-PurgeableResources { function Remove-PurgeableResources { param ( - [Parameter(Mandatory=$true, Position=0, ValueFromPipeline=$true)] + [Parameter(Position=0, ValueFromPipeline=$true)] [psobject[]] $Resource ) From e9e7248fbe45e182e42523314bba912229f36fd6 Mon Sep 17 00:00:00 2001 From: Heath Stewart Date: Thu, 12 Aug 2021 16:55:33 -0700 Subject: [PATCH 7/8] Resolve PR feedback --- eng/common/scripts/Helpers/Resource-Helpers.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index b0fdab6158f9..a8d845cad272 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -32,7 +32,7 @@ function Get-PurgeableResources { foreach ($r in $content.value) { [pscustomobject] @{ AzsdkResourceType = 'Managed HSM' - Id = $r. + Id = $r.id Name = $r.name Location = $r.properties.location DeletionDate = $r.properties.deletionDate -as [DateTime] From 33035665f425098bcec4dd94e3f101c79dfc6fad Mon Sep 17 00:00:00 2001 From: Heath Stewart Date: Thu, 12 Aug 2021 17:15:45 -0700 Subject: [PATCH 8/8] Fix logging, process each item when collection piped --- eng/common/scripts/Helpers/Resource-Helpers.ps1 | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index a8d845cad272..e4aca5544e05 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -43,10 +43,12 @@ function Get-PurgeableResources { } } -function Remove-PurgeableResources { +# A filter differs from a function by teating body as -process {} instead of -end {}. +# This allows you to pipe a collection and process each item in the collection. +filter Remove-PurgeableResources { param ( [Parameter(Position=0, ValueFromPipeline=$true)] - [psobject[]] $Resource + [object[]] $Resource ) if (!$Resource) { @@ -56,10 +58,9 @@ function Remove-PurgeableResources { $subscriptionId = (Get-AzContext).Subscription.Id foreach ($r in $Resource) { - Log "Attempting to purge $($r.AzsdkResourceType) '$($r.VaultName)'" - switch ($r.AzsdkResourceType) { 'Key Vault' { + Log "Attempting to purge $($r.AzsdkResourceType) '$($r.VaultName)'" if ($r.EnablePurgeProtection) { # We will try anyway but will ignore errors Write-Warning "Key Vault '$($r.VaultName)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days" @@ -69,6 +70,7 @@ function Remove-PurgeableResources { } 'Managed HSM' { + Log "Attempting to purge $($r.AzsdkResourceType) '$($r.Name)'" if ($r.EnablePurgeProtection) { # We will try anyway but will ignore errors Write-Warning "Managed HSM '$($r.Name)' has purge protection enabled and may not be purged for $($r.SoftDeleteRetentionInDays) days"