Use ARM templates for Key Vault tests

eng/New-TestResources.ps1

@@ -55,6 +55,9 @@ param (
     [ValidateNotNullOrEmpty()]
     [hashtable] $AdditionalParameters,
 
+    [Parameter()]
+    [switch] $CI = ($null -ne $env:SYSTEM_TEAMPROJECTID),
+
     [Parameter()]
     [switch] $Force
 )
@@ -122,7 +125,14 @@ if ($TestApplicationId -and !$TestApplicationOid) {
 }
 
 # Format the resource group name based on resource group naming recommendations and limitations.
-$resourceGroupName = "rg-{0}-$baseName" -f ($ServiceDirectory -replace '[\\\/]', '-').Substring(0, [Math]::Min($ServiceDirectory.Length, 90 - $BaseName.Length - 4)).Trim('-')
+$resourceGroupName = if ($CI) {
+    $BaseName = 't' + (New-Guid).ToString('n').Substring(0, 16)
+    Write-Verbose "Generated base name '$BaseName' for CI build"
+
+    "rg-{0}-$BaseName" -f ($ServiceDirectory -replace '[\\\/]', '-').Substring(0, [Math]::Min($ServiceDirectory.Length, 90 - $BaseName.Length - 4)).Trim('-')
+} else {
+    "rg-$BaseName"
+}
 
 # Tag the resource group to be deleted after a certain number of hours if specified.
 $tags = @{
@@ -135,7 +145,7 @@ if ($PSBoundParameters.ContainsKey('DeleteAfterHours')) {
     $tags.Add('DeleteAfter', $deleteAfter.ToString('o'))
 }
 
-if ($env:SYSTEM_TEAMPROJECTID) {
+if ($CI) {
     # Add tags for the current CI job.
     $tags += @{
         BuildId = "${env:BUILD_BUILDID}"
@@ -145,11 +155,12 @@ if ($env:SYSTEM_TEAMPROJECTID) {
     }
 
     # Set the resource group name variable.
+    Write-Host "Setting variable 'AZURE_RESOURCEGROUP_NAME': $resourceGroupName"
     Write-Host "##vso[task.setvariable variable=AZURE_RESOURCEGROUP_NAME;]$resourceGroupName"
 }
 
-Log "Creating resource group '${resourceGroupName}' in location '$Location'"
-$resourceGroup = New-AzResourceGroup -Name "${resourceGroupName}" -Location $Location -Tag $tags -Force:$Force
+Log "Creating resource group '$resourceGroupName' in location '$Location'"
+$resourceGroup = New-AzResourceGroup -Name "$resourceGroupName" -Location $Location -Tag $tags -Force:$Force
 if ($resourceGroup.ProvisioningState -eq 'Succeeded') {
     # New-AzResourceGroup would've written an error and stopped the pipeline by default anyway.
     Write-Verbose "Successfully created resource group '$($resourceGroup.ResourceGroupName)'"
@@ -208,7 +219,7 @@ foreach ($templateFile in $templateFiles) {
         Write-Verbose "Successfully deployed template '$templateFile' to resource group '$($resourceGroup.ResourceGroupName)'"
     }
 
-    if ($deployment.Outputs.Count -and !$env:SYSTEM_TEAMPROJECTID) {
+    if ($deployment.Outputs.Count -and !$CI) {
         # Write an extra new line to isolate the environment variables for easy reading.
         Log "Persist the following environment variables based on your detected shell ($shell):`n"
     }
@@ -223,10 +234,12 @@ foreach ($templateFile in $templateFiles) {
         if ($variable.Type -eq 'String' -or $variable.Type -eq 'SecureString') {
             $deploymentOutputs[$key] = $variable.Value
 
-            if ($env:SYSTEM_TEAMPROJECTID) {
-                # Running in Azure Pipelines. Unfortunately, there's no good way to know which outputs are truly secrets
-                # because we have to set all output variables to "String" instead of "SecureString" or we will never get back a value.
-                Write-Host "##vso[task.setvariable variable=$key;issecret=true;]$($variable.Value)"
+            if ($CI) {
+                # Treat all ARM template output variables as secrets since "SecureString" variables do not set values.
+                # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below.
+                Write-Host "Setting variable '$key': ***"
+                Write-Host "##vso[task.setvariable variable=_$key;issecret=true;]$($variable.Value)"
+                Write-Host "##vso[task.setvariable variable=$key;]$($variable.Value)"
             } else {
                 Write-Host ($shellExportFormat -f $key, $variable.Value)
             }
@@ -292,6 +305,9 @@ Optional location where resources should be created. By default this is 'westus2
 .PARAMETER AdditionalParameters
 Optional key-value pairs of parameters to pass to the ARM template(s).
 
+.PARAMETER CI
+Indicates the script is run as part of a Continuous Integration / Continuous Deployment (CI/CD) build (only Azure Pipelines is currently supported).
+
 .PARAMETER Force
 Force creation of resources instead of being prompted.
 

eng/Remove-TestResources.ps1

@@ -11,23 +11,27 @@
 [CmdletBinding(DefaultParameterSetName = 'Default', SupportsShouldProcess = $true, ConfirmImpact = 'Medium')]
 param (
     # Limit $BaseName to enough characters to be under limit plus prefixes, and https://docs.microsoft.com/azure/architecture/best-practices/resource-naming.
-    [Parameter(Mandatory = $true, Position = 0)]
+    [Parameter(ParameterSetName = 'Default', Mandatory = $true, Position = 0)]
+    [Parameter(ParameterSetName = 'Default+Provisioner', Mandatory = $true, Position = 0)]
     [ValidatePattern('^[-a-zA-Z0-9\.\(\)_]{0,80}(?<=[a-zA-Z0-9\(\)])$')]
     [string] $BaseName,
 
-    # TODO: When https://github.com/Azure/azure-sdk-for-net/issues/9061 is resolved, default this to previously saved data.
-    [Parameter(Mandatory = $true)]
-    [string] $ServiceDirectory,
+    [Parameter(ParameterSetName = 'ResourceGroup', Mandatory = $true)]
+    [Parameter(ParameterSetName = 'ResourceGroup+Provisioner', Mandatory = $true)]
+    [string] $ResourceGroupName,
 
-    [Parameter(ParameterSetName = 'Provisioner', Mandatory = $true)]
+    [Parameter(ParameterSetName = 'Default+Provisioner', Mandatory = $true)]
+    [Parameter(ParameterSetName = 'ResourceGroup+Provisioner', Mandatory = $true)]
     [ValidateNotNullOrEmpty()]
     [string] $TenantId,
 
-    [Parameter(ParameterSetName = 'Provisioner', Mandatory = $true)]
+    [Parameter(ParameterSetName = 'Default+Provisioner', Mandatory = $true)]
+    [Parameter(ParameterSetName = 'ResourceGroup+Provisioner', Mandatory = $true)]
     [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')]
     [string] $ProvisionerApplicationId,
 
-    [Parameter(ParameterSetName = 'Provisioner', Mandatory = $true)]
+    [Parameter(ParameterSetName = 'Default+Provisioner', Mandatory = $true)]
+    [Parameter(ParameterSetName = 'ResourceGroup+Provisioner', Mandatory = $true)]
     [string] $ProvisionerApplicationSecret,
 
     [Parameter()]
@@ -69,12 +73,14 @@ if ($ProvisionerApplicationId) {
     }
 }
 
-# Format the resource group name based on resource group naming recommendations and limitations.
-$resourceGroupName = "rg-{0}-$baseName" -f ($ServiceDirectory -replace '[\\\/]', '-').Substring(0, [Math]::Min($ServiceDirectory.Length, 90 - $BaseName.Length - 4)).Trim('-')
+if (!$ResourceGroupName) {
+    # Format the resource group name like in New-TestResources.ps1.
+    $ResourceGroupName = "rg-$BaseName"
+}
 
-Log "Deleting resource group '${resourceGroupName}'"
-if (Remove-AzResourceGroup -Name "${resourceGroupName}" -Force:$Force) {
-    Write-Verbose "Successfully deleted resource group '${resourceGroupName}'"
+Log "Deleting resource group '$ResourceGroupName'"
+if (Remove-AzResourceGroup -Name "$ResourceGroupName" -Force:$Force) {
+    Write-Verbose "Successfully deleted resource group '$ResourceGroupName'"
 }
 
 $exitActions.Invoke()
@@ -84,15 +90,15 @@ $exitActions.Invoke()
 Deletes the resource group deployed for a service directory from Azure.
 
 .DESCRIPTION
-Removes a resource group and all its resources previously deployed for the specified $ServiceDirectory using New-TestResources.ps1. The $ServiceDirectory must match the previously specified value, e.g. 'keyvault' as shown in examples.
+Removes a resource group and all its resources previously deployed using New-TestResources.ps1.
 
 If you are not currently logged into an account in the Az PowerShell module, you will be asked to log in with Connect-AzAccount. Alternatively, you (or a build pipeline) can pass $ProvisionerApplicationId and $ProvisionerApplicationSecret to authenticate a service principal with access to create resources.
 
 .PARAMETER BaseName
 A name to use in the resource group and passed to the ARM template as 'baseName'.
 
-.PARAMETER ServiceDirectory
-A directory under 'sdk' in the repository root - optionally with subdirectories specified - in which to discover ARM templates named 'test-resources.json'.
+.PARAMETER ResourceGroupName
+The name of the resource group to delete.
 
 .PARAMETER TenantId
 The tenant ID of a service principal when a provisioner is specified.
@@ -110,10 +116,10 @@ Do not save credentials for the provisioner in the current process.
 Force creation of resources instead of being prompted.
 
 .EXAMPLE
-./Remove-Template.ps1 -BaseName uuid123 -ServiceDirectory keyvault -Force
+./Remove-Template.ps1 -BaseName uuid123 -Force
 
 Use the currently logged-in account to delete the resource group provisioned by the sdk/keyvault/test-resources.json ARM template.
 
 .LINK
-Remove-TestResources.ps1
+New-TestResources.ps1
 #>

eng/pipelines/templates/jobs/archetype-sdk-tests.yml

@@ -1,5 +1,6 @@
 parameters:
   PreSteps: []
+  PostSteps: []
   EnvVars: {}
   MaxParallel: 0
   BuildInParallel: true
@@ -53,6 +54,27 @@ jobs:
           packageType: sdk
           version: "$(DotNetCoreSDKVersion)"
 
+      # TODO: Remove when eng/New-TestResources.ps1 bootstraps the Az module (https://github.com/Azure/azure-sdk-for-net/issues/9130).
+      - pwsh: Install-Module -Name Az -Scope CurrentUser -Force
+        displayName: "Install Azure PowerShell module"
+
+        # Chomp the string literal to more easily read and update the New-TestResources parameters.
+        # TODO: Should use separate provisioner and test runner principals.
+      - pwsh: >
+          eng/New-TestResources.ps1
+          -BaseName 'Generated'
+          -ServiceDirectory '${{ parameters.ServiceDirectory }}'
+          -TenantId '$(aad-azure-sdk-test-tenant-id)'
+          -ProvisionerApplicationId '$(aad-azure-sdk-test-client-id)'
+          -ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret)'
+          -TestApplicationId '$(aad-azure-sdk-test-client-id)'
+          -TestApplicationSecret '$(aad-azure-sdk-test-client-secret)'
+          -DeleteAfterHours 24
+          -CI
+          -Force
+          -Verbose
+        displayName: "Create test resources"
+
       - script: dotnet test eng/service.proj --framework $(TestTargetFramework) --logger "trx;LogFileName=$(TestTargetFramework).trx" --logger:"console;verbosity=normal" /p:ServiceDirectory=${{ parameters.ServiceDirectory }} /p:IncludeSrc=false /p:IncludeSamples=false /p:BuildInParallel=${{ parameters.BuildInParallel }}
         displayName: "Build & Test (all tests for $(TestTargetFramework))"
         env:
@@ -62,6 +84,18 @@ jobs:
           AZURE_TEST_MODE: "None"
           ${{ insert }}: ${{ parameters.EnvVars }}
 
+      - pwsh: >
+          eng/Remove-TestResources.ps1
+          -ResourceGroupName "${env:AZURE_RESOURCEGROUP_NAME}"
+          -TenantId '$(aad-azure-sdk-test-tenant-id)'
+          -ProvisionerApplicationId '$(aad-azure-sdk-test-client-id)'
+          -ProvisionerApplicationSecret '$(aad-azure-sdk-test-client-secret)'
+          -Force
+          -Verbose
+        displayName: "Clean up test resources"
+        condition: ne(variables['AZURE_RESOURCEGROUP_NAME'], '')
+        continueOnError: true
+
       - task: PublishTestResults@2
         condition: always()
         displayName: "Publish Results ($(TestTargetFramework))"
@@ -70,3 +104,5 @@ jobs:
           testRunTitle: "$(OSName) $(TestTargetFramework)"
           testResultsFormat: "VSTest"
           mergeTestResults: true
+
+      - ${{ parameters.PostSteps }}

sdk/keyvault/tests.yml

@@ -10,11 +10,7 @@ resources:
 jobs:
 - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml
   parameters:
-    MaxParallel: 1
     ServiceDirectory: keyvault
     EnvVars:
-      AZURE_TENANT_ID: $(aad-azure-sdk-test-tenant-id)
-      AZURE_CLIENT_ID: $(net-keyvault-azure-client-id)
-      AZURE_CLIENT_SECRET: $(net-keyvault-azure-client-secret)
-      AZURE_KEYVAULT_URL: $(net-keyvault-azure-keyvault-url)
+      # Runs samples with live tests.
       AZURE_KEYVAULT_TEST_MODE: Live