add resource group provisioniner and deprovisioner (#9798)

* add resource group provisioniner and deprovisioner
* with tags
* use condition for allocating resource group, configure storage to skip allocationg an RG, collapse KV configuration now that all can run in parallel
* keep KV tests parallelized per-package
* remove extra parens
* correct template expression syntax
* correct syntax for second conditional expression
* set resource group allocation for live tests for services whose live test migration status is known
* servicebus: eliminate old credentials
* Event Hubs is using a static resource today
* add identity configuration
* always remove the resource group when it has been created successfully
* remove AZURE_CLIENT_OID from service bus tests.yml

Author: danieljurek

eng/pipelines/templates/jobs/archetype-sdk-tests.yml

@@ -9,6 +9,7 @@ parameters:
   InjectedPackages: ''
   BuildDocs: true
   JobName: Test
+  AllocateResourceGroup: 'true'
   Matrix:
     Linux_Python35:
       OSName: 'Linux'
@@ -47,6 +48,21 @@ jobs:
       vmImage: $(OSVmImage)
 
     steps:
+      - ${{ if eq(parameters['AllocateResourceGroup'], 'true') }}:
+        - pwsh: |
+            $BaseName = 't' + (New-Guid).ToString('n').Substring(0, 16)
+            $RgName = "rg-${{ parameters.ServiceDirectory }}-$($BaseName)"
+            $ResourceGroupExpiration = (Get-Date).ToUniversalTime().AddHours(24).ToString("o")
+            Write-Host "Creating Resource Group: $($RgName)"
+            Write-Host "##vso[task.setvariable variable=AZURE_RESOURCEGROUP_NAME]$RgName"
+
+            az login --service-principal -u $(provisioner-aad-id) -p $(provisioner-aad-secret) --tenant $(provisioner-aad-tenant)
+            az account set --subscription $(provisioner-subscription)
+
+            az group create --location westus2 --name $RgName --tags DeleteAfter=$ResourceGroupExpiration BuildId='$(Build.BuildId)' BuildJob='$(Agent.JobName)' BuildNumber='$(Build.BuildNumber)' BuildReason='$(Build.Reason)'
+            az group wait --created --resource-group $RgName
+          displayName: Create resource group and environment variable for tests
+
       - template: ../steps/build-test.yml
         parameters:
           BuildTargetingString: ${{ parameters.BuildTargetingString }}
@@ -63,4 +79,10 @@ jobs:
           TestMarkArgument: ${{ parameters.TestMarkArgument }}
           InjectedPackages: ${{ parameters.InjectedPackages }}
           BuildDocs: ${{parameters.BuildDocs}}
-  
+
+      - ${{ if eq(parameters['AllocateResourceGroup'], 'true') }}:
+        - pwsh: |
+            az group delete --resource-group $env:AZURE_RESOURCEGROUP_NAME -y --no-wait
+          displayName: Delete resource group
+          condition: ne(variables['AZURE_RESOURCEGROUP_NAME'], '')
+

sdk/appconfiguration/tests.yml

@@ -10,6 +10,7 @@ resources:
 jobs:
   - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml
     parameters:
+      AllocateResourceGroup: 'false'
       BuildTargetingString: $(BuildTargetingString)
       ServiceDirectory: appconfiguration
       EnvVars:

sdk/cosmos/tests.yml

@@ -10,6 +10,7 @@ resources:
 jobs:
   - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml
     parameters:
+      AllocateResourceGroup: 'false'
       MaxParallel: 1
       ServiceDirectory: cosmos
       TestMarkArgument: not globaldb

sdk/eventhub/tests.yml

@@ -10,6 +10,7 @@ resources:
 jobs:
   - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml
     parameters:
+      AllocateResourceGroup: 'false'
       ServiceDirectory: eventhub
       Matrix:
         Linux_Python35:

sdk/identity/tests.yml

@@ -10,6 +10,7 @@ resources:
 jobs:
   - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml
     parameters:
+      AllocateResourceGroup: 'false'
       ServiceDirectory: identity
       EnvVars:
         AZURE_CLIENT_ID: $(python-identity-client-id)

sdk/servicebus/tests.yml

@@ -12,13 +12,10 @@ jobs:
     parameters:
       ServiceDirectory: servicebus
       EnvVars:
-        SERVICE_BUS_HOSTNAME: $(python-sb-livetest-service-bus-hostname)
-        SERVICE_BUS_SAS_POLICY: $(python-sb-livetest-service-sas-policy)
-        SERVICE_BUS_SAS_KEY: $(python-sb-livetest-service-sas-key)
-        SERVICE_BUS_CONNECTION_STR: $(python-sb-livetest-service-connection-str)
-        SERVICE_BUS_CONNECTION_STR_RO: $(python-sb-livetest-service-connection-str-ro)
-        SERVICE_BUS_CONNECTION_STR_WO: $(python-sb-livetest-service-connection-str-wo)
-        SERVICE_BUS_CONNECTION_STR_ENTITY: $(python-sb-livetest-service-connection-entity)
+        AZURE_SUBSCRIPTION_ID: $(azure-subscription-id)
+        AZURE_TENANT_ID: $(aad-azure-sdk-test-tenant-id)
+        AZURE_CLIENT_ID: $(aad-azure-sdk-test-client-id)
+        AZURE_CLIENT_SECRET: $(aad-azure-sdk-test-client-secret)
         AZURE_TEST_RUN_LIVE: 'true'
       Matrix:
         Linux_Python35:

sdk/storage/tests_invoke.yml

@@ -5,6 +5,7 @@ parameters:
 jobs:
   - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml
     parameters:
+      AllocateResourceGroup: 'false'
       BuildTargetingString: azure-storage-blob
       JobName: Test_Blob
       ServiceDirectory: storage
@@ -14,6 +15,7 @@ jobs:
 
   - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml
     parameters:
+      AllocateResourceGroup: 'false'
       BuildTargetingString: azure-storage-queue
       JobName: Test_Queue
       ServiceDirectory: storage
@@ -22,6 +24,7 @@ jobs:
 
   - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml
     parameters:
+      AllocateResourceGroup: 'false'
       MaxParallel: 1
       BuildTargetingString: azure-storage-file-datalake
       JobName: Test_Datalake
@@ -31,6 +34,7 @@ jobs:
 
   - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml
     parameters:
+      AllocateResourceGroup: 'false'
       BuildTargetingString: azure-storage-file-share
       JobName: Test_FileShares
       ServiceDirectory: storage