Consolidate live test cloud/platform configuration, use stages, support weekly tests

eng/common/pipelines/templates/steps/bypass-local-dns.yml

@@ -3,4 +3,9 @@ steps:
     # https://github.com/actions/virtual-environments/issues/798
     - script: sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf
       displayName: Bypass local DNS server to workaround issue resolving cognitiveservices names
-      condition: and(succeededOrFailed(), eq(variables['OSVmImage'], 'ubuntu-18.04'))
+      condition: |
+        and(
+          succeededOrFailed(),
+          eq(variables['OSVmImage'], 'ubuntu-18.04'),
+          eq(variables['Container'], '')
+        )

eng/common/pipelines/templates/steps/verify-agent-os.yml

@@ -1,4 +1,4 @@
-parameters: 
+parameters:
   OSVmImage: $(OSVmImage)
 
 steps:

eng/pipelines/templates/jobs/archetype-sdk-tests-host.yml

@@ -0,0 +1,40 @@
+parameters:
+- name: AdditionalParameters
+  type: object
+- name: CloudConfig
+  type: object
+  default: {}
+- name: Platforms
+  type: object
+  default: {}
+- name: UsePlatformContainer
+  type: boolean
+  default: false
+
+jobs:
+  # There is no way to specify the container and/or target fields with a runtime value
+  # in a way that supports both containers and VMs as hosts, so this compile time
+  # switch is a workaround to support specifying host type.
+  - template: ./archetype-sdk-tests-jobs.yml
+    parameters:
+      UsePlatformContainer: false
+      Platforms:
+        ${{ each platform in parameters.Platforms }}:
+          ${{ if eq(platform.value.Container, '') }}:
+            ${{ platform.key }}: ${{ platform.value }}
+      CloudConfig: ${{ parameters.CloudConfig }}
+      ${{ each param in parameters.AdditionalParameters }}:
+        ${{ param.key }}: ${{ param.value }}
+  # Only include the template if we know there is a platform with a container,
+  # otherwise azure pipelines will populate a blank job with an empty matrix
+  - ${{ if eq(parameters.UsePlatformContainer, 'true') }}:
+    - template: ./archetype-sdk-tests-jobs.yml
+      parameters:
+        UsePlatformContainer: true
+        Platforms:
+          ${{ each platform in parameters.Platforms }}:
+            ${{ if ne(platform.value.Container, '') }}:
+              ${{ platform.key }}: ${{ platform.value }}
+        CloudConfig: ${{ parameters.CloudConfig }}
+        ${{ each param in parameters.AdditionalParameters }}:
+          ${{ param.key }}: ${{ param.value }}

eng/pipelines/templates/jobs/archetype-sdk-tests-jobs.yml

@@ -1,7 +1,4 @@
 parameters:
-- name: Platforms
-  type: object
-  default: []
 - name: PreSteps
   type: object
   default: []
@@ -29,86 +26,89 @@ parameters:
 - name: TestSetupSteps
   type: stepList
   default: []
-- name: CloudConfigurations
+- name: CloudConfig
+  type: object
+  default: {}
+- name: Platforms
   type: object
-  default:
-    AzureCloud:
-      SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
+  default: {}
+- name: UsePlatformContainer
+  type: boolean
+  default: false
 
 jobs:
-  - ${{ each cloudConfig in parameters.CloudConfigurations }}:
-    - ${{ each platform in parameters.Platforms }}:
-      - job:
-        condition: and(succeededOrFailed(), eq(eq(variables['Record'], 'true'), eq('${{ platform.TestMode }}', 'Record')))
-        displayName: ${{ platform.DisplayName }}_${{ cloudConfig.key }}
-        variables:
-          - template: ../variables/globals.yml
-          - name: OSVmImage
-            value: ${{ platform.OSVmImage }}
+  - job:
+    strategy:
+      maxParallel: ${{ parameters.MaxParallel }}
+      matrix: ${{ parameters.Platforms }}
 
-        timeoutInMinutes: ${{ parameters.TimeoutInMinutes }}
+    condition: and(succeededOrFailed(), eq(eq(variables['Record'], 'true'), eq('$(TestMode)', 'Record')))
+    variables:
+      - template: ../variables/globals.yml
+      # ServiceDirectory references must get passed down by the caller as variable references
+      - name: ServiceDirectory
+        value: ${{ parameters.ServiceDirectory }}
+      - name: AdditionalTestArguments
+        value: ''
+      - name: ArmTemplateParameters
+        value: '@{}'
 
-        pool:
-          vmImage: $(OSVmImage)
-        ${{ if platform.Container }}:
-          container: ${{ platform.Container }}
+    timeoutInMinutes: ${{ parameters.TimeoutInMinutes }}
 
-        steps:
-          - ${{ if platform.PreSteps }}:
-            - ${{ platform.PreSteps }}
+    pool:
+      vmImage: $(OSVmImage)
 
-          - ${{ parameters.PreSteps }}
+    ${{ if eq(parameters.UsePlatformContainer, 'true') }}:
+      container: $[ variables['Container'] ]
 
-          - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml
+    steps:
+      - ${{ parameters.PreSteps }}
 
-          - ${{ each step in parameters.TestSetupSteps }}:
-            - ${{ each pair in step }}:
-                ${{ pair.key }}: ${{ pair.value }}
+      - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml
 
-          - template: /eng/common/TestResources/deploy-test-resources.yml
-            parameters:
-              Location: ${{ coalesce(parameters.Location, cloudConfig.value.Location, 'westus2') }}
-              ServiceDirectory: '${{ parameters.ServiceDirectory }}'
-              SubscriptionConfiguration: ${{ cloudConfig.value.SubscriptionConfiguration }}
-              ArmTemplateParameters: ${{ cloudConfig.value.ArmTemplateParameters }}
+      - ${{ each step in parameters.TestSetupSteps }}:
+        - ${{ each pair in step }}:
+            ${{ pair.key }}: ${{ pair.value }}
 
-          - script: >
-              dotnet test eng/service.proj
-              --framework ${{ platform.TestTargetFramework }}
-              --filter "TestCategory!=Manually"
-              --logger "trx"
-              --logger:"console;verbosity=normal"
-              /p:ServiceDirectory=${{ parameters.ServiceDirectory }}
-              /p:IncludeSrc=false /p:IncludeSamples=false /p:IncludePerf=false /p:IncludeStress=false
-              /p:BuildInParallel=${{ parameters.BuildInParallel }}
-              ${{ platform.AdditionalTestArguments }}
+      - template: /eng/common/TestResources/deploy-test-resources.yml
+        parameters:
+          Location: ${{ coalesce(parameters.Location, parameters.CloudConfig.Location, 'westus2') }}
+          ServiceDirectory: '${{ parameters.ServiceDirectory }}'
+          SubscriptionConfiguration: ${{ parameters.CloudConfig.SubscriptionConfiguration }}
+          ArmTemplateParameters: $(ArmTemplateParameters)
 
-            displayName: "Build & Test (all tests for ${{ platform.TestTargetFramework }})"
-            env:
-              DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
-              DOTNET_CLI_TELEMETRY_OPTOUT: 1
-              DOTNET_MULTILEVEL_LOOKUP: 0
-              AZURE_TEST_MODE: "${{ coalesce(platform.TestMode, 'None') }}"
-              ${{ each var in parameters.EnvVars }}:
-                ${{ var.key }}: ${{ var.value }}
-              ${{ each var in cloudConfig.value.EnvVars }}:
-                ${{ var.key }}: ${{ var.value }}
+      - pwsh: >
+          dotnet test eng/service.proj
+          --framework $(TestTargetFramework)
+          --filter "TestCategory!=Manually"
+          --logger "trx"
+          --logger:"console;verbosity=normal"
+          /p:ServiceDirectory=${{ parameters.ServiceDirectory }}
+          /p:IncludeSrc=false /p:IncludeSamples=false /p:IncludePerf=false /p:IncludeStress=false
+          /p:BuildInParallel=${{ parameters.BuildInParallel }}
+          $(AdditionalTestArguments)
 
-          - template: /eng/common/TestResources/remove-test-resources.yml
-            parameters:
-              ServiceDirectory: '${{ parameters.ServiceDirectory }}'
-              SubscriptionConfiguration: ${{ cloudConfig.value.SubscriptionConfiguration }}
+        displayName: "Build & Test (all tests for $(TestTargetFramework))"
+        env:
+          DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
+          DOTNET_CLI_TELEMETRY_OPTOUT: 1
+          DOTNET_MULTILEVEL_LOOKUP: 0
+          AZURE_TEST_MODE: "${{ coalesce(variables['TestMode'], 'None') }}"
+          ${{ each var in parameters.EnvVars }}:
+            ${{ var.key }}: ${{ var.value }}
 
-          - task: PublishTestResults@2
-            condition: always()
-            displayName: "Publish Results (${{ platform.TestTargetFramework }})"
-            inputs:
-              testResultsFiles: "**/*.trx"
-              testRunTitle: "$(OSName) ${{ platform.TestTargetFramework }}"
-              testResultsFormat: "VSTest"
-              mergeTestResults: true
+      - template: /eng/common/TestResources/remove-test-resources.yml
+        parameters:
+          ServiceDirectory: '${{ parameters.ServiceDirectory }}'
+          SubscriptionConfiguration: ${{ parameters.CloudConfig.SubscriptionConfiguration }}
 
-          - ${{ parameters.PostSteps }}
+      - task: PublishTestResults@2
+        condition: always()
+        displayName: "Publish Results ($(TestTargetFramework))"
+        inputs:
+          testResultsFiles: "**/*.trx"
+          testRunTitle: "$(TestTargetFramework)"
+          testResultsFormat: "VSTest"
+          mergeTestResults: true
 
-          - ${{ if platform.PostSteps }}:
-            - ${{ platform.PostSteps }}
+      - ${{ parameters.PostSteps }}

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -53,4 +53,4 @@ stages:
         TestPipeline: ${{ parameters.TestPipeline }}
         ArtifactName: packages
         TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }}
-        TargetDocRepoName: ${{ parameters.TargetDocRepoName }} 
+        TargetDocRepoName: ${{ parameters.TargetDocRepoName }}