Release pipeline and Integration test refactor

.azure-devops/create-release.yml

@@ -2,109 +2,148 @@ pr: none
 
 trigger: none
 
-jobs:
-
-  # track deployment in environments
-- deployment: 'StageGitHub'
-  displayName: 'Stage CLI extension on GitHub'
-  environment: 'production'
-
-- job: 'Build_Publish_Azure_IoT_CLI_Extension'
-  pool:
-    vmImage: 'Ubuntu-16.04'
-
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.6.x'
-      architecture: 'x64'
-
-  - template: templates/setup-ci-machine.yml
-
-  - template: templates/build-publish-azure-iot-cli-extension.yml
-
-- job: 'Build_Publish_Azure_CLI_Test_SDK'
-  pool:
-    vmImage: 'Ubuntu-16.04'
-
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.6.x'
-      architecture: 'x64'
-
-  - template: templates/setup-ci-machine.yml
-
-  - template: templates/build-publish-azure-cli-test-sdk.yml
-
-- job: 'Run_Tests'
-  dependsOn : Build_Publish_Azure_CLI_Test_SDK
-  pool:
-    vmImage: 'Ubuntu-16.04'
-
-  steps:
-  - template: templates/run-tests.yml
-    parameters:
-      pythonVersion: '3.6.x'
-      runUnitTestsOnly: 'true'
-
-  - script: 'pip install .'
-    displayName: 'Install the whl locally'
-    workingDirectory: '.'
-
-  - task: PythonScript@0
-    displayName : 'setupVersion'
-    name: 'setupVersion'
-    inputs:
-      scriptSource: 'inline'
-      script: |
-        from azext_iot.constants import VERSION
-        print("Version is " + VERSION)
-        print("##vso[task.setvariable variable=CLIVersion;isOutput=true]"+VERSION)
-        print("##vso[task.setvariable variable=ReleaseTitle;isOutput=true]"+"azure-iot "+VERSION)
-
-- job: 'Calculate_Sha_And_Create_Release'
-  dependsOn : Run_Tests
-  pool:
-    vmImage: 'vs2017-win2016'
-  variables:
-    CLIVersion: $[dependencies.Run_Tests.outputs['setupVersion.CLIVersion']]
-    ReleaseTitle: $[dependencies.Run_Tests.outputs['setupVersion.ReleaseTitle']]
-
-  steps:
-  - task: DownloadBuildArtifacts@0
-    displayName : 'Download Extension wheel from Build Artifacts'
-    inputs:
-      buildType: 'current'
-      downloadType: 'single'
-      artifactName: 'azure-iot'
-      downloadPath: '$(System.ArtifactsDirectory)/extension'
-
-  - task: PowerShell@2
-    displayName: 'Calculate sha for downloaded extension'
-    inputs:
-      targetType: 'inline'
-      script: |
-        $extensions = Get-ChildItem -Filter "*.whl" -Recurse | Select-Object FullName
-        Foreach ($extension in $extensions)
-        {
-            Write-Host "calculating sha256 for " $extension.FullName   
-            (Get-Filehash -Path $extension.Fullname -Algorithm SHA256).Hash.ToLower()
-        }
-        Write-Host "done"
-      workingDirectory: '$(System.ArtifactsDirectory)/extension'
-
-  - task: GitHubRelease@0
-    inputs:
-      gitHubConnection: AzIoTCLIGitHub
-      repositoryName: $(Build.Repository.Name)
-      action: 'create'
-      target: '$(Build.SourceVersion)'
-      tagSource: manual
-      tag: 'v$(CLIVersion)'
-      title: $(ReleaseTitle)
-      assets: '$(System.ArtifactsDirectory)/extension/**/*.whl' 
-      assetUploadMode: 'replace'
-      isDraft: true
-      isPreRelease: false
-      addChangeLog: false
+variables:
+  pythonVersion: '3.6.x'
+  architecture: 'x64'
+
+stages:
+  - stage: 'build'
+    displayName: 'Build and Publish Artifacts'
+    jobs:
+
+    - job: 'Build_Publish_Azure_IoT_CLI_Extension'
+      pool:
+        vmImage: 'Ubuntu-16.04'
+
+      steps:
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: $pythonVersion
+          architecture: $architecture
+
+      - template: templates/setup-ci-machine.yml
+
+      - template: templates/build-publish-azure-iot-cli-extension.yml
+
+    - job: 'Build_Publish_Azure_CLI_Test_SDK'
+      pool:
+        vmImage: 'Ubuntu-16.04'
+
+      steps:
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: $pythonVersion
+          architecture: $architecture
+
+      - template: templates/setup-ci-machine.yml
+
+      - template: templates/build-publish-azure-cli-test-sdk.yml
+
+  - stage: 'test'
+    displayName: 'Run tests'
+    pool:
+      vmImage: 'Ubuntu-16.04'
+    dependsOn: build
+    jobs:
+    - job: 'testCentral'
+      displayName: 'Test IoT Central'
+      steps:
+      - template: templates/run-tests.yml
+        parameters:
+          path: 'azext_iot/tests/central'
+          name: 'IoT-Central'
+
+    - job: 'testADT'
+      displayName: 'Test Azure DigitalTwins'
+      steps:
+      - template: templates/run-tests.yml
+        parameters:
+          path: 'azext_iot/tests/digitaltwins'
+          name: 'Azure-DigitalTwins'
+
+    - job: 'testDPS'
+      displayName: 'Test DPS'
+      steps:
+      - template: templates/run-tests.yml
+        parameters:
+          path: 'azext_iot/tests/dps'
+          name: 'Device-Provisioning-Service'
+
+    - job: 'testHub'
+      displayName: 'Test IoT Hub'
+      steps:
+      - template: templates/run-tests.yml
+        parameters:
+          path: 'azext_iot/tests/iothub'
+          name: 'IoT-Hub'
+
+    - job: 'unitTests'
+      displayName: 'Unit tests and code coverage'
+      steps:
+      - template: templates/run-tests.yml
+        parameters:
+          runIntTests: 'false'
+          runUnitTests: 'true'
+
+    - job: 'recordVersion'
+      displayName: 'Install and verify version'
+      steps:
+      - template: templates/setup-dev-test-env.yml
+        parameters:
+          pythonVersion: $pythonVersion
+          architecture: $architecture
+
+      - template: templates/install-and-record-version.yml
+
+  - stage: 'release'
+    displayName: 'Stage GitHub release'
+    dependsOn: [build, test]
+    jobs:
+    - deployment: 'StageGitHub'
+      displayName: 'Stage CLI extension on GitHub'
+      environment: 'production'
+
+    - job: 'Calculate_Sha_And_Create_Release'
+      pool:
+        vmImage: 'vs2017-win2016'
+      variables:
+        CLIVersion: $[ stageDependencies.test.recordVersion.outputs['setupVersion.CLIVersion'] ]
+        ReleaseTitle: $[ stageDependencies.test.recordVersion.outputs['setupVersion.ReleaseTitle'] ]
+
+      steps:
+      - task: DownloadBuildArtifacts@0
+        displayName : 'Download Extension wheel from Build Artifacts'
+        inputs:
+          buildType: 'current'
+          downloadType: 'single'
+          artifactName: 'azure-iot'
+          downloadPath: '$(System.ArtifactsDirectory)/extension'
+
+      - task: PowerShell@2
+        displayName: 'Calculate sha for downloaded extension'
+        inputs:
+          targetType: 'inline'
+          script: |
+            $extensions = Get-ChildItem -Filter "*.whl" -Recurse | Select-Object FullName
+            Foreach ($extension in $extensions)
+            {
+                Write-Host "calculating sha256 for " $extension.FullName   
+                (Get-Filehash -Path $extension.Fullname -Algorithm SHA256).Hash.ToLower()
+            }
+            Write-Host "done"
+          workingDirectory: '$(System.ArtifactsDirectory)/extension'
+
+      - task: GitHubRelease@0
+        inputs:
+          gitHubConnection: AzIoTCLIGitHub
+          repositoryName: $(Build.Repository.Name)
+          action: 'create'
+          target: '$(Build.SourceVersion)'
+          tagSource: manual
+          tag: 'v$(CLIVersion)'
+          title: $(ReleaseTitle)
+          assets: '$(System.ArtifactsDirectory)/extension/**/*.whl' 
+          assetUploadMode: 'replace'
+          isDraft: true
+          isPreRelease: false
+          addChangeLog: false

.azure-devops/merge.yml

@@ -58,6 +58,8 @@ jobs:
   - template: templates/run-tests.yml
     parameters:
       pythonVersion: '$(python.version)'
+      runUnitTests: 'true'
+      runIntTests: 'false'
 
 - job: 'run_unit_tests_macOs'
   dependsOn: ['build_and_publish_azure_iot_cli_ext', 'build_and_publish_azure_cli_test_sdk']
@@ -68,6 +70,8 @@ jobs:
   - template: templates/run-tests.yml
     parameters:
       pythonVersion: '3.8.x'
+      runUnitTests: 'true'
+      runIntTests: 'false'
 
 - job: 'run_unit_tests_windows'
   dependsOn : [ 'build_and_publish_azure_iot_cli_ext', 'build_and_publish_azure_cli_test_sdk']
@@ -83,6 +87,8 @@ jobs:
   - template: templates/run-tests.yml
     parameters:
       pythonVersion: '3.8.x'
+      runUnitTests: 'true'
+      runIntTests: 'false'
 
 - job: 'run_style_check'
   dependsOn: ['build_and_publish_azure_iot_cli_ext', 'build_and_publish_azure_cli_test_sdk']

.azure-devops/templates/install-and-record-version.yml

@@ -0,0 +1,15 @@
+steps:
+- script: 'pip install .'
+  displayName: 'Install the whl locally'
+  workingDirectory: '.'
+
+- task: PythonScript@0
+  displayName : 'setupVersion'
+  name: 'setupVersion'
+  inputs:
+    scriptSource: 'inline'
+    script: |
+      from azext_iot.constants import VERSION
+      print("Version is " + VERSION)
+      print("##vso[task.setvariable variable=CLIVersion;isOutput=true]"+VERSION)
+      print("##vso[task.setvariable variable=ReleaseTitle;isOutput=true]"+"azure-iot "+VERSION)

.azure-devops/templates/run-tests.yml

@@ -1,57 +1,46 @@
 parameters:
-  pythonVersion: ''
-  runUnitTestsOnly: 'true'
+  pythonVersion: '3.6.x'
+  architecture: 'x64'
+  runUnitTests: 'false'
+  runIntTests: 'true'
   runWithAzureCliReleased: 'true'
+  path: 'azext_iot/tests'
+  name: 'All'
 
 steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: ${{ parameters.pythonVersion }}
-      architecture: 'x64'
-
-  - ${{ if eq(parameters.runWithAzureCliReleased, 'false') }}:
-    - template: install-azure-cli-edge.yml
-
-  - ${{ if eq(parameters.runWithAzureCliReleased, 'true') }}:
-    - template: install-azure-cli-released.yml
-
-  - template: download-install-local-azure-test-sdk.yml
-
-  - template: setup-ci-machine.yml
-
-  - template: download-install-local-azure-iot-cli-extension.yml
-
-  - template: set-pythonpath.yml
+  - template: setup-dev-test-env.yml
+    parameters:
+      architecture: ${{ parameters.architecture }}
+      pythonVersion: ${{ parameters.pythonVersion }}
+      runWithAzureCliReleased: ${{ parameters.runWithAzureCliReleased }}
 
   - template: set-testenv-sentinel.yml
 
-  - ${{ if eq(parameters.runUnitTestsOnly, 'false') }}:
-
+  - ${{ if eq(parameters.runIntTests, 'true') }}:
     - task: AzureCLI@2
-      displayName: 'Execute full IoT extension test run'
+      displayName: '${{ parameters.name }} integration tests'
       inputs:
         azureSubscription: AzIoTCLIService
         scriptType: bash
         scriptLocation: inlineScript
-        inlineScript: pytest -vv -r azext_iot/tests/ --cov=azext_iot --cov-config .coveragerc --junitxml=junit/test-iotext-full-results.xml --cov-report=xml
-
-  - ${{ if eq(parameters.runUnitTestsOnly, 'true') }}:
+        inlineScript: pytest -vv ${{ parameters.path }} -k "_int"  --junitxml=junit/test-iotext-int-${{ parameters.name }}.xml
 
-    - script: pytest -vv -r azext_iot/tests/ -k "_unit" --cov=azext_iot --cov-config .coveragerc --junitxml=junit/test-iotext-unit-results.xml --cov-report=xml
-      displayName: 'Execute IoT extension unit tests'
-
-  - task: PublishCodeCoverageResults@1
-    inputs:
-      codeCoverageTool: 'cobertura'
-      summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
-      reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov'
-      additionalCodeCoverageFiles: '$(System.DefaultWorkingDirectory)/htmlcov/**/*.*'
+  - ${{ if eq(parameters.runUnitTests, 'true') }}:
+    - script: pytest -vv ${{ parameters.path }} -k "_unit" --cov=azext_iot --cov-config .coveragerc --junitxml=junit/test-iotext-unit-${{ parameters.name }}.xml --cov-report=xml
+      displayName: '${{ parameters.name }} unit tests'
+
+    - task: PublishCodeCoverageResults@1
+      inputs:
+        codeCoverageTool: 'cobertura'
+        summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
+        reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov'
+        additionalCodeCoverageFiles: '$(System.DefaultWorkingDirectory)/htmlcov/**/*.*'
 
   - task: PublishTestResults@2
     condition: succeededOrFailed()
     displayName: 'Publish Test Results'
     inputs:
       testResultsFormat: 'JUnit'
       testResultsFiles: '**/test-*.xml'
-      testRunTitle: 'Publish test results for Python ${{ parameters.pythonVersion }} on OS $(Agent.OS)'
+      testRunTitle: 'Publish ${{ parameters.name }} test results for Python ${{ parameters.pythonVersion }} on OS $(Agent.OS)'
       searchFolder: '$(System.DefaultWorkingDirectory)'