Release pipeline and Integration test refactor

.azure-devops/create-release.yml

@@ -2,109 +2,135 @@ pr: none
 
 trigger: none
 
-jobs:
-
-  # track deployment in environments
-- deployment: 'StageGitHub'
-  displayName: 'Stage CLI extension on GitHub'
-  environment: 'production'
-
-- job: 'Build_Publish_Azure_IoT_CLI_Extension'
-  pool:
-    vmImage: 'Ubuntu-16.04'
-
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.6.x'
-      architecture: 'x64'
-
-  - template: templates/setup-ci-machine.yml
-
-  - template: templates/build-publish-azure-iot-cli-extension.yml
-
-- job: 'Build_Publish_Azure_CLI_Test_SDK'
-  pool:
-    vmImage: 'Ubuntu-16.04'
-
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.6.x'
-      architecture: 'x64'
-
-  - template: templates/setup-ci-machine.yml
-
-  - template: templates/build-publish-azure-cli-test-sdk.yml
-
-- job: 'Run_Tests'
-  dependsOn : Build_Publish_Azure_CLI_Test_SDK
-  pool:
-    vmImage: 'Ubuntu-16.04'
-
-  steps:
-  - template: templates/run-tests.yml
-    parameters:
-      pythonVersion: '3.6.x'
-      runUnitTestsOnly: 'true'
-
-  - script: 'pip install .'
-    displayName: 'Install the whl locally'
-    workingDirectory: '.'
-
-  - task: PythonScript@0
-    displayName : 'setupVersion'
-    name: 'setupVersion'
-    inputs:
-      scriptSource: 'inline'
-      script: |
-        from azext_iot.constants import VERSION
-        print("Version is " + VERSION)
-        print("##vso[task.setvariable variable=CLIVersion;isOutput=true]"+VERSION)
-        print("##vso[task.setvariable variable=ReleaseTitle;isOutput=true]"+"azure-iot "+VERSION)
-
-- job: 'Calculate_Sha_And_Create_Release'
-  dependsOn : Run_Tests
-  pool:
-    vmImage: 'vs2017-win2016'
-  variables:
-    CLIVersion: $[dependencies.Run_Tests.outputs['setupVersion.CLIVersion']]
-    ReleaseTitle: $[dependencies.Run_Tests.outputs['setupVersion.ReleaseTitle']]
-
-  steps:
-  - task: DownloadBuildArtifacts@0
-    displayName : 'Download Extension wheel from Build Artifacts'
-    inputs:
-      buildType: 'current'
-      downloadType: 'single'
-      artifactName: 'azure-iot'
-      downloadPath: '$(System.ArtifactsDirectory)/extension'
-
-  - task: PowerShell@2
-    displayName: 'Calculate sha for downloaded extension'
-    inputs:
-      targetType: 'inline'
-      script: |
-        $extensions = Get-ChildItem -Filter "*.whl" -Recurse | Select-Object FullName
-        Foreach ($extension in $extensions)
-        {
-            Write-Host "calculating sha256 for " $extension.FullName   
-            (Get-Filehash -Path $extension.Fullname -Algorithm SHA256).Hash.ToLower()
-        }
-        Write-Host "done"
-      workingDirectory: '$(System.ArtifactsDirectory)/extension'
-
-  - task: GitHubRelease@0
-    inputs:
-      gitHubConnection: AzIoTCLIGitHub
-      repositoryName: $(Build.Repository.Name)
-      action: 'create'
-      target: '$(Build.SourceVersion)'
-      tagSource: manual
-      tag: 'v$(CLIVersion)'
-      title: $(ReleaseTitle)
-      assets: '$(System.ArtifactsDirectory)/extension/**/*.whl' 
-      assetUploadMode: 'replace'
-      isDraft: true
-      isPreRelease: false
-      addChangeLog: false
+stages:
+  - stage: 'build'
+    displayName: 'Build and Publish Artifacts'
+    jobs:
+
+    - job: 'Build_Publish_Azure_IoT_CLI_Extension'
+      pool:
+        vmImage: 'Ubuntu-16.04'
+
+      steps:
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: '3.6.x'
+          architecture: 'x64'
+
+      - template: templates/setup-ci-machine.yml
+
+      - template: templates/build-publish-azure-iot-cli-extension.yml
+
+    - job: 'Build_Publish_Azure_CLI_Test_SDK'
+      pool:
+        vmImage: 'Ubuntu-16.04'
+
+      steps:
+      - task: UsePythonVersion@0
+        inputs:
+          versionSpec: '3.6.x'
+          architecture: 'x64'
+
+      - template: templates/setup-ci-machine.yml
+
+      - template: templates/build-publish-azure-cli-test-sdk.yml
+
+  - stage: 'test'
+    displayName: 'Run tests'
+    pool:
+      vmImage: 'Ubuntu-16.04'
+    dependsOn: build
+    jobs:
+    - job: 'testCentral'
+      displayName: 'Test IoT Central'
+      steps:
+      - template: templates/run-tests.yml
+        parameters:
+          pythonVersion: '3.6.x'
+          path: 'azext_iot/tests/central'
+          name: 'IoT-Central'
+    - job: 'testADT'
+      displayName: 'Test Azure DigitalTwins'
+      steps:
+      - template: templates/run-tests.yml
+        parameters:
+          pythonVersion: '3.6.x'
+          path: 'azext_iot/tests/digitaltwins'
+          name: 'Azure-DigitalTwins'
+    - job: 'testDPS'
+      displayName: 'Test DPS'
+      steps:
+      - template: templates/run-tests.yml
+        parameters:
+          pythonVersion: '3.6.x'
+          path: 'azext_iot/tests/dps'
+          name: 'Device-Provisioning-Service'
+    - job: 'testHub'
+      displayName: 'Test IoT Hub'
+      steps:
+      - template: templates/run-tests.yml
+        parameters:
+          pythonVersion: '3.6.x'
+          path: 'azext_iot/tests/iothub'
+          name: 'IoT-Hub'
+    - job: 'unitTests'
+      displayName: 'Unit tests and code coverage'
+      steps:
+      - template: templates/run-tests.yml
+        parameters:
+          pythonVersion: '3.6.x'
+          runIntTests: 'false'
+          runUnitTests: 'true'
+
+  - stage: 'release'
+    displayName: 'Stage GitHub release'
+    dependsOn: 'test'
+    jobs:
+    - deployment: 'StageGitHub'
+      displayName: 'Stage CLI extension on GitHub'
+      environment: 'production'
+
+    - job: 'Calculate_Sha_And_Create_Release'
+      pool:
+        vmImage: 'vs2017-win2016'
+      variables:
+        CLIVersion: $[dependencies.Run_Tests.outputs['setupVersion.CLIVersion']]
+        ReleaseTitle: $[dependencies.Run_Tests.outputs['setupVersion.ReleaseTitle']]
+
+      steps:
+      - task: DownloadBuildArtifacts@0
+        displayName : 'Download Extension wheel from Build Artifacts'
+        inputs:
+          buildType: 'current'
+          downloadType: 'single'
+          artifactName: 'azure-iot'
+          downloadPath: '$(System.ArtifactsDirectory)/extension'
+
+      - task: PowerShell@2
+        displayName: 'Calculate sha for downloaded extension'
+        inputs:
+          targetType: 'inline'
+          script: |
+            $extensions = Get-ChildItem -Filter "*.whl" -Recurse | Select-Object FullName
+            Foreach ($extension in $extensions)
+            {
+                Write-Host "calculating sha256 for " $extension.FullName   
+                (Get-Filehash -Path $extension.Fullname -Algorithm SHA256).Hash.ToLower()
+            }
+            Write-Host "done"
+          workingDirectory: '$(System.ArtifactsDirectory)/extension'
+
+      - task: GitHubRelease@0
+        inputs:
+          gitHubConnection: AzIoTCLIGitHub
+          repositoryName: $(Build.Repository.Name)
+          action: 'create'
+          target: '$(Build.SourceVersion)'
+          tagSource: manual
+          tag: 'v$(CLIVersion)'
+          title: $(ReleaseTitle)
+          assets: '$(System.ArtifactsDirectory)/extension/**/*.whl' 
+          assetUploadMode: 'replace'
+          isDraft: true
+          isPreRelease: false
+          addChangeLog: false

.azure-devops/templates/run-tests.yml

@@ -1,7 +1,10 @@
 parameters:
   pythonVersion: ''
-  runUnitTestsOnly: 'true'
+  runUnitTests: 'false'
+  runIntTests: 'true'
   runWithAzureCliReleased: 'true'
+  path: 'azext_iot/tests'
+  name: 'All'
 
 steps:
   - task: UsePythonVersion@0
@@ -25,33 +28,46 @@ steps:
 
   - template: set-testenv-sentinel.yml
 
-  - ${{ if eq(parameters.runUnitTestsOnly, 'false') }}:
-
+  - ${{ if eq(parameters.runIntTests, 'true') }}:
     - task: AzureCLI@2
-      displayName: 'Execute full IoT extension test run'
+      displayName: '${{ parameters.name }} integration tests'
       inputs:
         azureSubscription: AzIoTCLIService
         scriptType: bash
         scriptLocation: inlineScript
-        inlineScript: pytest -vv -r azext_iot/tests/ --cov=azext_iot --cov-config .coveragerc --junitxml=junit/test-iotext-full-results.xml --cov-report=xml
-
-  - ${{ if eq(parameters.runUnitTestsOnly, 'true') }}:
+        inlineScript: pytest -vv ${{ parameters.path }} -k "_int"  --junitxml=junit/test-iotext-int-${{ parameters.name }}.xml
 
-    - script: pytest -vv -r azext_iot/tests/ -k "_unit" --cov=azext_iot --cov-config .coveragerc --junitxml=junit/test-iotext-unit-results.xml --cov-report=xml
-      displayName: 'Execute IoT extension unit tests'
-
-  - task: PublishCodeCoverageResults@1
-    inputs:
-      codeCoverageTool: 'cobertura'
-      summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
-      reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov'
-      additionalCodeCoverageFiles: '$(System.DefaultWorkingDirectory)/htmlcov/**/*.*'
+  - ${{ if eq(parameters.runUnitTests, 'true') }}:
+    - script: pytest -vv ${{ parameters.path }} -k "_unit" --cov=azext_iot --cov-config .coveragerc --junitxml=junit/test-iotext-unit-${{ parameters.name }}.xml --cov-report=xml
+      displayName: '${{ parameters.name }} unit tests'
+
+    - task: PublishCodeCoverageResults@1
+      inputs:
+        codeCoverageTool: 'cobertura'
+        summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
+        reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov'
+        additionalCodeCoverageFiles: '$(System.DefaultWorkingDirectory)/htmlcov/**/*.*'
 
   - task: PublishTestResults@2
     condition: succeededOrFailed()
     displayName: 'Publish Test Results'
     inputs:
       testResultsFormat: 'JUnit'
       testResultsFiles: '**/test-*.xml'
-      testRunTitle: 'Publish test results for Python ${{ parameters.pythonVersion }} on OS $(Agent.OS)'
+      testRunTitle: 'Publish ${{ parameters.name }} test results for Python ${{ parameters.pythonVersion }} on OS $(Agent.OS)'
       searchFolder: '$(System.DefaultWorkingDirectory)'
+
+  - script: 'pip install .'
+    displayName: 'Install the whl locally'
+    workingDirectory: '.'
+
+  - task: PythonScript@0
+    displayName : 'setupVersion'
+    name: 'setupVersion'
+    inputs:
+      scriptSource: 'inline'
+      script: |
+        from azext_iot.constants import VERSION
+        print("Version is " + VERSION)
+        print("##vso[task.setvariable variable=CLIVersion;isOutput=true]"+VERSION)
+        print("##vso[task.setvariable variable=ReleaseTitle;isOutput=true]"+"azure-iot "+VERSION)

azext_iot/tests/test_iot_central_int.py → azext_iot/tests/central/test_iot_central_int.py

@@ -10,22 +10,22 @@
 import time
 import pytest
 
-from .conftest import get_context_path
+from azext_iot.tests.conftest import get_context_path
 
 from azure.iot.device import Message
 from azext_iot.common import utility
 from azext_iot.central.models.enum import DeviceStatus, Role
 from azext_iot.monitor.parsers import strings
 
-from . import CaptureOutputLiveScenarioTest, helpers
+from azext_iot.tests import CaptureOutputLiveScenarioTest, helpers
 
 APP_ID = os.environ.get("azext_iot_central_app_id")
 APP_PRIMARY_KEY = os.environ.get("azext_iot_central_primarykey")
 APP_SCOPE_ID = os.environ.get("azext_iot_central_scope_id")
 device_template_path = get_context_path(
-    __file__, "central/json/device_template_int_test.json"
+    __file__, "json/device_template_int_test.json"
 )
-sync_command_params = get_context_path(__file__, "central/json/sync_command_args.json")
+sync_command_params = get_context_path(__file__, "json/sync_command_args.json")
 
 if not all([APP_ID]):
     raise ValueError("Set azext_iot_central_app_id to run central integration tests.")

azext_iot/tests/test_iot_central_unit.py → azext_iot/tests/central/test_iot_central_unit.py

@@ -22,8 +22,8 @@
 from azext_iot.central.models.template import Template
 from azext_iot.monitor.property import PropertyMonitor
 from azext_iot.monitor.models.enum import Severity
-from .helpers import load_json
-from .test_constants import FileNames
+from azext_iot.tests.helpers import load_json
+from azext_iot.tests.test_constants import FileNames
 from azext_iot.constants import PNP_DTDLV2_COMPONENT_MARKER
 
 device_id = "mydevice"

azext_iot/tests/test_iot_central_validator_unit.py → azext_iot/tests/central/test_iot_central_validator_unit.py

@@ -10,8 +10,8 @@
 from azext_iot.central.models.template import Template
 from azext_iot.monitor.central_validator import validate, extract_schema_type
 
-from .helpers import load_json
-from .test_constants import FileNames
+from azext_iot.tests.helpers import load_json
+from azext_iot.tests.test_constants import FileNames
 
 
 class TestTemplateValidations:

azext_iot/tests/digitaltwins/__init__.py

@@ -6,8 +6,8 @@
 
 import pytest
 import os
-from ..generators import generate_generic_id
-from ..settings import DynamoSettings
+from azext_iot.tests.generators import generate_generic_id
+from azext_iot.tests.settings import DynamoSettings
 from azure.cli.testsdk import LiveScenarioTest
 from azext_iot.common.embedded_cli import EmbeddedCLI
 

azext_iot/tests/digitaltwins/test_dt_provider_unit.py

@@ -9,7 +9,7 @@
 import responses
 import json
 from azext_iot.digitaltwins.providers.base import DigitalTwinsProvider
-from ..generators import generate_generic_id
+from azext_iot.tests.generators import generate_generic_id
 
 
 resource_group = generate_generic_id()

azext_iot/tests/digitaltwins/test_dt_resource_lifecycle_int.py

@@ -8,7 +8,7 @@
 from time import sleep
 from knack.log import get_logger
 from azext_iot.digitaltwins.common import ADTEndpointType
-from ..settings import DynamoSettings
+from azext_iot.tests.settings import DynamoSettings
 from . import DTLiveScenarioTest
 from . import (
     MOCK_RESOURCE_TAGS,