diff --git a/.coveragerc b/.coveragerc index d574f09a5b4d..4014e9186c74 100644 --- a/.coveragerc +++ b/.coveragerc @@ -6,6 +6,11 @@ omit = */test* env* +[paths] +source = + sdk/ + **/sdk + [report] exclude_lines = pragma: no cover diff --git a/eng/ci_tools.txt b/eng/ci_tools.txt index 496f9a71065d..444dcba79cde 100644 --- a/eng/ci_tools.txt +++ b/eng/ci_tools.txt @@ -3,7 +3,7 @@ cryptography==3.1 setuptools==44.1.0; python_version == '2.7' setuptools==46.4.0; python_version >= '3.5' virtualenv==20.0.23 -wheel==0.34.2 +wheel==0.34.2 Jinja2==2.11.2 packaging==20.4 tox==3.15.0 @@ -18,6 +18,7 @@ coverage==4.5.4 codecov==2.1.0 beautifulsoup4==4.9.1 pkginfo==1.5.0.1 +pip==20.2 # locking packages defined as deps from azure-sdk-tools or azure-devtools pytoml==0.1.21 diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 2c2776d19644..786a8c092eac 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -39,31 +39,31 @@ parameters: Pool: $(LinuxPool) OSVmImage: PythonVersion: '2.7' - CoverageArg: '' + CoverageArg: '--disablecov' RunForPR: true Linux_Python35: Pool: $(LinuxPool) OSVmImage: PythonVersion: '3.5' - CoverageArg: '' + CoverageArg: '--disablecov' RunForPR: false Linux_Python38: Pool: $(LinuxPool) OSVmImage: PythonVersion: '3.8' - CoverageArg: '' + CoverageArg: '--disablecov' RunForPR: true Windows_Python35: Pool: $(WindowsPool) OSVmImage: PythonVersion: '3.5' - CoverageArg: '' + CoverageArg: '--disablecov' RunForPR: true MacOS_Python27: Pool: OSVmImage: 'macOS-10.15' PythonVersion: '2.7' - CoverageArg: '' + CoverageArg: '--disablecov' RunForPR: false Linux_pypy3: Pool: $(LinuxPool) @@ -100,7 +100,7 @@ jobs: steps: - template: ../steps/build-artifacts.yml - parameters: + parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} BuildTargetingString: ${{ parameters.BuildTargetingString }} BeforePublishSteps: ${{ parameters.BeforePublishSteps }} @@ -133,7 +133,7 @@ jobs: CheckLinkGuidance: $true - template: ../steps/analyze.yml - parameters: + parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} BuildTargetingString: ${{ parameters.BuildTargetingString }} TestMarkArgument: ${{ parameters.TestMarkArgument }} @@ -168,7 +168,7 @@ jobs: ${{ if or(eq(matrixEntry.value.RunForPR, 'true'), ne(variables['Build.Reason'], 'PullRequest')) }}: ${{ matrixEntry.key }}: ${{ insert }}: ${{ matrixEntry.value }} - + pool: name: $[coalesce(variables['Pool'], '')] vmImage: $[coalesce(variables['OSVmImage'], '')] @@ -195,7 +195,7 @@ jobs: { $toxenvvar = '$(Run.ToxCustomEnvs)' } - + echo "##vso[task.setvariable variable=toxenv]$toxenvvar" displayName: "Set Tox Environment" @@ -211,10 +211,10 @@ jobs: ToxTestEnv: $(toxenv) ToxEnvParallel: ${{ parameters.ToxEnvParallel }} InjectedPackages: $(InjectedPackages) - BeforeTestSteps: + BeforeTestSteps: - task: DownloadPipelineArtifact@0 inputs: - artifactName: 'artifacts' + artifactName: 'artifacts' targetPath: $(Build.ArtifactStagingDirectory) - template: ../steps/set-dev-build.yml @@ -237,6 +237,6 @@ jobs: steps: - template: ../steps/test_regression.yml - parameters: + parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} BuildTargetingString: ${{ parameters.BuildTargetingString }} diff --git a/eng/pipelines/templates/jobs/archetype-sdk-tests.yml b/eng/pipelines/templates/jobs/archetype-sdk-tests.yml index 54c1003a4b6f..f044ad328a23 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-tests.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-tests.yml @@ -1,49 +1,85 @@ parameters: - ServiceDirectory: '' - EnvVars: {} - MaxParallel: 0 - BeforeTestSteps: [] - AfterTestSteps: [] - BuildTargetingString: 'azure-*' - AdditionalTestArgs: '' - TestMarkArgument: '' - InjectedPackages: '' - BuildDocs: true - JobName: Test - AllocateResourceGroup: true - DeployArmTemplate: false - TestTimeoutInMinutes: 120 - TestSamples: false - Location: '' - Matrix: - Linux_Python35: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - PythonVersion: '3.5' - CoverageArg: '--disablecov' - MacOs_Python37: - Pool: Azure Pipelines - OSVmImage: 'macOS-10.15' - PythonVersion: '3.7' - CoverageArg: '--disablecov' - Windows_Python27: - Pool: azsdk-pool-mms-win-2019-general - OSVmImage: MMS2019 - PythonVersion: '2.7' - CoverageArg: '--disablecov' - Linux_PyPy3: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - PythonVersion: 'pypy3' - CoverageArg: '--disablecov' - Linux_Python39: - Pool: azsdk-pool-mms-ubuntu-1804-general - OSVmImage: MMSUbuntu18.04 - PythonVersion: '3.9' - CoverageArg: '' - CloudConfigurations: - AzureCloud: - SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) + - name: ServiceDirectory + type: string + default: '' + - name: EnvVars + type: object + default: {} + - name: MaxParallel + type: number + default: 0 + - name: BeforeTestSteps + type: object + default: [] + - name: AfterTestSteps + type: object + default: [] + - name: BuildTargetingString + type: string + default: 'azure-*' + - name: AdditionalTestArgs + type: string + default: '' + - name: TestMarkArgument + type: string + default: '' + - name: InjectedPackages + type: string + default: '' + - name: BuildDocs + type: boolean + default: true + - name: JobName + type: string + default: 'Test' + - name: AllocateResourceGroup + type: boolean + default: true + - name: DeployArmTemplate + type: boolean + default: false + - name: TestTimeoutInMinutes + type: number + default: 120 + - name: TestSamples + type: boolean + default: false + - name: Location + type: string + default: '' + - name: Matrix + type: object + default: + Linux_Python35: + Pool: azsdk-pool-mms-ubuntu-1804-general + OSVmImage: MMSUbuntu18.04 + PythonVersion: '3.5' + CoverageArg: '--disablecov' + MacOs_Python37: + Pool: Azure Pipelines + OSVmImage: 'macOS-10.15' + PythonVersion: '3.7' + CoverageArg: '--disablecov' + Windows_Python27: + Pool: azsdk-pool-mms-win-2019-general + OSVmImage: MMS2019 + PythonVersion: '2.7' + CoverageArg: '--disablecov' + Linux_PyPy3: + Pool: azsdk-pool-mms-ubuntu-1804-general + OSVmImage: MMSUbuntu18.04 + PythonVersion: 'pypy3' + CoverageArg: '--disablecov' + Linux_Python39: + Pool: azsdk-pool-mms-ubuntu-1804-general + OSVmImage: MMSUbuntu18.04 + PythonVersion: '3.9' + CoverageArg: '' + - name: CloudConfigurations + type: object + default: + AzureCloud: + SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) jobs: - ${{ each cloudConfig in parameters.CloudConfigurations }}: diff --git a/eng/pipelines/templates/jobs/publish-coverage.yml b/eng/pipelines/templates/jobs/publish-coverage.yml new file mode 100644 index 000000000000..f9d94a77acf9 --- /dev/null +++ b/eng/pipelines/templates/jobs/publish-coverage.yml @@ -0,0 +1,67 @@ +parameters: + - name: ServiceDirectory + type: string + default: '' + - name: Artifacts + type: object + default: [] + - name: DependsOn + type: string + default: '' + + +jobs: + - job: Publish_Code_Coverage + pool: + name: azsdk-pool-mms-ubuntu-1804-general + vmImage: MMSUbuntu18.04 + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + + # Download all existing artifacts (most of the time, it's whatever was uploaded before) + - pwsh: | + New-Item -Path $(Build.SourcesDirectory) -Name "_all_coverage_files" -ItemType "directory" + displayName: 'Create all coverages directory' + continueOnError: false + condition: succeededOrFailed() + + - task: DownloadPipelineArtifact@2 + inputs: + source: current + path: '$(Build.SourcesDirectory)/_all_coverage_files' + + - script: | + pip install -r eng/ci_tools.txt + pip freeze + displayName: 'Prep Environment' + + - pwsh: | + Get-ChildItem -Recurse .\_all_coverage_files\ | + ForEach-Object { + If (Test-Path $_ -PathType Leaf) { + Get-Content $_ + } + } + displayName: Show all coverage files + + - task: PythonScript@0 + displayName: 'Create Coverage Report' + inputs: + scriptPath: 'scripts/devops_tasks/create_coverage.py' + + - pwsh: | + Get-Content $(Build.SourcesDirectory)\.coverage + Get-Content $(Build.SourcesDirectory)\coverage.xml + continueOnError: true + displayName: Show .coverage and coverage.xml file + + - task: PublishCodeCoverageResults@1 + displayName: 'Publish Code Coverage to DevOps' + continueOnError: true + condition: succeededOrFailed() + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml' diff --git a/eng/pipelines/templates/stages/archetype-sdk-tests.yml b/eng/pipelines/templates/stages/archetype-sdk-tests.yml new file mode 100644 index 000000000000..58e592884538 --- /dev/null +++ b/eng/pipelines/templates/stages/archetype-sdk-tests.yml @@ -0,0 +1,114 @@ +parameters: + - name: ServiceDirectory + type: string + default: '' + - name: EnvVars + type: object + default: {} + - name: MaxParallel + type: number + default: 0 + - name: BeforeTestSteps + type: object + default: [] + - name: AfterTestSteps + type: object + default: [] + - name: BuildTargetingString + type: string + default: 'azure-*' + - name: AdditionalTestArgs + type: string + default: '' + - name: TestMarkArgument + type: string + default: '' + - name: InjectedPackages + type: string + default: '' + - name: BuildDocs + type: boolean + default: true + - name: JobName + type: string + default: 'Test' + - name: AllocateResourceGroup + type: boolean + default: true + - name: DeployArmTemplate + type: boolean + default: false + - name: TestTimeoutInMinutes + type: number + default: 120 + - name: TestSamples + type: boolean + default: false + - name: Location + type: string + default: '' + - name: Matrix + type: object + default: + Linux_Python35: + Pool: azsdk-pool-mms-ubuntu-1804-general + OSVmImage: MMSUbuntu18.04 + PythonVersion: '3.5' + CoverageArg: '--disablecov' + MacOs_Python37: + Pool: Azure Pipelines + OSVmImage: 'macOS-10.15' + PythonVersion: '3.7' + CoverageArg: '--disablecov' + Windows_Python27: + Pool: azsdk-pool-mms-win-2019-general + OSVmImage: MMS2019 + PythonVersion: '2.7' + CoverageArg: '--disablecov' + Linux_PyPy3: + Pool: azsdk-pool-mms-ubuntu-1804-general + OSVmImage: MMSUbuntu18.04 + PythonVersion: 'pypy3' + CoverageArg: '--disablecov' + Linux_Python39: + Pool: azsdk-pool-mms-ubuntu-1804-general + OSVmImage: MMSUbuntu18.04 + PythonVersion: '3.9' + CoverageArg: '' + - name: CloudConfigurations + type: object + default: + AzureCloud: + SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) + + +stages: + - stage: Live_${{ parameters.JobName }} + jobs: + - template: ../jobs/archetype-sdk-tests.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + EnvVars: ${{ parameters.EnvVars }} + MaxParallel: ${{ parameters.MaxParallel }} + BeforeTestSteps: ${{ parameters.BeforeTestSteps }} + AfterTestSteps: ${{ parameters.AfterTestSteps }} + AdditionalTestArgs: ${{ parameters.AdditionalTestArgs }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} + TestMarkArgument: ${{ parameters.TestMarkArgument }} + InjectedPackages: ${{ parameters.InjectedPackages }} + BuildDocs: ${{ parameters.BuildDocs }} + JobName: ${{ parameters.JobName }} + AllocateResourceGroup: ${{ parameters.AllocateResourceGroup }} + DeployArmTemplate: ${{ parameters.DeployArmTemplate }} + TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} + TestSamples: ${{ parameters.TestSamples }} + Location: ${{ parameters.Location }} + Matrix: ${{ parameters.Matrix }} + CloudConfigurations: ${{ parameters.CloudConfigurations }} + + + - stage: Publish_Coverage_${{ parameters.JobName }} + jobs: + - template: ../jobs/publish-coverage.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index c288c859149b..e4c41d912486 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -33,7 +33,7 @@ steps: - ${{if eq(variables['System.TeamProject'], 'internal') }}: - template: ../steps/auth-dev-feed.yml - parameters: + parameters: DevFeedName: ${{ parameters.DevFeedName }} - ${{ parameters.BeforeTestSteps }} @@ -55,6 +55,22 @@ steps: - ${{ parameters.AfterTestSteps }} + - pwsh: | + Get-ChildItem .\_coverage\ | + Foreach-Object { + Get-Content $_ + } + displayName: 'Show .coverage files' + condition: and(succeededOrFailed(), ${{ parameters.RunCoverage }}) + + - task: PublishPipelineArtifact@1 + displayName: 'Publish .coverage files' + continueOnError: true + condition: and(succeededOrFailed(), ${{ parameters.RunCoverage }}) + inputs: + targetPath: '$(Build.SourcesDirectory)/_coverage' + artifactType: 'pipeline' + - task: PublishTestResults@2 condition: always() inputs: @@ -62,10 +78,6 @@ steps: testRunTitle: '$(OSName) Python ${{ parameters.PythonVersion }}' failTaskOnFailedTests: true - - template: publish-coverage.yml - parameters: - RunCoverage: ${{ parameters.RunCoverage }} - - ${{ if eq(parameters['TestSamples'], true) }}: - task: PythonScript@0 displayName: 'Test Samples' diff --git a/eng/pipelines/templates/steps/publish-coverage.yml b/eng/pipelines/templates/steps/publish-coverage.yml deleted file mode 100644 index 1bc8f41c52b6..000000000000 --- a/eng/pipelines/templates/steps/publish-coverage.yml +++ /dev/null @@ -1,18 +0,0 @@ -parameters: - RunCoverage: false - -steps: - - - script: | - codecov -t $(codecov-python-repository-token) - displayName: 'Publish Code Cov' - continueOnError: true - condition: and(ne(variables['codecov-python-repository-token'], ''), succeededOrFailed(), ${{ parameters.RunCoverage }}) - - - task: PublishCodeCoverageResults@1 - displayName: 'Publish Code Coverage to DevOps' - continueOnError: true - condition: and(succeededOrFailed(), ${{ parameters.RunCoverage }}) - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml' diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index f7cefd5876a8..e536c9e42d91 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -26,7 +26,7 @@ deps = -r ../../../eng/dependency_tools.txt [packaging] -pkgs = +pkgs = wheel==0.34.2 packaging==20.4 @@ -46,7 +46,7 @@ setenv = deps = {[base]deps} changedir = {toxinidir} install_command = python -m pip install {opts} {packages} --cache-dir {toxinidir}/../.tox_pip_cache_{envname} -commands = +commands = {envbindir}/python {toxinidir}/../../../eng/tox/create_package_and_install.py -d {envtmpdir} -p {toxinidir} -w {envtmpdir} {envbindir}/python -m pip freeze pytest \ @@ -63,7 +63,7 @@ changedir = {toxinidir} deps = {[base]deps} -e {toxinidir}/../../scripts/pylint_custom_plugin -commands = +commands = {envbindir}/python {toxinidir}/../../../eng/tox/run_pylint.py -t {toxinidir} @@ -75,7 +75,7 @@ changedir = {toxinidir} deps = {[base]deps} mypy==0.782; python_version >= '3.5' -commands = +commands = {envbindir}/python {toxinidir}/../../../eng/tox/run_mypy.py -t {toxinidir} @@ -83,7 +83,7 @@ commands = skipsdist = true skip_install = true changedir = {toxinidir} -deps = +deps = {[base]deps} commands = {envbindir}/python {toxinidir}/../../../eng/tox/create_package_and_install.py \ @@ -104,7 +104,7 @@ skipsdist = false skip_install = false usedevelop = true changedir = {toxinidir} -deps = +deps = {[base]deps} commands = pytest \ @@ -118,7 +118,7 @@ skipsdist = false skip_install = false changedir = {toxinidir} passenv = * -deps = +deps = {[base]deps} sphinx==3.0.4 sphinx_rtd_theme==0.5.0 @@ -161,9 +161,9 @@ commands = skipsdist = true skip_install = true changedir = {toxinidir} -deps = +deps = commands = - {envbindir}/python {toxinidir}/../../../eng/tox/create_package_and_install.py -d {envtmpdir} -p {toxinidir} --skip-install True + {envbindir}/python {toxinidir}/../../../eng/tox/create_package_and_install.py -d {envtmpdir} -p {toxinidir} --skip-install True {envbindir}/python {toxinidir}/../../../eng/tox/verify_whl.py -d {envtmpdir} -t {toxinidir} @@ -171,7 +171,7 @@ commands = skipsdist = true skip_install = true changedir = {toxinidir} -deps = +deps = commands = {envbindir}/python {toxinidir}/setup.py --q sdist --format zip -d {envtmpdir} {envbindir}/python {toxinidir}/../../../eng/tox/verify_sdist.py -d {envtmpdir} -t {toxinidir} @@ -180,7 +180,7 @@ commands = [testenv:devtest] deps = {[base]deps} changedir = {toxinidir} -commands = +commands = {envbindir}/python {toxinidir}/../../../eng/tox/create_package_and_install.py -d {envtmpdir} -p {toxinidir} {envbindir}/python {toxinidir}/../../../eng/tox/install_dev_build_dependency.py -t {toxinidir} pytest \ @@ -200,20 +200,20 @@ commands = [testenv:latestdependency] -deps = +deps = {[dependencytools]deps} {[tools]deps} -changedir = +changedir = {toxinidir} passenv = * setenv = DEPENDENCY_TYPE=Latest -commands = +commands = {[deptestcommands]commands} [testenv:mindependency] -deps = +deps = azure-mgmt-keyvault<7.0.0 azure-mgmt-resource<15.0.0 azure-mgmt-storage<15.0.0 @@ -223,7 +223,7 @@ changedir = {toxinidir} passenv = * setenv = DEPENDENCY_TYPE=Minimum -commands = +commands = {[deptestcommands]commands} diff --git a/scripts/devops_tasks/common_tasks.py b/scripts/devops_tasks/common_tasks.py index 3afb50e3461d..8333cd093205 100644 --- a/scripts/devops_tasks/common_tasks.py +++ b/scripts/devops_tasks/common_tasks.py @@ -312,6 +312,7 @@ def create_code_coverage_params(parsed_args, package_name): else: current_package_name = package_name.replace("-", ".") coverage_args.append("--cov={}".format(current_package_name)) + coverage_args.append("--cov-append") logging.info( "Code coverage is enabled for package {0}, pytest arguements: {1}".format( current_package_name, coverage_args @@ -458,6 +459,6 @@ def get_installed_packages(paths = None): # WorkingSet returns installed packages in given path # working_set returns installed packages in default path # if paths is set then find installed packages from given paths - ws = WorkingSet(paths) if paths else working_set + ws = WorkingSet(paths) if paths else working_set return ["{0}=={1}".format(p.project_name, p.version) for p in ws] diff --git a/scripts/devops_tasks/create_coverage.py b/scripts/devops_tasks/create_coverage.py new file mode 100644 index 000000000000..e4f286253d31 --- /dev/null +++ b/scripts/devops_tasks/create_coverage.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import sys +import os +import logging +import re +from common_tasks import run_check_call + +logging.getLogger().setLevel(logging.INFO) + +root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) +coverage_dir = os.path.join(root_dir, "_all_coverage_files/") + + +def collect_tox_coverage_files(): + coverage_version_cmd = [sys.executable, "-m", "coverage", "--version"] + run_check_call(coverage_version_cmd, root_dir) + + logging.info("Running collect tox coverage files...") + root_coverage_dir = os.path.join(root_dir, "_coverage/") + + coverage_files = [] + for root, _, files in os.walk(coverage_dir): + for f in files: + if re.match(".coverage_*", f): + coverage_files.append(os.path.join(root, f)) + fix_dot_coverage_file(os.path.join(root, f)) + + logging.info(".coverage files: {}".format(coverage_files)) + + if len(coverage_files): + cov_cmd_array = [sys.executable, "-m", "coverage", "combine"] + cov_cmd_array.extend(coverage_files) + + # merge them with coverage combine and copy to root + run_check_call(cov_cmd_array, root_dir) + + logging.info("after running coverage combine") + for root, _, files in os.walk(root_dir): + for f in files: + if re.match(".coverage*", f): + print(os.path.join(root, f)) + + +def generate_coverage_xml(): + if os.path.exists(coverage_dir): + logging.info("Generating coverage XML") + commands = ["coverage", "xml", "-i", "--omit", '"*test*,*example*"'] + run_check_call(commands, root_dir, always_exit = False) + else: + logging.error("Coverage file is not available in {} to generate coverage XML".format(coverage_dir)) + + +def fix_dot_coverage_file(coverage_file): + print("running 'fix_dot_coverage_file' on {}".format(coverage_file)) + + out = None + with open(coverage_file) as cov_file: + line = cov_file.read() + out = re.sub("\/\.tox\/[\s\S]*?\/site-packages", "/", line) + + if out: + with open(coverage_file, 'w') as cov_file: + cov_file.write(out) + + +if __name__ == "__main__": + collect_tox_coverage_files() + generate_coverage_xml() \ No newline at end of file diff --git a/scripts/devops_tasks/setup_execute_tests.py b/scripts/devops_tasks/setup_execute_tests.py index a099f98fc122..6cba58c535a0 100644 --- a/scripts/devops_tasks/setup_execute_tests.py +++ b/scripts/devops_tasks/setup_execute_tests.py @@ -42,7 +42,7 @@ def combine_coverage_files(coverage_files): if os.path.isfile(tox_ini_file): # for every individual coverage file, run coverage combine to combine path for coverage_file in coverage_files: - cov_cmd_array = [sys.executable, "-m", "coverage", "combine"] + cov_cmd_array = [sys.executable, "-m", "coverage", "combine", "--append"] # tox.ini file has coverage paths to combine # Pas tox.ini as coverage config file cov_cmd_array.extend([config_file_flag, coverage_file]) diff --git a/scripts/devops_tasks/tox_harness.py b/scripts/devops_tasks/tox_harness.py index abc8a7ab520d..57640c5b9e4b 100644 --- a/scripts/devops_tasks/tox_harness.py +++ b/scripts/devops_tasks/tox_harness.py @@ -110,12 +110,8 @@ def collect_tox_coverage_files(targeted_packages): clean_coverage(coverage_dir) - # coverage report has paths starting .tox and azure # coverage combine fixes this with the help of tox.ini[coverage:paths] - combine_coverage_files(targeted_packages) - coverage_files = [] - # generate coverage files for package_dir in [package for package in targeted_packages]: coverage_file = os.path.join(package_dir, ".coverage") if os.path.isfile(coverage_file): @@ -125,31 +121,8 @@ def collect_tox_coverage_files(targeted_packages): shutil.copyfile(coverage_file, destination_file) coverage_files.append(destination_file) - logging.info("Visible uncombined .coverage files: {}".format(coverage_files)) - - if len(coverage_files): - cov_cmd_array = [sys.executable, "-m", "coverage", "combine"] - cov_cmd_array.extend(coverage_files) - - # merge them with coverage combine and copy to root - run_check_call(cov_cmd_array, os.path.join(root_dir, "_coverage/")) - - source = os.path.join(coverage_dir, "./.coverage") - dest = os.path.join(root_dir, ".coverage") + logging.info("Uploading .coverage files: {}".format(coverage_files)) - shutil.move(source, dest) - # Generate coverage XML - generate_coverage_xml() - - -def generate_coverage_xml(): - coverage_path = os.path.join(root_dir, ".coverage") - if os.path.exists(coverage_path): - logging.info("Generating coverage XML") - commands = ["coverage", "xml", "-i", "--omit", '"*test*,*example*"'] - run_check_call(commands, root_dir, always_exit = False) - else: - logging.error("Coverage file is not available in {} to generate coverage XML".format(coverage_path)) def individual_workload(tox_command_tuple, workload_results): diff --git a/sdk/appconfiguration/tests.yml b/sdk/appconfiguration/tests.yml index a206ebeecd6a..077713587e17 100644 --- a/sdk/appconfiguration/tests.yml +++ b/sdk/appconfiguration/tests.yml @@ -1,10 +1,9 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: AllocateResourceGroup: false - BuildTargetingString: $(BuildTargetingString) ServiceDirectory: appconfiguration DeployArmTemplate: true EnvVars: diff --git a/sdk/communication/tests.yml b/sdk/communication/tests.yml index cc8a08643902..9d8fb79431a9 100644 --- a/sdk/communication/tests.yml +++ b/sdk/communication/tests.yml @@ -1,7 +1,7 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: AllocateResourceGroup: 'false' BuildTargetingString: $(BuildTargetingString) diff --git a/sdk/cosmos/tests.yml b/sdk/cosmos/tests.yml index 118af435beb2..10e20422e1a7 100644 --- a/sdk/cosmos/tests.yml +++ b/sdk/cosmos/tests.yml @@ -1,7 +1,7 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: AllocateResourceGroup: 'false' MaxParallel: 1 diff --git a/sdk/digitaltwins/tests.yml b/sdk/digitaltwins/tests.yml index 8df329522fd6..8832d1f01e0e 100644 --- a/sdk/digitaltwins/tests.yml +++ b/sdk/digitaltwins/tests.yml @@ -1,7 +1,7 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: AllocateResourceGroup: 'false' BuildTargetingString: $(BuildTargetingString) diff --git a/sdk/eventgrid/tests.yml b/sdk/eventgrid/tests.yml index 023cd81888a4..cb42ebb00e2e 100644 --- a/sdk/eventgrid/tests.yml +++ b/sdk/eventgrid/tests.yml @@ -1,7 +1,7 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: ServiceDirectory: eventgrid BuildTargetingString: azure-eventgrid* diff --git a/sdk/eventhub/tests.yml b/sdk/eventhub/tests.yml index 4bf7d3acd3ab..da472e8ec61c 100644 --- a/sdk/eventhub/tests.yml +++ b/sdk/eventhub/tests.yml @@ -1,9 +1,9 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: - AllocateResourceGroup: 'false' + AllocateResourceGroup: false ServiceDirectory: eventhub Matrix: Linux_Python35: diff --git a/sdk/formrecognizer/tests.yml b/sdk/formrecognizer/tests.yml index 13b111649c4c..07c8fe885c45 100644 --- a/sdk/formrecognizer/tests.yml +++ b/sdk/formrecognizer/tests.yml @@ -1,7 +1,7 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: BuildTargetingString: azure-ai-formrecognizer ServiceDirectory: formrecognizer @@ -13,7 +13,7 @@ jobs: ${{ if contains(variables['Build.DefinitionName'], 'prod') }}: Matrix: Linux_Python35: - Pool: azsdk-pool-mms-ubuntu-1804-general + Pool: azsdk-pool-mms-ubuntu-1804-general OSVmImage: MMSUbuntu18.04 PythonVersion: '3.5' CoverageArg: '--disablecov' @@ -23,34 +23,34 @@ jobs: PythonVersion: '3.7' CoverageArg: '--disablecov' Windows_Python27: - Pool: azsdk-pool-mms-win-2019-general + Pool: azsdk-pool-mms-win-2019-general OSVmImage: MMS2019 PythonVersion: '2.7' CoverageArg: '--disablecov' Linux_PyPy3: - Pool: azsdk-pool-mms-ubuntu-1804-general + Pool: azsdk-pool-mms-ubuntu-1804-general OSVmImage: MMSUbuntu18.04 PythonVersion: 'pypy3' CoverageArg: '--disablecov' Linux_Python39: - Pool: azsdk-pool-mms-ubuntu-1804-general + Pool: azsdk-pool-mms-ubuntu-1804-general OSVmImage: MMSUbuntu18.04 PythonVersion: '3.9' CoverageArg: '' ${{ if not(contains(variables['Build.DefinitionName'], 'prod')) }}: Matrix: Linux_Python35: - Pool: azsdk-pool-mms-ubuntu-1804-general + Pool: azsdk-pool-mms-ubuntu-1804-general OSVmImage: MMSUbuntu18.04 PythonVersion: '3.5' CoverageArg: '--disablecov' Windows_Python27: - Pool: azsdk-pool-mms-win-2019-general + Pool: azsdk-pool-mms-win-2019-general OSVmImage: MMS2019 PythonVersion: '2.7' CoverageArg: '--disablecov' Linux_Python39: - Pool: azsdk-pool-mms-ubuntu-1804-general + Pool: azsdk-pool-mms-ubuntu-1804-general OSVmImage: MMSUbuntu18.04 PythonVersion: '3.9' CoverageArg: '' diff --git a/sdk/identity/tests.yml b/sdk/identity/tests.yml index 506f98ab8739..23554239c4ea 100644 --- a/sdk/identity/tests.yml +++ b/sdk/identity/tests.yml @@ -1,7 +1,7 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: AllocateResourceGroup: 'false' ServiceDirectory: identity diff --git a/sdk/keyvault/tests.yml b/sdk/keyvault/tests.yml index ce281c725734..c6c782116612 100644 --- a/sdk/keyvault/tests.yml +++ b/sdk/keyvault/tests.yml @@ -1,7 +1,7 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: ServiceDirectory: keyvault BuildTargetingString: azure-keyvault-keys @@ -14,7 +14,7 @@ jobs: AZURE_CLIENT_SECRET: $(aad-azure-sdk-test-client-secret) AZURE_CLIENT_OID: $(aad-azure-sdk-test-client-oid) - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: ServiceDirectory: keyvault BuildTargetingString: azure-keyvault-secrets @@ -27,7 +27,7 @@ jobs: AZURE_CLIENT_SECRET: $(aad-azure-sdk-test-client-secret) AZURE_CLIENT_OID: $(aad-azure-sdk-test-client-oid) - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: ServiceDirectory: keyvault BuildTargetingString: azure-keyvault-certificates diff --git a/sdk/metricsadvisor/tests.yml b/sdk/metricsadvisor/tests.yml index 74258345d64b..7b45f5234df3 100644 --- a/sdk/metricsadvisor/tests.yml +++ b/sdk/metricsadvisor/tests.yml @@ -1,7 +1,7 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: BuildTargetingString: azure-ai-metricsadvisor ServiceDirectory: metricsadvisor diff --git a/sdk/monitor/tests.yml b/sdk/monitor/tests.yml index fd0c57f4e093..beb1b7e9361b 100644 --- a/sdk/monitor/tests.yml +++ b/sdk/monitor/tests.yml @@ -1,7 +1,7 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: ServiceDirectory: monitor TestTimeoutInMinutes: 300 @@ -14,7 +14,7 @@ jobs: AZURE_TEST_RUN_LIVE: 'true' Matrix: Linux_Python35: - Pool: azsdk-pool-mms-ubuntu-1804-general + Pool: azsdk-pool-mms-ubuntu-1804-general OSVmImage: MMSUbuntu18.04 PythonVersion: '3.5' CoverageArg: '--disablecov' @@ -24,7 +24,7 @@ jobs: PythonVersion: '3.7' CoverageArg: '--disablecov' Linux_Python39: - Pool: azsdk-pool-mms-ubuntu-1804-general + Pool: azsdk-pool-mms-ubuntu-1804-general OSVmImage: MMSUbuntu18.04 PythonVersion: '3.9' CoverageArg: '' diff --git a/sdk/schemaregistry/tests.yml b/sdk/schemaregistry/tests.yml index 7b2df8b2ef75..922f16c2b1ed 100644 --- a/sdk/schemaregistry/tests.yml +++ b/sdk/schemaregistry/tests.yml @@ -1,7 +1,7 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: ServiceDirectory: schemaregistry TestTimeoutInMinutes: 300 @@ -18,7 +18,7 @@ jobs: AZURE_TEST_RUN_LIVE: 'true' Matrix: Linux_Python35: - Pool: azsdk-pool-mms-ubuntu-1804-general + Pool: azsdk-pool-mms-ubuntu-1804-general OSVmImage: MMSUbuntu18.04 PythonVersion: '3.5' CoverageArg: '--disablecov' @@ -28,12 +28,12 @@ jobs: PythonVersion: '3.7' CoverageArg: '--disablecov' Windows_Python27: - Pool: azsdk-pool-mms-win-2019-general + Pool: azsdk-pool-mms-win-2019-general OSVmImage: MMS2019 PythonVersion: '2.7' CoverageArg: '--disablecov' Linux_Python38: - Pool: azsdk-pool-mms-ubuntu-1804-general + Pool: azsdk-pool-mms-ubuntu-1804-general OSVmImage: MMSUbuntu18.04 PythonVersion: '3.8' CoverageArg: '' diff --git a/sdk/search/tests.yml b/sdk/search/tests.yml index 594b6ee0209e..0afd6785c54b 100644 --- a/sdk/search/tests.yml +++ b/sdk/search/tests.yml @@ -1,7 +1,7 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: BuildTargetingString: $(BuildTargetingString) ServiceDirectory: search diff --git a/sdk/servicebus/tests.yml b/sdk/servicebus/tests.yml index 71ed728bf9c3..1f38b1076654 100644 --- a/sdk/servicebus/tests.yml +++ b/sdk/servicebus/tests.yml @@ -1,7 +1,7 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: ServiceDirectory: servicebus TestTimeoutInMinutes: 300 @@ -14,7 +14,7 @@ jobs: AZURE_TEST_RUN_LIVE: 'true' Matrix: Linux_Python35: - Pool: azsdk-pool-mms-ubuntu-1804-general + Pool: azsdk-pool-mms-ubuntu-1804-general OSVmImage: MMSUbuntu18.04 PythonVersion: '3.5' CoverageArg: '--disablecov' @@ -24,12 +24,12 @@ jobs: PythonVersion: '3.7' CoverageArg: '--disablecov' Windows_Python27: - Pool: azsdk-pool-mms-win-2019-general + Pool: azsdk-pool-mms-win-2019-general OSVmImage: MMS2019 PythonVersion: '2.7' CoverageArg: '--disablecov' Linux_Python39: - Pool: azsdk-pool-mms-ubuntu-1804-general + Pool: azsdk-pool-mms-ubuntu-1804-general OSVmImage: MMSUbuntu18.04 PythonVersion: '3.9' CoverageArg: '' \ No newline at end of file diff --git a/sdk/storage/tests.yml b/sdk/storage/tests.yml index 3a415c102331..065c877474ae 100644 --- a/sdk/storage/tests.yml +++ b/sdk/storage/tests.yml @@ -1,6 +1,6 @@ trigger: none -jobs: +stages: - template: ./tests_invoke.yml parameters: InjectedPackages: $(InjectedPackages) @@ -29,4 +29,4 @@ jobs: AZURE_TENANT_ID: $(aad-azure-sdk-test-tenant-id) AZURE_SUBSCRIPTION_ID: $(azure-subscription-id) AZURE_CLIENT_SECRET: $(aad-azure-sdk-test-client-secret) - AZURE_CLIENT_ID: $(aad-azure-sdk-test-client-id) + AZURE_CLIENT_ID: $(aad-azure-sdk-test-client-id) \ No newline at end of file diff --git a/sdk/storage/tests_invoke.yml b/sdk/storage/tests_invoke.yml index b8ad483d1d38..45fb203d3efe 100644 --- a/sdk/storage/tests_invoke.yml +++ b/sdk/storage/tests_invoke.yml @@ -2,8 +2,8 @@ parameters: EnvVars: {} InjectedPackages: '' -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: AllocateResourceGroup: 'false' BuildTargetingString: azure-storage-blob @@ -12,7 +12,7 @@ jobs: EnvVars: ${{ parameters.EnvVars }} InjectedPackages: ${{ parameters.InjectedPackages }} - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: AllocateResourceGroup: 'false' BuildTargetingString: azure-storage-blob-changefeed @@ -21,7 +21,7 @@ jobs: EnvVars: ${{ parameters.EnvVars }} InjectedPackages: ${{ parameters.InjectedPackages }} - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: AllocateResourceGroup: 'false' BuildTargetingString: azure-storage-queue @@ -30,7 +30,7 @@ jobs: EnvVars: ${{ parameters.EnvVars }} InjectedPackages: ${{ parameters.InjectedPackages }} - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: AllocateResourceGroup: 'false' MaxParallel: 1 @@ -40,7 +40,7 @@ jobs: EnvVars: ${{ parameters.EnvVars }} InjectedPackages: ${{ parameters.InjectedPackages }} - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: AllocateResourceGroup: 'false' BuildTargetingString: azure-storage-file-share diff --git a/sdk/tables/tests.yml b/sdk/tables/tests.yml index 65ed3c745d15..8ddca5aabb69 100644 --- a/sdk/tables/tests.yml +++ b/sdk/tables/tests.yml @@ -1,7 +1,7 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: BuildTargetingString: azure-data-tables ServiceDirectory: tables diff --git a/sdk/template/tests.yml b/sdk/template/tests.yml index d152b6ad3882..f3215b43e4d5 100644 --- a/sdk/template/tests.yml +++ b/sdk/template/tests.yml @@ -1,7 +1,7 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: AllocateResourceGroup: 'false' BuildTargetingString: $(BuildTargetingString) diff --git a/sdk/textanalytics/tests.yml b/sdk/textanalytics/tests.yml index 6031dd294f9c..b5f5ffaec89b 100644 --- a/sdk/textanalytics/tests.yml +++ b/sdk/textanalytics/tests.yml @@ -1,7 +1,7 @@ trigger: none -jobs: - - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml +stages: + - template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: BuildTargetingString: $(BuildTargetingString) ServiceDirectory: textanalytics