diff --git a/eng/CredScanSuppression.json b/eng/CredScanSuppression.json index c8ee89d2523f..22b090557085 100644 --- a/eng/CredScanSuppression.json +++ b/eng/CredScanSuppression.json @@ -36,6 +36,7 @@ "sdk/confidentialledger/azure-confidentialledger/tests/_shared/constants.py", "sdk/keyvault/azure-keyvault-certificates/tests/ca.key", "sdk/identity/azure-identity/tests/certificate.pfx", + "sdk/identity/azure-identity/tests/certificate.pem", "sdk/identity/azure-identity/tests/certificate-with-password.pfx", "sdk/identity/azure-identity/tests/credscan_ignore.py", "sdk/identity/azure-identity/tests/ec-certificate.pem", diff --git a/eng/pipelines/templates/jobs/build-conda-dependencies.yml b/eng/pipelines/templates/jobs/build-conda-dependencies.yml index 45df27ef2782..de94c2545058 100644 --- a/eng/pipelines/templates/jobs/build-conda-dependencies.yml +++ b/eng/pipelines/templates/jobs/build-conda-dependencies.yml @@ -9,7 +9,8 @@ jobs: timeoutInMinutes: 90 pool: name: azsdk-pool-mms-win-2022-general - vmImage: MMS2022 + image: azsdk-pool-mms-win-2022-1espt + os: windows variables: VS_INSTALLER_URL: "https://aka.ms/vs/17/release/vs_enterprise.exe" VS_INSTALLER_PATH: "$(Build.BinariesDirectory)/vs_enterprise.exe" @@ -62,7 +63,7 @@ jobs: sdk_build_conda -c %arguments% displayName: Assemble Conda Packages - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactPath: '$(Build.SourcesDirectory)/conda/assembled' ArtifactName: 'windows_distributions' @@ -74,7 +75,7 @@ jobs: inputs: BuildDropPath: '$(Build.SourcesDirectory)/conda/output' - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactPath: '$(Build.SourcesDirectory)/conda/output' ArtifactName: 'windows_conda' @@ -84,7 +85,9 @@ jobs: timeoutInMinutes: 90 pool: name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 + image: azsdk-pool-mms-ubuntu-2004-1espt + os: linux + steps: - bash: | sudo apt-get install -y build-essential @@ -101,6 +104,7 @@ jobs: pool: name: Azure Pipelines vmImage: macos-11 + os: macOS variables: MacOSXDeploymentTarget: '10.9' OpenSSLDir: $(Agent.BuildDirectory)/openssl-macosx$(MacOSXDeploymentTarget) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 5fa207d660b4..8ee11130f7b8 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -29,11 +29,11 @@ parameters: - name: InjectedPackages type: string default: '' - - name: Matrix - type: string - name: DependsOn + type: object + default: [] + - name: Matrix type: string - default: '' - name: UsePlatformContainer type: boolean default: false @@ -43,6 +43,8 @@ parameters: - name: TestProxy type: boolean default: false + - name: OSName + type: string jobs: - job: @@ -56,24 +58,27 @@ jobs: timeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} dependsOn: - - ${{ parameters.DependsOn }} + - ${{ parameters.DependsOn }} strategy: matrix: $[ ${{ parameters.Matrix }} ] pool: name: $(Pool) - vmImage: $(OSVmImage) - - ${{ if eq(parameters.UsePlatformContainer, 'true') }}: - # Add a default so the job doesn't fail when the matrix is empty - container: $[ variables['Container'] ] + # 1es pipeline templates converts `image` to demands: ImageOverride under the hood + # which is incompatible with image selection in the default non-1es hosted pools + ${{ if eq(parameters.OSName, 'macOS') }}: + vmImage: $(OSVmImage) + ${{ else }}: + image: $(OSVmImage) + os: ${{ parameters.OSName }} variables: - template: ../variables/globals.yml - name: InjectedPackages value: ${{ parameters.InjectedPackages }} + steps: - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml parameters: @@ -81,7 +86,7 @@ jobs: - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml parameters: - AgentImage: $(OSVmImage) + AgentImage: ${{ parameters.OSName }} - task: UsePythonVersion@0 inputs: @@ -111,7 +116,6 @@ jobs: ServiceDirectory: ${{ parameters.ServiceDirectory }} TestMarkArgument: ${{ parameters.TestMarkArgument }} AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"' - OSVmImage: $(OSVmImage) CoverageArg: $(CoverageArg) PythonVersion: $(PythonVersion) ToxTestEnv: $(toxenv) @@ -127,7 +131,7 @@ jobs: - template: ../steps/set-dev-build.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} - + - ${{ each step in parameters.BeforeTestSteps }}: - ${{ step }} AfterTestSteps: ${{ parameters.AfterTestSteps }} diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 28c3d62c27c7..f30d5b0a9ed3 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -66,14 +66,15 @@ parameters: default: false jobs: - + - ${{ if eq(parameters['AdvancedBuild'], false) }}: - job: 'Build' timeoutInMinutes: 90 pool: name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 + image: azsdk-pool-mms-ubuntu-2004-1espt + os: linux steps: - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml @@ -88,27 +89,34 @@ jobs: Artifacts: ${{ parameters.Artifacts }} - ${{ if eq(parameters['AdvancedBuild'], true) }}: - - job: 'Build' + - job: 'Build_Linux' timeoutInMinutes: 90 - strategy: - matrix: - Linux: - imageName: 'MMSUbuntu22.04' - poolName: 'azsdk-pool-mms-ubuntu-2204-general' - ArtifactName: 'linux' - Windows: - imageName: 'MMS2022' - poolName: 'azsdk-pool-mms-win-2022-general' - ArtifactName: 'windows' - Mac: - imageName: 'macos-11' - poolName: 'Azure Pipelines' - ArtifactName: 'mac' + pool: + name: azsdk-pool-mms-ubuntu-2004-general + image: azsdk-pool-mms-ubuntu-2004-1espt + os: linux + + steps: + - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml + parameters: + BuildTargetingString: ${{ parameters.BuildTargetingString }} + + - template: ../steps/build-package-artifacts.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + BeforePublishSteps: ${{ parameters.BeforePublishSteps }} + TestPipeline: ${{ parameters.TestPipeline }} + Artifacts: ${{ parameters.Artifacts }} + ArtifactSuffix: linux + + - job: 'Build_Windows' + timeoutInMinutes: 90 pool: - name: $(poolName) - vmImage: $(imageName) + name: azsdk-pool-mms-win-2022-general + image: azsdk-pool-mms-win-2022-1espt + os: windows steps: - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml @@ -121,17 +129,41 @@ jobs: BeforePublishSteps: ${{ parameters.BeforePublishSteps }} TestPipeline: ${{ parameters.TestPipeline }} Artifacts: ${{ parameters.Artifacts }} - ArtifactSuffix: $(ArtifactName) + ArtifactSuffix: windows + + - job: 'Build_MacOS' + timeoutInMinutes: 90 + + pool: + name: 'Azure Pipelines' + vmImage: macos-11 + os: macOS + + steps: + - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml + parameters: + BuildTargetingString: ${{ parameters.BuildTargetingString }} + + - template: ../steps/build-package-artifacts.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + BeforePublishSteps: ${{ parameters.BeforePublishSteps }} + TestPipeline: ${{ parameters.TestPipeline }} + Artifacts: ${{ parameters.Artifacts }} + ArtifactSuffix: mac - job: 'CoalesceBuildArtifacts' displayName: Combine Built Artifacts - dependsOn: - - 'Build' + dependsOn: + - 'Build_Linux' + - 'Build_Windows' + - 'Build_MacOS' timeoutInMinutes: 90 pool: name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 + image: azsdk-pool-mms-ubuntu-2004-1espt + os: linux steps: - task: DownloadPipelineArtifact@2 @@ -149,22 +181,25 @@ jobs: artifactName: 'packages_linux' targetPath: $(Build.ArtifactStagingDirectory)/packages - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactPath: '$(Build.ArtifactStagingDirectory)/packages' ArtifactName: 'packages' - job: 'Build_Extended' displayName: Build Extended - dependsOn: - - 'Build' + dependsOn: - ${{ if eq(parameters['AdvancedBuild'], true) }}: - 'CoalesceBuildArtifacts' + - ${{ else }}: + - 'Build' + timeoutInMinutes: 90 pool: name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 + image: azsdk-pool-mms-ubuntu-2004-1espt + os: linux steps: - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml @@ -189,7 +224,8 @@ jobs: pool: name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 + image: azsdk-pool-mms-ubuntu-2004-1espt + os: linux steps: - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml @@ -220,35 +256,16 @@ jobs: ValidateFormatting: ${{ parameters.ValidateFormatting }} GenerateApiReviewForManualOnly: ${{ parameters.GenerateApiReviewForManualOnly }} - - job: Compliance - pool: - name: azsdk-pool-mms-win-2022-general - vmImage: MMS2022 - - variables: - Codeql.SkipTaskAutoInjection: false - Codeql.Enabled: true - Codeql.Language: python - Codeql.BuildIdentifier: "${{ parameters.ServiceDirectory }}" - Codeql.SourceRoot: "sdk/${{ parameters.ServiceDirectory }}" - - # per the guidance of the codeql team: - # https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/codeql/snippets/codeql-3000-other-issues#timeouts - timeoutInMinutes: 360 - - steps: - - template: /eng/common/pipelines/templates/steps/credscan.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - BaselineFilePath: $(Build.SourcesDirectory)\eng\python.gdnbaselines - - - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml + - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml parameters: JobTemplatePath: /eng/pipelines/templates/jobs/ci.tests.yml + OsVmImage: azsdk-pool-mms-ubuntu-2004-1espt + Pool: azsdk-pool-mms-ubuntu-2004-general DependsOn: - - 'Build' - ${{ if eq(parameters['AdvancedBuild'], true) }}: - 'CoalesceBuildArtifacts' + - ${{ else }}: + - 'Build' MatrixConfigs: ${{ parameters.MatrixConfigs }} MatrixFilters: ${{ parameters.MatrixFilters }} MatrixReplace: ${{ parameters.MatrixReplace }} @@ -267,9 +284,11 @@ jobs: UnsupportedToxEnvironments: ${{ parameters.UnsupportedToxEnvironments }} TestProxy: ${{ parameters.TestProxy }} - - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml + - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml parameters: JobTemplatePath: /eng/pipelines/templates/jobs/regression.yml + OsVmImage: azsdk-pool-mms-ubuntu-2004-1espt + Pool: azsdk-pool-mms-ubuntu-2004-general GenerateJobName: generate_regression_matrix SparseCheckoutPaths: [ "scripts/", "sdk/", "tools/azure-sdk-tools/" ] MatrixConfigs: @@ -295,9 +314,10 @@ jobs: CloudConfig: Cloud: Public DependsOn: - - 'Build' - ${{ if eq(parameters['AdvancedBuild'], true) }}: - 'CoalesceBuildArtifacts' + - ${{ else }}: + - 'Build' AdditionalParameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} ServiceDirectory: ${{ parameters.ServiceDirectory }} diff --git a/eng/pipelines/templates/jobs/live.tests.yml b/eng/pipelines/templates/jobs/live.tests.yml index d34fc272dd55..c09cc3f02d74 100644 --- a/eng/pipelines/templates/jobs/live.tests.yml +++ b/eng/pipelines/templates/jobs/live.tests.yml @@ -58,9 +58,8 @@ parameters: - name: TestProxy type: boolean default: false - - name: ToxTestEnv + - name: OSName type: string - default: 'whl' jobs: - job: @@ -85,7 +84,13 @@ jobs: pool: name: $(Pool) - vmImage: $(OSVmImage) + # 1es pipeline templates converts `image` to demands: ImageOverride under the hood + # which is incompatible with image selection in the default non-1es hosted pools + ${{ if eq(parameters.OSName, 'macOS') }}: + vmImage: $(OSVmImage) + ${{ else }}: + image: $(OSVmImage) + os: ${{ parameters.OSName }} ${{ if eq(parameters.UsePlatformContainer, 'true') }}: # Add a default so the job doesn't fail when the matrix is empty @@ -94,7 +99,7 @@ jobs: steps: - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml parameters: - AgentImage: $(OSVmImage) + AgentImage: ${{ parameters.OSName }} - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml parameters: @@ -134,7 +139,7 @@ jobs: PostSteps: ${{ parameters.PostSteps }} PythonVersion: $(PythonVersion) OSVmImage: $(OSVmImage) - ToxTestEnv: ${{ parameters.ToxTestEnv }} + ToxTestEnv: "whl" AdditionalTestArgs: ${{ parameters.AdditionalTestArgs }} TestMarkArgument: ${{ parameters.TestMarkArgument }} InjectedPackages: ${{ parameters.InjectedPackages }} diff --git a/eng/pipelines/templates/jobs/regression.yml b/eng/pipelines/templates/jobs/regression.yml index 472ad702dbef..b58d10b002d1 100644 --- a/eng/pipelines/templates/jobs/regression.yml +++ b/eng/pipelines/templates/jobs/regression.yml @@ -25,6 +25,8 @@ parameters: - name: BuildTargetingString type: string default: 'azure-*' + - name: OSName + type: string # The variable $(DependentService) is set from the matrix configuration. @@ -53,7 +55,8 @@ jobs: pool: name: $(Pool) - vmImage: $(OSVmImage) + image: $(OSVmImage) + os: ${{ parameters.OSName }} ${{ if eq(parameters.UsePlatformContainer, 'true') }}: # Add a default so the job doesn't fail when the matrix is empty diff --git a/eng/pipelines/templates/jobs/smoke.tests.yml b/eng/pipelines/templates/jobs/smoke.tests.yml index 8f6240a37259..25e246be3713 100644 --- a/eng/pipelines/templates/jobs/smoke.tests.yml +++ b/eng/pipelines/templates/jobs/smoke.tests.yml @@ -13,8 +13,9 @@ jobs: - job: smoke_test_eligibility displayName: Check Smoke Test Eligibility pool: - name: "azsdk-pool-mms-ubuntu-2004-general" - vmImage: "MMSUbuntu20.04" + name: azsdk-pool-mms-ubuntu-2004-general + image: azsdk-pool-mms-ubuntu-2004-1espt + os: linux steps: - ${{ if and(ne(variables['Skip.Release'], 'true'), ne(parameters.Artifact.skipPublishPackage, 'true')) }}: - pwsh: | @@ -33,8 +34,8 @@ jobs: env: RunSmokeTests: $(RunSmokeTests) - - job: run_smoke_test - displayName: Run Smoke Test + - job: run_smoke_test_linux + displayName: Run Smoke Test Linux ${{ if eq(parameters.Daily, false) }}: dependsOn: smoke_test_eligibility condition: and(succeeded(), eq(dependencies.smoke_test_eligibility.outputs['output_eligibility.RunSmokeTests'], true)) @@ -43,84 +44,48 @@ jobs: Python_38_Linux (AzureCloud): PythonVersion: '3.8' Pool: "azsdk-pool-mms-ubuntu-2004-general" - OSVmImage: "MMSUbuntu20.04" + OSVmImage: azsdk-pool-mms-ubuntu-2004-1espt SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) ArmTemplateParameters: $(azureCloudArmParameters) ${{ if eq(parameters.Daily, true) }}: Python_27_Linux (AzureCloud): PythonVersion: '2.7' SkipAsyncInstall: true - Pool: "azsdk-pool-mms-ubuntu-2004-general" - OSVmImage: "MMSUbuntu20.04" + Pool: azsdk-pool-mms-ubuntu-2004-general + OSVmImage: azsdk-pool-mms-ubuntu-2004-1espt SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) ArmTemplateParameters: $(azureCloudArmParameters) Python_37_Linux (AzureCloud): PythonVersion: '3.7' - Pool: "azsdk-pool-mms-ubuntu-2004-general" - OSVmImage: "MMSUbuntu20.04" + Pool: azsdk-pool-mms-ubuntu-2004-general + OSVmImage: azsdk-pool-mms-ubuntu-2004-1espt SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) ArmTemplateParameters: $(azureCloudArmParameters) Python_38_Linux (AzureCloud Canary): PythonVersion: '3.8' - Pool: "azsdk-pool-mms-ubuntu-2004-general" - OSVmImage: "MMSUbuntu20.04" + Pool: azsdk-pool-mms-ubuntu-2004-general + OSVmImage: azsdk-pool-mms-ubuntu-2004-1espt SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources-preview) ArmTemplateParameters: $(azureCloudArmParameters) Location: 'eastus2euap' - Python_37_Windows (AzureCloud): - PythonVersion: '3.7' - Pool: "azsdk-pool-mms-win-2022-general" - OSVmImage: "MMS2022" - SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) - ArmTemplateParameters: $(azureCloudArmParameters) - Python_38_Windows (AzureCloud): - PythonVersion: '3.8' - Pool: "azsdk-pool-mms-ubuntu-2004-general" - OSVmImage: "MMSUbuntu20.04" - SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) - ArmTemplateParameters: $(azureCloudArmParameters) - Python_37_Mac (AzureCloud): - PythonVersion: '3.7' - Pool: Azure Pipelines - OSVmImage: macos-11 - SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) - ArmTemplateParameters: $(azureCloudArmParameters) - Python_38_Mac (AzureCloud): - PythonVersion: '3.8' - Pool: Azure Pipelines - OSVmImage: macos-11 - SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) - ArmTemplateParameters: $(azureCloudArmParameters) Python_38_Linux (AzureUSGovernment): PythonVersion: '3.8' - Pool: "azsdk-pool-mms-ubuntu-2004-general" - OSVmImage: "MMSUbuntu120.04" - SubscriptionConfiguration: $(sub-config-gov-test-resources) - ArmTemplateParameters: $(azureUSGovernmentArmParameters) - Python_37_Windows (AzureUSGovernment): - PythonVersion: '3.7' - Pool: "azsdk-pool-mms-ubuntu-2004-general" - OSVmImage: "MMSUbuntu20.04" + Pool: azsdk-pool-mms-ubuntu-2004-general + OSVmImage: azsdk-pool-mms-ubuntu-2004-1espt SubscriptionConfiguration: $(sub-config-gov-test-resources) ArmTemplateParameters: $(azureUSGovernmentArmParameters) Python_38_Linux (AzureChinaCloud): PythonVersion: '3.8' - Pool: "azsdk-pool-mms-ubuntu-2004-general" - OSVmImage: "MMSUbuntu20.04" - SubscriptionConfiguration: $(sub-config-cn-test-resources) - Location: 'chinanorth' - ArmTemplateParameters: $(azureChinaCloudArmParameters) - Python_37_Windows (AzureChinaCloud): - PythonVersion: '3.7' - Pool: "azsdk-pool-mms-ubuntu-2004-general" - OSVmImage: "MMSUbuntu20.04" + Pool: azsdk-pool-mms-ubuntu-2004-general + OSVmImage: azsdk-pool-mms-ubuntu-2004-1espt SubscriptionConfiguration: $(sub-config-cn-test-resources) Location: 'chinanorth' ArmTemplateParameters: $(azureChinaCloudArmParameters) - + pool: name: $(Pool) - vmImage: $(OSVmImage) + image: $(OSVmImage) + os: linux variables: - template: /eng/pipelines/templates/variables/globals.yml @@ -139,107 +104,115 @@ jobs: value: $(Build.SourcesDirectory)/common/smoketest/requirements-release.txt steps: - - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml + - template: /eng/pipelines/templates/steps/smoke-test-steps.yml parameters: - AgentImage: $(OSVmImage) - - - task: UsePythonVersion@0 - displayName: "Use Python $(PythonVersion)" - inputs: - versionSpec: $(PythonVersion) - - - script: | - python -m pip install pip==20.0.2 - pip --version - displayName: pip --version - - - ${{ if eq(parameters.Daily, false) }}: - - download: current - artifact: ${{ parameters.ArtifactName }} - timeoutInMinutes: 5 - - - pwsh: | - $packages = Get-ChildItem "$(Pipeline.Workspace)/${{ parameters.ArtifactName }}/${{ parameters.Artifact.name }}/*.tar.gz" - Write-Host "Artifacts found:" - $artifacts = $packages | ForEach-Object { - if ($_.Name -match "([a-zA-Z\-]+)\-(.*).tar.gz") { - Write-Host "$($matches[1]): $($matches[2])" - return @{ "name" = $matches[1]; "version" = $matches[2] } - } - } - if ($artifacts.name -notcontains "${{parameters.Artifact.name}}") { - Write-Host "Can't find package ${{parameters.Artifact.name}}" - exit 1 - } - $dependencies = Get-Content $(requirements) | ForEach-Object { - $line = $_ - if ($line -match "([a-zA-Z\-]+)(\W+)(.*)") { - $override = ($artifacts | Where-Object { $_.Name -eq $matches[1] }).Version - if ($override) { - $line = $line -replace '([a-zA-Z\-]+)(\W+)(.*)', ('${1}${2}' + $override) - Write-Host "Overriding dependency to: $line" - } - } - return $line - } - - $dependencies | Out-File $(requirements) - - displayName: Override requirements with pipeline build artifact versions - - # Retry for pip install due to delay in package availability after publish - # The package is expected to be available for download/installation within 10 minutes - - pwsh: | - $ErrorActionPreference = "Continue" - while ($retries++ -lt 15) { - Write-Host "python -m pip install -r $(requirements) --no-deps --upgrade --no-cache-dir" - python -m pip install -r "$(requirements)" --no-deps --upgrade --no-cache-dir - if ($LASTEXITCODE) { - if ($retries -ge 15) { - exit $LASTEXITCODE - } - Write-Host "Installation failed, retrying in 1 minute..." - sleep 60 - } else { - break - } - } - displayName: Install requirements without dependencies - - - ${{ if eq(parameters.Daily, true) }}: - - pwsh: | - python -m pip install -r "$(requirements)" --pre --no-deps --upgrade ` - --index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple - - displayName: Install requirements from dev feed without dependencies - - - pwsh: python -m pip install -r $(Build.SourcesDirectory)/common/smoketest/requirements_async.txt - displayName: "Install requirements_async.txt" - condition: and(succeeded(), ne(variables['SkipAsyncInstall'], true)) - - - pwsh: | - python $(Build.SourcesDirectory)/common/smoketest/dependencies.py -r "$(requirements)" ` - | Out-File $(Build.SourcesDirectory)/common/smoketest/requirements_dependencies.txt - displayName: Create dependency list from installed packages - - - script: python -m pip install -r $(Build.SourcesDirectory)/common/smoketest/requirements_dependencies.txt - displayName: Install package dependencies from PyPI - - - script: python -m pip freeze - displayName: Show installed packages (pip freeze) - - - template: /eng/common/TestResources/deploy-test-resources.yml - parameters: - ServiceDirectory: '$(Build.SourcesDirectory)/common/smoketest/' - SubscriptionConfiguration: $(SubscriptionConfiguration) - Location: $(Location) - ArmTemplateParameters: $(ArmTemplateParameters) - - - script: python $(Build.SourcesDirectory)/common/smoketest/program.py - displayName: Run Smoke Test + Artifact: ${{ parameters.Artifact }} + ArtifactName: ${{ parameters.ArtifactName }} + Daily: ${{ parameters.Daily }} + + - ${{ if eq(parameters.Daily, true) }}: + - job: run_smoke_test_windows + displayName: Run Smoke Test Windows + ${{ if eq(parameters.Daily, false) }}: + dependsOn: smoke_test_eligibility + condition: and(succeeded(), eq(dependencies.smoke_test_eligibility.outputs['output_eligibility.RunSmokeTests'], true)) + strategy: + matrix: + Python_37_Windows (AzureCloud): + PythonVersion: '3.7' + Pool: azsdk-pool-mms-win-2022-general + OSVmImage: azsdk-pool-mms-win-2022-1espt + SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) + ArmTemplateParameters: $(azureCloudArmParameters) + Python_38_Windows (AzureCloud): + PythonVersion: '3.8' + Pool: azsdk-pool-mms-win-2022-general + OSVmImage: azsdk-pool-mms-win-2022-1espt + SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) + ArmTemplateParameters: $(azureCloudArmParameters) + Python_37_Windows (AzureUSGovernment): + PythonVersion: '3.7' + Pool: azsdk-pool-mms-win-2022-general + OSVmImage: azsdk-pool-mms-win-2022-1espt + SubscriptionConfiguration: $(sub-config-gov-test-resources) + ArmTemplateParameters: $(azureUSGovernmentArmParameters) + Python_37_Windows (AzureChinaCloud): + PythonVersion: '3.7' + Pool: azsdk-pool-mms-win-2022-general + OSVmImage: azsdk-pool-mms-win-2022-1espt + SubscriptionConfiguration: $(sub-config-cn-test-resources) + Location: 'chinanorth' + ArmTemplateParameters: $(azureChinaCloudArmParameters) + pool: + name: $(Pool) + image: $(OSVmImage) + os: windows + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - name: Location + value: '' + - name: azureCloudArmParameters + value: "@{ storageEndpointSuffix = 'core.windows.net'; azureCloud = 'AzureCloud'; }" + - name: azureUSGovernmentArmParameters + value: "@{ storageEndpointSuffix = 'core.usgovcloudapi.net'; azureCloud = 'AzureUSGovernment'; }" + - name: azureChinaCloudArmParameters + value: "@{ storageEndpointSuffix = 'core.chinacloudapi.cn'; azureCloud = 'AzureChinaCloud'; }" + - name: requirements + ${{ if eq(parameters.Daily, true) }}: + value: $(Build.SourcesDirectory)/common/smoketest/requirements.txt + ${{ if eq(parameters.Daily, false) }}: + value: $(Build.SourcesDirectory)/common/smoketest/requirements-release.txt + steps: + - template: /eng/pipelines/templates/steps/smoke-test-steps.yml + parameters: + Artifact: ${{ parameters.Artifact }} + ArtifactName: ${{ parameters.ArtifactName }} + Daily: ${{ parameters.Daily }} + + - job: run_smoke_test_macos + displayName: Run Smoke Test MacOS + ${{ if eq(parameters.Daily, false) }}: + dependsOn: smoke_test_eligibility + condition: and(succeeded(), eq(dependencies.smoke_test_eligibility.outputs['output_eligibility.RunSmokeTests'], true)) + strategy: + matrix: + Python_37_Mac (AzureCloud): + PythonVersion: '3.7' + Pool: Azure Pipelines + OSVmImage: macos-11 + SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) + ArmTemplateParameters: $(azureCloudArmParameters) + Python_38_Mac (AzureCloud): + PythonVersion: '3.8' + Pool: Azure Pipelines + OSVmImage: macos-11 + SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources) + ArmTemplateParameters: $(azureCloudArmParameters) - - template: /eng/common/TestResources/remove-test-resources.yml - parameters: - ServiceDirectory: '$(Build.SourcesDirectory)/common/smoketest/' - SubscriptionConfiguration: $(SubscriptionConfiguration) + pool: + name: $(Pool) + vmImage: $(OSVmImage) + os: macOS + + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - name: Location + value: '' + - name: azureCloudArmParameters + value: "@{ storageEndpointSuffix = 'core.windows.net'; azureCloud = 'AzureCloud'; }" + - name: azureUSGovernmentArmParameters + value: "@{ storageEndpointSuffix = 'core.usgovcloudapi.net'; azureCloud = 'AzureUSGovernment'; }" + - name: azureChinaCloudArmParameters + value: "@{ storageEndpointSuffix = 'core.chinacloudapi.cn'; azureCloud = 'AzureChinaCloud'; }" + - name: requirements + ${{ if eq(parameters.Daily, true) }}: + value: $(Build.SourcesDirectory)/common/smoketest/requirements.txt + ${{ if eq(parameters.Daily, false) }}: + value: $(Build.SourcesDirectory)/common/smoketest/requirements-release.txt + steps: + - template: /eng/pipelines/templates/steps/smoke-test-steps.yml + parameters: + Artifact: ${{ parameters.Artifact }} + ArtifactName: ${{ parameters.ArtifactName }} + Daily: ${{ parameters.Daily }} \ No newline at end of file diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index 8ab88f791d45..8b22e457e7c2 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -25,8 +25,9 @@ stages: environment: github pool: + image: azsdk-pool-mms-ubuntu-2004-1espt name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 + os: linux strategy: runOnce: @@ -85,8 +86,9 @@ stages: dependsOn: TagRepository pool: + image: azsdk-pool-mms-ubuntu-2004-1espt name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 + os: linux strategy: runOnce: @@ -159,7 +161,8 @@ stages: pool: name: azsdk-pool-mms-win-2022-general - vmImage: MMS2022 + image: azsdk-pool-mms-win-2022-1espt + os: windows strategy: runOnce: @@ -190,8 +193,9 @@ stages: dependsOn: PublishPackage pool: + image: azsdk-pool-mms-ubuntu-2004-1espt name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 + os: linux strategy: runOnce: @@ -224,8 +228,9 @@ stages: dependsOn: PublishPackage pool: + image: azsdk-pool-mms-ubuntu-2004-1espt name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 + os: linux strategy: runOnce: @@ -262,8 +267,9 @@ stages: displayName: "Publish package to daily feed" condition: or(eq(variables['SetDevVersion'], 'true'), and(eq(variables['Build.Reason'],'Schedule'), eq(variables['System.TeamProject'], 'internal'))) pool: + image: azsdk-pool-mms-ubuntu-2004-1espt name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 + os: linux steps: - checkout: none - download: current @@ -300,8 +306,9 @@ stages: dependsOn: PublishPackages condition: or(eq(variables['SetDevVersion'], 'true'), and(eq(variables['Build.Reason'],'Schedule'), eq(variables['System.TeamProject'], 'internal'))) pool: + image: azsdk-pool-mms-ubuntu-2004-1espt name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 + os: linux steps: - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 29de298f3d87..0d7e28f83ee9 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -1,127 +1,163 @@ +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + parameters: -- name: ServiceDirectory - type: string - default: not-specified -- name: Artifacts - type: object - default: [] -- name: TestPipeline - type: boolean - default: false -- name: BeforeTestSteps - type: object - default: [] -- name: AfterTestSteps - type: object - default: [] -- name: BeforePublishSteps - type: object - default: [] -- name: TestMarkArgument - type: string - default: '' -- name: BuildTargetingString - type: string - default: azure-* -- name: TestTimeoutInMinutes - type: number - default: 60 -- name: ToxEnvParallel - type: string - default: --tenvparallel -- name: InjectedPackages - type: string - default: '' -- name: BuildDocs - type: boolean - default: true -- name: DevFeedName - type: string - default: public/azure-sdk-for-python -- name: TargetDocRepoOwner - type: string - default: MicrosoftDocs -- name: TargetDocRepoName - type: string - default: azure-docs-sdk-python -- name: MatrixConfigs - type: object - default: - - Name: Python_ci_test_base - Path: eng/pipelines/templates/stages/platform-matrix.json - Selection: sparse - GenerateVMJobs: true -- name: AdditionalMatrixConfigs - type: object - default: [] -- name: MatrixFilters - type: object - default: [] -- name: MatrixReplace - type: object - default: [] -- name: VerifyAutorest - type: boolean - default: false -- name: ValidateFormatting - type: boolean - default: false -- name: TestProxy - type: boolean - default: false -- name: GenerateApiReviewForManualOnly - type: boolean - default: false -- name: AdvancedBuild - type: boolean - default: false + - name: ServiceDirectory + type: string + default: not-specified + - name: Artifacts + type: object + default: [] + - name: TestPipeline + type: boolean + default: false + - name: BeforeTestSteps + type: object + default: [] + - name: AfterTestSteps + type: object + default: [] + - name: BeforePublishSteps + type: object + default: [] + - name: TestMarkArgument + type: string + default: '' + - name: BuildTargetingString + type: string + default: azure-* + - name: TestTimeoutInMinutes + type: number + default: 60 + - name: ToxEnvParallel + type: string + default: --tenvparallel + - name: InjectedPackages + type: string + default: '' + - name: BuildDocs + type: boolean + default: true + - name: DevFeedName + type: string + default: public/azure-sdk-for-python + - name: TargetDocRepoOwner + type: string + default: MicrosoftDocs + - name: TargetDocRepoName + type: string + default: azure-docs-sdk-python + - name: MatrixConfigs + type: object + default: + - Name: Python_ci_test_base + Path: eng/pipelines/templates/stages/platform-matrix.json + Selection: sparse + GenerateVMJobs: true + - name: AdditionalMatrixConfigs + type: object + default: [] + - name: MatrixFilters + type: object + default: [] + - name: MatrixReplace + type: object + default: [] + - name: VerifyAutorest + type: boolean + default: false + - name: ValidateFormatting + type: boolean + default: false + - name: TestProxy + type: boolean + default: false + - name: GenerateApiReviewForManualOnly + type: boolean + default: false + - name: AdvancedBuild + type: boolean + default: false + +extends: + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + ${{ else }}: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + sdl: + sourceAnalysisPool: + name: azsdk-pool-mms-win-2022-general + image: azsdk-pool-mms-win-2022-1espt + os: windows + eslint: + enabled: false + justificationForDisabling: "ESLint injected task has failures because it uses an old version of mkdirp. We should not fail for tools not controlled by the repo. See: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=3556850" + codeql: + compiled: + enabled: false + justificationForDisabling: "CodeQL times our pipelines out by running for 2+ hours before being force canceled." + psscriptanalyzer: + enabled: true + break: true + policy: M365 + credscan: + suppressionsFile: '$(Build.SourcesDirectory)/eng/CredScanSuppression.json' + scanFolder: '$(Build.SourcesDirectory)/credscan.tsv' + toolVersion: '2.3.12.23' + baselineFiles: $(Build.SourcesDirectory)/eng/python.gdnbaselines + stages: + - stage: Build + jobs: + - template: /eng/pipelines/templates/jobs/ci.yml@self + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + Artifacts: ${{ parameters.Artifacts }} + ${{ if eq(parameters.ServiceDirectory, 'template') }}: + TestPipeline: true + BeforeTestSteps: ${{ parameters.BeforeTestSteps }} + AfterTestSteps: ${{ parameters.AfterTestSteps }} + BeforePublishSteps: ${{ parameters.BeforePublishSteps }} + TestMarkArgument: ${{ parameters.TestMarkArgument }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} + TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} + ToxEnvParallel: ${{ parameters.ToxEnvParallel }} + InjectedPackages: ${{ parameters.InjectedPackages }} + BuildDocs: ${{ parameters.BuildDocs }} + DevFeedName: ${{ parameters.DevFeedName }} + MatrixConfigs: + - ${{ each config in parameters.MatrixConfigs }}: + - ${{ config }} + - ${{ each config in parameters.AdditionalMatrixConfigs }}: + - ${{ config }} + MatrixFilters: ${{ parameters.MatrixFilters }} + MatrixReplace: ${{ parameters.MatrixReplace }} + VerifyAutorest: ${{ parameters.VerifyAutorest }} + ValidateFormatting: ${{ parameters.ValidateFormatting }} + TestProxy: ${{ parameters.TestProxy }} + GenerateApiReviewForManualOnly: ${{ parameters.GenerateApiReviewForManualOnly }} + AdvancedBuild: ${{ parameters.AdvancedBuild }} + + variables: + - template: /eng/pipelines/templates/variables/globals.yml@self -variables: - - template: /eng/pipelines/templates/variables/globals.yml + # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch. + - ${{if and(in(variables['Build.Reason'], 'Manual', ''), eq(variables['System.TeamProject'], 'internal'))}}: + - template: archetype-python-release.yml@self + parameters: + DependsOn: "Build" + ServiceDirectory: ${{ parameters.ServiceDirectory }} + Artifacts: ${{ parameters.Artifacts }} + ${{ if eq(parameters.ServiceDirectory, 'template') }}: + TestPipeline: true + ArtifactName: packages_extended + DocArtifact: documentation + TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} + TargetDocRepoName: ${{ parameters.TargetDocRepoName }} + DevFeedName: ${{ parameters.DevFeedName }} -stages: - - stage: Build - jobs: - - template: /eng/pipelines/templates/jobs/ci.yml - parameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - Artifacts: ${{ parameters.Artifacts }} - ${{ if eq(parameters.ServiceDirectory, 'template') }}: - TestPipeline: true - BeforeTestSteps: ${{ parameters.BeforeTestSteps }} - AfterTestSteps: ${{ parameters.AfterTestSteps }} - BeforePublishSteps: ${{ parameters.BeforePublishSteps }} - TestMarkArgument: ${{ parameters.TestMarkArgument }} - BuildTargetingString: '${{ parameters.BuildTargetingString }}' - TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} - ToxEnvParallel: ${{ parameters.ToxEnvParallel }} - InjectedPackages: ${{ parameters.InjectedPackages }} - BuildDocs: ${{ parameters.BuildDocs }} - DevFeedName: ${{ parameters.DevFeedName }} - MatrixConfigs: - - ${{ each config in parameters.MatrixConfigs }}: - - ${{ config }} - - ${{ each config in parameters.AdditionalMatrixConfigs }}: - - ${{ config }} - MatrixFilters: ${{ parameters.MatrixFilters }} - MatrixReplace: ${{ parameters.MatrixReplace }} - VerifyAutorest: ${{ parameters.VerifyAutorest }} - ValidateFormatting: ${{ parameters.ValidateFormatting }} - TestProxy: ${{ parameters.TestProxy }} - GenerateApiReviewForManualOnly: ${{ parameters.GenerateApiReviewForManualOnly }} - AdvancedBuild: ${{ parameters.AdvancedBuild }} - # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch. - - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}: - - template: archetype-python-release.yml - parameters: - DependsOn: Build - ServiceDirectory: ${{ parameters.ServiceDirectory }} - Artifacts: ${{ parameters.Artifacts }} - ${{ if eq(parameters.ServiceDirectory, 'template') }}: - TestPipeline: true - ArtifactName: packages_extended - DocArtifact: documentation - TargetDocRepoOwner: ${{ parameters.TargetDocRepoOwner }} - TargetDocRepoName: ${{ parameters.TargetDocRepoName }} - DevFeedName: ${{ parameters.DevFeedName }} diff --git a/eng/pipelines/templates/stages/archetype-sdk-tests.yml b/eng/pipelines/templates/stages/archetype-sdk-tests.yml index 289a9bc2e06e..e58bb471bb17 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-tests.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-tests.yml @@ -1,3 +1,10 @@ +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + parameters: - name: ServiceDirectory type: string @@ -96,62 +103,93 @@ parameters: type: string default: 'whl' -stages: -- ${{ each cloud in parameters.CloudConfig }}: - - ${{ if or(contains(parameters.Clouds, cloud.key), and(contains(variables['Build.DefinitionName'], 'tests-weekly'), contains(parameters.SupportedClouds, cloud.key))) }}: - - ${{ if not(contains(parameters.UnsupportedClouds, cloud.key)) }}: - - stage: ${{ cloud.key }}_${{ parameters.JobName }} - dependsOn: [] - jobs: - - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml - parameters: - SparseCheckoutPaths: - # Python recording files are implicit excluded here since they are using '.yaml' file extension. - - "sdk/${{ parameters.ServiceDirectory }}/**/*.json" - JobTemplatePath: /eng/pipelines/templates/jobs/live.tests.yml - AdditionalParameters: - ServiceDirectory: ${{ parameters.ServiceDirectory }} - TestResourceDirectories: ${{ parameters.TestResourceDirectories }} - PreSteps: - - ${{ parameters.PlatformPreSteps }} - - ${{ parameters.PreSteps }} - PostSteps: - - ${{ parameters.PlatformPostSteps }} - - ${{ parameters.PostSteps }} - EnvVars: ${{ parameters.EnvVars }} - MaxParallel: ${{ parameters.MaxParallel }} - BeforeTestSteps: ${{ parameters.BeforeTestSteps }} - AfterTestSteps: ${{ parameters.AfterTestSteps }} - AdditionalTestArgs: ${{ parameters.AdditionalTestArgs }} - BuildTargetingString: ${{ parameters.BuildTargetingString }} - TestMarkArgument: ${{ parameters.TestMarkArgument }} - InjectedPackages: ${{ parameters.InjectedPackages }} - BuildDocs: ${{ parameters.BuildDocs }} - TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} - TestProxy: ${{ parameters.TestProxy }} - ToxTestEnv: ${{ parameters.ToxTestEnv }} - MatrixConfigs: - # Enumerate platforms and additional platforms based on supported clouds (sparse platform<-->cloud matrix). - - ${{ each config in parameters.MatrixConfigs }}: - - ${{ config }} - - ${{ each config in parameters.AdditionalMatrixConfigs }}: - - ${{ config }} - MatrixFilters: - - ${{ each cloudFilter in cloud.value.MatrixFilters }}: - - ${{ cloudFilter }} - - ${{ parameters.MatrixFilters }} - MatrixReplace: - - ${{ each cloudReplace in cloud.value.MatrixReplace }}: - - ${{ cloudReplace }} - - ${{ parameters.MatrixReplace }} - CloudConfig: - SubscriptionConfiguration: ${{ cloud.value.SubscriptionConfiguration }} - SubscriptionConfigurations: ${{ cloud.value.SubscriptionConfigurations }} - Location: ${{ coalesce(parameters.Location, cloud.value.Location) }} - Cloud: ${{ cloud.key }} - -- template: /eng/pipelines/templates/stages/python-analyze-weekly.yml +extends: + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + ${{ else }}: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates parameters: - BuildTargetingString: ${{ parameters.BuildTargetingString }} - ServiceDirectory: ${{ parameters.ServiceDirectory }} - JobName: ${{ parameters.JobName }} + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + sdl: + sourceAnalysisPool: + name: azsdk-pool-mms-win-2022-general + image: azsdk-pool-mms-win-2022-1espt + os: windows + eslint: + enabled: false + justificationForDisabling: 'ESLint injected task has failures because it uses an old version of mkdirp. We should not fail for tools not controlled by the repo. See: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=3499746' + codeql: + compiled: + enabled: false + justificationForDisabling: CodeQL times our pipelines out by running for 2+ hours before being force canceled. + psscriptanalyzer: + compiled: true + break: true + policy: M365 + credscan: + suppressionsFile: $(Build.SourcesDirectory)/eng/CredScanSuppression.json + scanFolder: $(Build.SourcesDirectory)/credscan.tsv + toolVersion: 2.3.12.23 + baselineFiles: $(Build.SourcesDirectory)/eng/.gdnbaselines + + stages: + - ${{ each cloud in parameters.CloudConfig }}: + - ${{ if or(contains(parameters.Clouds, cloud.key), and(contains(variables['Build.DefinitionName'], 'tests-weekly'), contains(parameters.SupportedClouds, cloud.key))) }}: + - ${{ if not(contains(parameters.UnsupportedClouds, cloud.key)) }}: + - stage: ${{ cloud.key }}_${{ parameters.JobName }} + dependsOn: [] + jobs: + - template: /eng/common/pipelines/templates/jobs/generate-job-matrix.yml@self + parameters: + SparseCheckoutPaths: + - sdk/${{ parameters.ServiceDirectory }}/**/*.json + JobTemplatePath: /eng/pipelines/templates/jobs/live.tests.yml + OsVmImage: azsdk-pool-mms-ubuntu-2004-1espt + Pool: azsdk-pool-mms-ubuntu-2004-general + AdditionalParameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + TestResourceDirectories: ${{ parameters.TestResourceDirectories }} + PreSteps: + - ${{ parameters.PlatformPreSteps }} + - ${{ parameters.PreSteps }} + PostSteps: + - ${{ parameters.PlatformPostSteps }} + - ${{ parameters.PostSteps }} + EnvVars: ${{ parameters.EnvVars }} + MaxParallel: ${{ parameters.MaxParallel }} + BeforeTestSteps: ${{ parameters.BeforeTestSteps }} + AfterTestSteps: ${{ parameters.AfterTestSteps }} + AdditionalTestArgs: ${{ parameters.AdditionalTestArgs }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} + TestMarkArgument: ${{ parameters.TestMarkArgument }} + InjectedPackages: ${{ parameters.InjectedPackages }} + BuildDocs: ${{ parameters.BuildDocs }} + TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} + TestProxy: ${{ parameters.TestProxy }} + ToxTestEnv: ${{ parameters.ToxTestEnv }} + MatrixConfigs: + # Enumerate platforms and additional platforms based on supported clouds (sparse platform<-->cloud matrix). + - ${{ each config in parameters.MatrixConfigs }}: + - ${{ config }} + - ${{ each config in parameters.AdditionalMatrixConfigs }}: + - ${{ config }} + MatrixFilters: + - ${{ each cloudFilter in cloud.value.MatrixFilters }}: + - ${{ cloudFilter }} + - ${{ parameters.MatrixFilters }} + MatrixReplace: + - ${{ each cloudReplace in cloud.value.MatrixReplace }}: + - ${{ cloudReplace }} + - ${{ parameters.MatrixReplace }} + CloudConfig: + SubscriptionConfiguration: ${{ cloud.value.SubscriptionConfiguration }} + SubscriptionConfigurations: ${{ cloud.value.SubscriptionConfigurations }} + Location: ${{ coalesce(parameters.Location, cloud.value.Location) }} + Cloud: ${{ cloud.key }} + + - template: /eng/pipelines/templates/stages/python-analyze-weekly.yml@self + parameters: + BuildTargetingString: ${{ parameters.BuildTargetingString }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + JobName: ${{ parameters.JobName }} + diff --git a/eng/pipelines/templates/stages/conda-sdk-client.yml b/eng/pipelines/templates/stages/conda-sdk-client.yml index 927d2c78c3d6..49d482e444bd 100644 --- a/eng/pipelines/templates/stages/conda-sdk-client.yml +++ b/eng/pipelines/templates/stages/conda-sdk-client.yml @@ -1,742 +1,778 @@ +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + parameters: -- name: release_msrest - displayName: 'msrest' - type: boolean - default: true -- name: release_msal - displayName: 'msal' - type: boolean - default: true -- name: release_uamqp - displayName: 'uamqp' - type: boolean - default: true -- name: release_msal_extensions - displayName: 'msal-extensions' - type: boolean - default: true -- name: release_azure_core - displayName: 'azure-core' - type: boolean - default: true -- name: release_azure_identity - displayName: 'azure-identity' - type: boolean - default: true -- name: release_azure_storage - displayName: 'azure-storage' - type: boolean - default: true -- name: release_azure_ai_formrecognizer - displayName: 'azure-ai-formrecognizer' - type: boolean - default: true -- name: release_azure_ai_language_conversations - displayName: 'azure-ai-language-conversations' - type: boolean - default: true -- name: release_azure_ai_language_questionanswering - displayName: 'azure-ai-language-questionanswering' - type: boolean - default: true -- name: release_azure_ai_metricsadvisor - displayName: 'azure-ai-metricsadvisor' - type: boolean - default: true -- name: release_azure_ai_ml - displayName: 'azure-ai-ml' - type: boolean - default: true -- name: release_azure_ai_textanalytics - displayName: 'azure-ai-textanalytics' - type: boolean - default: true -- name: release_azure_ai_translation_document - displayName: 'azure-ai-translation-document' - type: boolean - default: true -- name: release_azure_appconfiguration - displayName: 'azure-appconfiguration' - type: boolean - default: true -- name: release_azure_communication - displayName: 'azure-communication' - type: boolean - default: true -- name: release_azure_confidentialledger - displayName: 'azure-confidentialledger' - type: boolean - default: true -- name: release_azure_containerregistry - displayName: 'azure-containerregistry' - type: boolean - default: true -- name: release_azure_cosmos - displayName: 'azure-cosmos' - type: boolean - default: true -- name: release_azure_data_tables - displayName: 'azure-data-tables' - type: boolean - default: true -- name: release_azure_developer_loadtesting - displayName: 'azure-developer-loadtesting' - type: boolean - default: true -- name: release_azure_digitaltwins_core - displayName: 'azure-digitaltwins-core' - type: boolean - default: true -- name: release_azure_eventgrid - displayName: 'azure-eventgrid' - type: boolean - default: true -- name: release_azure_eventhub - displayName: 'azure-eventhub' - type: boolean - default: true -- name: release_azure_iot_deviceupdate - displayName: 'azure-iot-deviceupdate' - type: boolean - default: true -- name: release_azure_keyvault - displayName: 'azure-keyvault' - type: boolean - default: true -- name: release_azure_messaging_webpubsubservice - displayName: 'azure-messaging-webpubsubservice' - type: boolean - default: true -- name: release_azure_monitor_query - displayName: 'azure-monitor-query' - type: boolean - default: true -- name: release_azure_monitor_ingestion - displayName: 'azure-monitor-ingestion' - type: boolean - default: true -- name: release_azure_schemaregistry - displayName: 'azure-schemaregistry' - type: boolean - default: true -- name: release_azure_search_documents - displayName: 'azure-search-documents' - type: boolean - default: true -- name: release_azure_security_attestation - displayName: 'azure-security-attestation' - type: boolean - default: true -- name: release_azure_servicebus - displayName: 'azure-servicebus' - type: boolean - default: true -- name: release_azure_mgmt - displayName: 'azure-mgmt' - type: boolean - default: true + - name: release_msrest + displayName: 'msrest' + type: boolean + default: true + - name: release_msal + displayName: 'msal' + type: boolean + default: true + - name: release_uamqp + displayName: 'uamqp' + type: boolean + default: true + - name: release_msal_extensions + displayName: 'msal-extensions' + type: boolean + default: true + - name: release_azure_core + displayName: 'azure-core' + type: boolean + default: true + - name: release_azure_identity + displayName: 'azure-identity' + type: boolean + default: true + - name: release_azure_storage + displayName: 'azure-storage' + type: boolean + default: true + - name: release_azure_ai_formrecognizer + displayName: 'azure-ai-formrecognizer' + type: boolean + default: true + - name: release_azure_ai_language_conversations + displayName: 'azure-ai-language-conversations' + type: boolean + default: true + - name: release_azure_ai_language_questionanswering + displayName: 'azure-ai-language-questionanswering' + type: boolean + default: true + - name: release_azure_ai_metricsadvisor + displayName: 'azure-ai-metricsadvisor' + type: boolean + default: true + - name: release_azure_ai_ml + displayName: 'azure-ai-ml' + type: boolean + default: true + - name: release_azure_ai_textanalytics + displayName: 'azure-ai-textanalytics' + type: boolean + default: true + - name: release_azure_ai_translation_document + displayName: 'azure-ai-translation-document' + type: boolean + default: true + - name: release_azure_appconfiguration + displayName: 'azure-appconfiguration' + type: boolean + default: true + - name: release_azure_communication + displayName: 'azure-communication' + type: boolean + default: true + - name: release_azure_confidentialledger + displayName: 'azure-confidentialledger' + type: boolean + default: true + - name: release_azure_containerregistry + displayName: 'azure-containerregistry' + type: boolean + default: true + - name: release_azure_cosmos + displayName: 'azure-cosmos' + type: boolean + default: true + - name: release_azure_data_tables + displayName: 'azure-data-tables' + type: boolean + default: true + - name: release_azure_developer_loadtesting + displayName: 'azure-developer-loadtesting' + type: boolean + default: true + - name: release_azure_digitaltwins_core + displayName: 'azure-digitaltwins-core' + type: boolean + default: true + - name: release_azure_eventgrid + displayName: 'azure-eventgrid' + type: boolean + default: true + - name: release_azure_eventhub + displayName: 'azure-eventhub' + type: boolean + default: true + - name: release_azure_iot_deviceupdate + displayName: 'azure-iot-deviceupdate' + type: boolean + default: true + - name: release_azure_keyvault + displayName: 'azure-keyvault' + type: boolean + default: true + - name: release_azure_messaging_webpubsubservice + displayName: 'azure-messaging-webpubsubservice' + type: boolean + default: true + - name: release_azure_monitor_query + displayName: 'azure-monitor-query' + type: boolean + default: true + - name: release_azure_monitor_ingestion + displayName: 'azure-monitor-ingestion' + type: boolean + default: true + - name: release_azure_schemaregistry + displayName: 'azure-schemaregistry' + type: boolean + default: true + - name: release_azure_search_documents + displayName: 'azure-search-documents' + type: boolean + default: true + - name: release_azure_security_attestation + displayName: 'azure-security-attestation' + type: boolean + default: true + - name: release_azure_servicebus + displayName: 'azure-servicebus' + type: boolean + default: true + - name: release_azure_mgmt + displayName: 'azure-mgmt' + type: boolean + default: true + +extends: + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + ${{ else }}: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + sdl: + sourceAnalysisPool: + name: azsdk-pool-mms-win-2022-1es-pt + image: azsdk-pool-mms-win-2022-1espt + os: windows + eslint: + enabled: false + justificationForDisabling: 'ESLint injected task has failures because it uses an old version of mkdirp. We should not fail for tools not controlled by the repo. See: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=3499746' + codeql: + compiled: + enabled: false + justificationForDisabling: CodeQL times our pipelines out by running for 2+ hours before being force canceled. + psscriptanalyzer: + compiled: true + break: true + policy: M365 + credscan: + suppressionsFile: $(Build.SourcesDirectory)/eng/CredScanSuppression.json + scanFolder: $(Build.SourcesDirectory)/credscan.tsv + toolVersion: 2.3.12.23 + baselineFiles: $(Build.SourcesDirectory)/eng/python.gdnbaselines + + stages: + - stage: Build_Dependencies + displayName: Build Platform-Specific Binary Conda Packages + variables: + - template: /eng/pipelines/templates/variables/globals.yml@self + + jobs: + - template: eng/pipelines/templates/jobs/build-conda-dependencies.yml@self + parameters: + CondaArtifacts: + - name: uamqp + common_root: uamqp + in_batch: true + conda_py_versions: + - 38 + - 39 + - 310 + - 311 + checkout: + - package: uamqp + download_uri: https://files.pythonhosted.org/packages/0b/d8/fc24d95e6f6c80851ae6738c78da081cd535c924b02c5a4928b108b9ed42/uamqp-1.6.5.tar.gz + + - stage: Build_Universal_Dependencies + displayName: Build Universal Conda Packages + dependsOn: Build_Dependencies + variables: + - template: /eng/pipelines/templates/variables/globals.yml@self -variables: - - template: /eng/pipelines/templates/variables/globals.yml + jobs: + - job: 'Build' + timeoutInMinutes: 240 -stages: - - stage: Build_Dependencies - displayName: Build Platform-Specific Binary Conda Packages - jobs: - - template: ../jobs/build-conda-dependencies.yml - parameters: - CondaArtifacts: - - name: uamqp - common_root: uamqp - in_batch: true - conda_py_versions: - - "38" - - "39" - - "310" - - "311" - checkout: - - package: uamqp - download_uri: https://files.pythonhosted.org/packages/0b/d8/fc24d95e6f6c80851ae6738c78da081cd535c924b02c5a4928b108b9ed42/uamqp-1.6.5.tar.gz + pool: + name: azsdk-pool-mms-ubuntu-2004-general + image: azsdk-pool-mms-ubuntu-2004-1espt + os: linux - - stage: Build_Universal_Dependencies - displayName: Build Universal Conda Packages - dependsOn: Build_Dependencies - jobs: - - job: 'Build' - timeoutInMinutes: 240 - - pool: - name: azsdk-pool-mms-ubuntu-2004-general - vmImage: MMSUbuntu20.04 + steps: + - download: current + artifact: windows_conda + timeoutInMinutes: 5 - steps: - - download: current - artifact: windows_conda - timeoutInMinutes: 5 + - download: current + artifact: linux_conda + timeoutInMinutes: 5 - - download: current - artifact: linux_conda - timeoutInMinutes: 5 + - download: current + artifact: mac_conda + timeoutInMinutes: 5 - - download: current - artifact: mac_conda - timeoutInMinutes: 5 + - template: /eng/pipelines/templates/steps/build-conda-artifacts.yml@self + parameters: + Arguments: '--channel "$(Pipeline.Workspace)/mac_conda" "$(Pipeline.Workspace)/windows_conda" "$(Pipeline.Workspace)/linux_conda"' - - template: ../steps/build-conda-artifacts.yml - parameters: - Arguments: '--channel "$(Pipeline.Workspace)/mac_conda" "$(Pipeline.Workspace)/windows_conda" "$(Pipeline.Workspace)/linux_conda"' + CondaArtifacts: + - name: azure-core + common_root: azure + service: core + in_batch: ${{ parameters.release_azure_core }} + checkout: + - package: azure-core + version: 1.30.0 + - package: azure-mgmt-core + version: 1.4.0 + - package: azure-common + version: 1.1.28 + - name: msrest + in_batch: true + checkout: + - package: msrest + download_uri: https://files.pythonhosted.org/packages/68/77/8397c8fb8fc257d8ea0fa66f8068e073278c65f05acb17dcb22a02bfdc42/msrest-0.7.1.zip + - name: msal + in_batch: true + checkout: + - package: msal + download_uri: https://files.pythonhosted.org/packages/bb/45/c4dfbe24dd546d141287fa26476ce3206d461d8e4a24be77c84b835e647d/msal-1.26.0.tar.gz + - name: msal-extensions + common_root: msal + in_batch: true + checkout: + - package: msal-extensions + download_uri: https://files.pythonhosted.org/packages/cb/ba/618771542cdc4bc5314c395076c397d67e2bdcd88564c6ca712a2497d1c6/msal-extensions-1.1.0.tar.gz + - name: azure-identity + service: identity + in_batch: ${{ parameters.release_azure_identity }} + checkout: + - package: azure-identity + version: 1.15.0 + - name: azure-storage + common_root: azure/storage + in_batch: ${{ parameters.release_azure_storage }} + service: storage + checkout: + - package: azure-storage-blob + version: 12.19.0 + - package: azure-storage-queue + version: 12.9.0 + - package: azure-storage-file-share + version: 12.15.0 + - package: azure-storage-file-datalake + version: 12.14.0 + - name: azure-ai-ml + service: ml + in_batch: ${{ parameters.release_azure_ai_ml }} + channels: + - conda-forge + checkout: + - package: azure-ai-ml + version: 1.13.0 + - name: azure-ai-formrecognizer + common_root: azure + service: formrecognizer + in_batch: ${{ parameters.release_azure_ai_formrecognizer }} + checkout: + - package: azure-ai-formrecognizer + version: 3.3.2 + - name: azure-ai-language-conversations + common_root: azure + service: cognitivelanguage + in_batch: ${{ parameters.release_azure_ai_language_conversations }} + checkout: + - package: azure-ai-language-conversations + version: 1.1.0 + - name: azure-ai-language-questionanswering + service: cognitivelanguage + in_batch: ${{ parameters.release_azure_ai_language_questionanswering }} + checkout: + - package: azure-ai-language-questionanswering + version: 1.1.0 + - name: azure-ai-metricsadvisor + service: cognitivelanguage + in_batch: ${{ parameters.release_azure_ai_metricsadvisor }} + checkout: + - package: azure-ai-metricsadvisor + version: 1.0.0 + - name: azure-ai-textanalytics + service: textanalytics + in_batch: ${{ parameters.release_azure_ai_textanalytics }} + checkout: + - package: azure-ai-textanalytics + version: 5.3.0 + - name: azure-ai-translation-document + service: translation + in_batch: ${{ parameters.release_azure_ai_translation_document }} + checkout: + - package: azure-ai-translation-document + version: 1.0.0 + - name: azure-appconfiguration + service: appconfiguration + in_batch: ${{ parameters.release_azure_appconfiguration }} + checkout: + - package: azure-appconfiguration + version: 1.5.0 + - name: azure-communication + service: communication + common_root: azure/communication + in_batch: ${{ parameters.release_azure_communication }} + checkout: + - package: azure-communication-chat + version: 1.2.0 + - package: azure-communication-email + version: 1.0.0 + - package: azure-communication-identity + version: 1.5.0 + - package: azure-communication-phonenumbers + version: 1.1.0 + - package: azure-communication-sms + version: 1.0.1 + - package: azure-communication-rooms + version: 1.0.0 + - package: azure-communication-jobrouter + version: 1.0.0 + - package: azure-communication-callautomation + version: 1.1.0 + - name: azure-confidentialledger + service: confidentialledger + in_batch: ${{ parameters.release_azure_confidentialledger }} + checkout: + - package: azure-confidentialledger + version: 1.1.1 + - name: azure-containerregistry + service: containerregistry + in_batch: ${{ parameters.release_azure_containerregistry }} + checkout: + - package: azure-containerregistry + version: 1.2.0 + - name: azure-cosmos + service: cosmos + in_batch: ${{ parameters.release_azure_cosmos }} + checkout: + - package: azure-cosmos + version: 4.5.1 + - name: azure-data-tables + service: tables + in_batch: ${{ parameters.release_azure_data_tables }} + checkout: + - package: azure-data-tables + version: 12.5.0 + - name: azure-developer-loadtesting + service: loadtesting + in_batch: ${{ parameters.release_azure_developer_loadtesting }} + checkout: + - package: azure-developer-loadtesting + version: 1.0.0 + - name: azure-digitaltwins-core + service: digitaltwins + in_batch: ${{ parameters.release_azure_digitaltwins_core }} + checkout: + - package: azure-digitaltwins-core + version: 1.2.0 + - name: azure-eventgrid + service: eventgrid + in_batch: ${{ parameters.release_azure_eventgrid }} + checkout: + - package: azure-eventgrid + version: 4.17.0 + - name: azure-eventhub + service: eventhub + common_root: azure/eventhub + in_batch: ${{ parameters.release_azure_eventhub }} + checkout: + - package: azure-eventhub-checkpointstoreblob + version: 1.1.4 + - package: azure-eventhub-checkpointstoreblob-aio + version: 1.1.4 + - package: azure-eventhub + version: 5.11.6 + - name: azure-iot-deviceupdate + service: deviceupdate + in_batch: ${{ parameters.release_azure_iot_deviceupdate }} + checkout: + - package: azure-iot-deviceupdate + version: 1.0.0 + - name: azure-keyvault + service: keyvault + common_root: azure/keyvault + in_batch: ${{ parameters.release_azure_keyvault }} + checkout: + - package: azure-keyvault-administration + version: 4.4.0 + - package: azure-keyvault-certificates + version: 4.8.0 + - package: azure-keyvault-keys + version: 4.9.0 + - package: azure-keyvault-secrets + version: 4.8.0 + - name: azure-messaging-webpubsubservice + service: webpubsub + in_batch: ${{ parameters.release_azure_messaging_webpubsubservice }} + checkout: + - package: azure-messaging-webpubsubservice + version: 1.0.1 + - name: azure-monitor-ingestion + service: monitor + in_batch: ${{ parameters.release_azure_monitor_ingestion }} + checkout: + - package: azure-monitor-ingestion + version: 1.0.3 + - name: azure-monitor-query + service: monitor + in_batch: ${{ parameters.release_azure_monitor_query }} + checkout: + - package: azure-monitor-query + version: 1.2.1 + - name: azure-schemaregistry + service: schemaregistry + common_root: azure/schemaregistry + in_batch: ${{ parameters.release_azure_schemaregistry }} + checkout: + - package: azure-schemaregistry + version: 1.2.0 + - package: azure-schemaregistry-avroencoder + version: 1.0.0 + - name: azure-search-documents + service: search + in_batch: ${{ parameters.release_azure_search_documents }} + checkout: + - package: azure-search-documents + version: 11.4.0 + - name: azure-security-attestation + service: attestation + in_batch: ${{ parameters.release_azure_security_attestation }} + checkout: + - package: azure-security-attestation + version: 1.0.0 + - name: azure-servicebus + service: servicebus + in_batch: ${{ parameters.release_azure_servicebus }} + checkout: + - package: azure-servicebus + version: 7.11.4 + - name: azure-mgmt + service: mgmt + in_batch: ${{ parameters.release_azure_mgmt }} + common_root: azure/mgmt + checkout: + - package: azure-mgmt-advisor + version: 9.0.0 + - package: azure-mgmt-alertsmanagement + version: 1.0.0 + - package: azure-mgmt-apicenter + version: 1.0.0 + - package: azure-mgmt-apimanagement + version: 4.0.0 + - package: azure-mgmt-appconfiguration + version: 3.0.0 + - package: azure-mgmt-appcontainers + version: 3.0.0 + - package: azure-mgmt-applicationinsights + version: 4.0.0 + - package: azure-mgmt-appplatform + version: 9.0.0 + - package: azure-mgmt-attestation + version: 1.0.0 + - package: azure-mgmt-authorization + version: 4.0.0 + - package: azure-mgmt-automanage + version: 1.0.0 + - package: azure-mgmt-automation + version: 1.0.0 + - package: azure-mgmt-avs + version: 8.0.0 + - package: azure-mgmt-azurearcdata + version: 1.0.0 + - package: azure-mgmt-azurestack + version: 1.0.0 + - package: azure-mgmt-azurestackhci + version: 7.0.0 + - package: azure-mgmt-baremetalinfrastructure + version: 1.0.0 + - package: azure-mgmt-batch + version: 17.2.0 + - package: azure-mgmt-billing + version: 6.0.0 + - package: azure-mgmt-botservice + version: 2.0.0 + - package: azure-mgmt-cdn + version: 13.0.0 + - package: azure-mgmt-changeanalysis + version: 1.0.0 + - package: azure-mgmt-chaos + version: 1.0.0 + - package: azure-mgmt-cognitiveservices + version: 13.5.0 + - package: azure-mgmt-commerce + version: 6.0.0 + - package: azure-mgmt-communication + version: 2.0.0 + - package: azure-mgmt-compute + version: 30.5.0 + - package: azure-mgmt-confidentialledger + version: 1.0.0 + - package: azure-mgmt-confluent + version: 2.0.0 + - package: azure-mgmt-connectedvmware + version: 1.0.0 + - package: azure-mgmt-consumption + version: 10.0.0 + - package: azure-mgmt-containerinstance + version: 10.1.0 + - package: azure-mgmt-containerregistry + version: 10.3.0 + - package: azure-mgmt-containerservice + version: 29.1.0 + - package: azure-mgmt-containerservicefleet + version: 1.0.0 + - package: azure-mgmt-cosmosdb + version: 9.4.0 + - package: azure-mgmt-costmanagement + version: 4.0.1 + - package: azure-mgmt-customproviders + version: 1.0.0 + - package: azure-mgmt-dashboard + version: 1.1.0 + - package: azure-mgmt-databox + version: 2.0.0 + - package: azure-mgmt-databoxedge + version: 1.0.0 + - package: azure-mgmt-databricks + version: 2.0.0 + - package: azure-mgmt-datadog + version: 2.1.0 + - package: azure-mgmt-datafactory + version: 5.0.0 + - package: azure-mgmt-datamigration + version: 10.0.0 + - package: azure-mgmt-dataprotection + version: 1.3.0 + - package: azure-mgmt-datashare + version: 1.0.0 + - package: azure-mgmt-deploymentmanager + version: 1.0.0 + - package: azure-mgmt-desktopvirtualization + version: 1.1.0 + - package: azure-mgmt-devcenter + version: 1.0.0 + - package: azure-mgmt-deviceupdate + version: 1.1.0 + - package: azure-mgmt-devtestlabs + version: 9.0.0 + - package: azure-mgmt-digitaltwins + version: 6.4.0 + - package: azure-mgmt-dns + version: 8.1.0 + - package: azure-mgmt-dnsresolver + version: 1.0.0 + - package: azure-mgmt-dynatrace + version: 2.0.0 + - package: azure-mgmt-edgeorder + version: 1.0.0 + - package: azure-mgmt-elastic + version: 1.0.0 + - package: azure-mgmt-elasticsan + version: 1.0.0 + - package: azure-mgmt-eventgrid + version: 10.2.0 + - package: azure-mgmt-eventhub + version: 11.0.0 + - package: azure-mgmt-extendedlocation + version: 1.1.0 + - package: azure-mgmt-fluidrelay + version: 1.0.0 + - package: azure-mgmt-frontdoor + version: 1.1.0 + - package: azure-mgmt-graphservices + version: 1.0.0 + - package: azure-mgmt-hanaonazure + version: 1.0.0 + - package: azure-mgmt-hdinsight + version: 9.0.0 + - package: azure-mgmt-healthcareapis + version: 2.0.0 + - package: azure-mgmt-hybridcompute + version: 8.0.0 + - package: azure-mgmt-hybridconnectivity + version: 1.0.0 + - package: azure-mgmt-hybridcontainerservice + version: 1.0.0 + - package: azure-mgmt-hybridkubernetes + version: 1.1.0 + - package: azure-mgmt-hybridnetwork + version: 2.0.0 + - package: azure-mgmt-imagebuilder + version: 1.3.0 + - package: azure-mgmt-iothub + version: 3.0.0 + - package: azure-mgmt-iothubprovisioningservices + version: 1.1.0 + - package: azure-mgmt-keyvault + version: 10.3.0 + - package: azure-mgmt-kubernetesconfiguration + version: 3.1.0 + - package: azure-mgmt-kusto + version: 3.3.0 + - package: azure-mgmt-labservices + version: 2.0.0 + - package: azure-mgmt-loadtesting + version: 1.0.0 + - package: azure-mgmt-loganalytics + version: 12.0.0 + - package: azure-mgmt-logic + version: 10.0.0 + - package: azure-mgmt-logz + version: 1.0.0 + - package: azure-mgmt-machinelearningservices + version: 1.0.0 + - package: azure-mgmt-maintenance + version: 2.1.0 + - package: azure-mgmt-managednetworkfabric + version: 1.0.0 + - package: azure-mgmt-managedservices + version: 6.0.0 + - package: azure-mgmt-managementgroups + version: 1.0.0 + - package: azure-mgmt-managementpartner + version: 1.0.0 + - package: azure-mgmt-maps + version: 2.1.0 + - package: azure-mgmt-marketplaceordering + version: 1.1.0 + - package: azure-mgmt-media + version: 10.2.0 + - package: azure-mgmt-mixedreality + version: 1.0.0 + - package: azure-mgmt-mobilenetwork + version: 3.1.0 + - package: azure-mgmt-monitor + version: 6.0.2 + - package: azure-mgmt-msi + version: 7.0.0 + - package: azure-mgmt-netapp + version: 11.0.0 + - package: azure-mgmt-network + version: 25.3.0 + - package: azure-mgmt-networkanalytics + version: 1.0.0 + - package: azure-mgmt-networkcloud + version: 1.0.0 + - package: azure-mgmt-newrelicobservability + version: 1.0.0 + - package: azure-mgmt-nginx + version: 3.0.0 + - package: azure-mgmt-notificationhubs + version: 8.0.0 + - package: azure-mgmt-operationsmanagement + version: 1.0.0 + - package: azure-mgmt-orbital + version: 2.0.0 + - package: azure-mgmt-paloaltonetworksngfw + version: 1.0.0 + - package: azure-mgmt-peering + version: 1.0.0 + - package: azure-mgmt-policyinsights + version: 1.0.0 + - package: azure-mgmt-portal + version: 1.0.0 + - package: azure-mgmt-powerbidedicated + version: 1.0.0 + - package: azure-mgmt-privatedns + version: 1.1.0 + - package: azure-mgmt-purview + version: 1.0.0 + - package: azure-mgmt-qumulo + version: 1.0.0 + - package: azure-mgmt-quota + version: 1.1.0 + - package: azure-mgmt-rdbms + version: 10.1.0 + - package: azure-mgmt-recoveryservices + version: 2.5.0 + - package: azure-mgmt-recoveryservicesbackup + version: 9.0.0 + - package: azure-mgmt-recoveryservicessiterecovery + version: 1.2.0 + - package: azure-mgmt-redhatopenshift + version: 1.4.0 + - package: azure-mgmt-redis + version: 14.3.0 + - package: azure-mgmt-redisenterprise + version: 2.0.0 + - package: azure-mgmt-relay + version: 1.1.0 + - package: azure-mgmt-reservations + version: 2.3.0 + - package: azure-mgmt-resource + version: 23.0.1 + - package: azure-mgmt-resourceconnector + version: 1.0.0 + - package: azure-mgmt-resourcemover + version: 1.1.0 + - package: azure-mgmt-search + version: 9.1.0 + - package: azure-mgmt-security + version: 6.0.0 + - package: azure-mgmt-securityinsight + version: 1.0.0 + - package: azure-mgmt-selfhelp + version: 1.0.0 + - package: azure-mgmt-serialconsole + version: 1.0.0 + - package: azure-mgmt-servicebus + version: 8.2.0 + - package: azure-mgmt-servicefabric + version: 2.1.0 + - package: azure-mgmt-servicelinker + version: 1.1.0 + - package: azure-mgmt-servicenetworking + version: 1.0.0 + - package: azure-mgmt-signalr + version: 1.2.0 + - package: azure-mgmt-sql + version: 3.0.1 + - package: azure-mgmt-storage + version: 21.1.0 + - package: azure-mgmt-storagecache + version: 1.5.0 + - package: azure-mgmt-storagemover + version: 2.0.0 + - package: azure-mgmt-storagepool + version: 1.0.0 + - package: azure-mgmt-storagesync + version: 1.0.0 + - package: azure-mgmt-streamanalytics + version: 1.0.0 + - package: azure-mgmt-subscription + version: 3.1.1 + - package: azure-mgmt-support + version: 6.0.0 + - package: azure-mgmt-synapse + version: 2.0.0 + - package: azure-mgmt-timeseriesinsights + version: 1.0.0 + - package: azure-mgmt-trafficmanager + version: 1.1.0 + - package: azure-mgmt-voiceservices + version: 1.0.0 + - package: azure-mgmt-web + version: 7.2.0 + - package: azure-mgmt-webpubsub + version: 1.1.0 + - package: azure-mgmt-workloads + version: 1.0.0 - CondaArtifacts: - - name: azure-core - common_root: azure - service: core - in_batch: ${{ parameters.release_azure_core }} - checkout: - - package: azure-core - version: 1.30.0 - - package: azure-mgmt-core - version: 1.4.0 - - package: azure-common - version: 1.1.28 - - name: msrest - in_batch: true - checkout: - - package: msrest - download_uri: https://files.pythonhosted.org/packages/68/77/8397c8fb8fc257d8ea0fa66f8068e073278c65f05acb17dcb22a02bfdc42/msrest-0.7.1.zip - - name: msal - in_batch: true - checkout: - - package: msal - download_uri: https://files.pythonhosted.org/packages/bb/45/c4dfbe24dd546d141287fa26476ce3206d461d8e4a24be77c84b835e647d/msal-1.26.0.tar.gz - - name: msal-extensions - common_root: msal - in_batch: true - checkout: - - package: msal-extensions - download_uri: https://files.pythonhosted.org/packages/cb/ba/618771542cdc4bc5314c395076c397d67e2bdcd88564c6ca712a2497d1c6/msal-extensions-1.1.0.tar.gz - - name: azure-identity - service: identity - in_batch: ${{ parameters.release_azure_identity }} - checkout: - - package: azure-identity - version: 1.15.0 - - name: azure-storage - common_root: azure/storage - in_batch: ${{ parameters.release_azure_storage }} - service: storage - checkout: - - package: azure-storage-blob - version: 12.19.0 - - package: azure-storage-queue - version: 12.9.0 - - package: azure-storage-file-share - version: 12.15.0 - - package: azure-storage-file-datalake - version: 12.14.0 - - name: azure-ai-ml - service: ml - in_batch: ${{ parameters.release_azure_ai_ml }} - channels: - - "conda-forge" - checkout: - - package: azure-ai-ml - version: 1.13.0 - - name: azure-ai-formrecognizer - common_root: azure - service: formrecognizer - in_batch: ${{ parameters.release_azure_ai_formrecognizer }} - checkout: - - package: azure-ai-formrecognizer - version: 3.3.2 - - name: azure-ai-language-conversations - common_root: azure - service: cognitivelanguage - in_batch: ${{ parameters.release_azure_ai_language_conversations }} - checkout: - - package: azure-ai-language-conversations - version: 1.1.0 - - name: azure-ai-language-questionanswering - service: cognitivelanguage - in_batch: ${{ parameters.release_azure_ai_language_questionanswering }} - checkout: - - package: azure-ai-language-questionanswering - version: 1.1.0 - - name: azure-ai-metricsadvisor - service: cognitivelanguage - in_batch: ${{ parameters.release_azure_ai_metricsadvisor }} - checkout: - - package: azure-ai-metricsadvisor - version: 1.0.0 - - name: azure-ai-textanalytics - service: textanalytics - in_batch: ${{ parameters.release_azure_ai_textanalytics }} - checkout: - - package: azure-ai-textanalytics - version: 5.3.0 - - name: azure-ai-translation-document - service: translation - in_batch: ${{ parameters.release_azure_ai_translation_document }} - checkout: - - package: azure-ai-translation-document - version: 1.0.0 - - name: azure-appconfiguration - service: appconfiguration - in_batch: ${{ parameters.release_azure_appconfiguration }} - checkout: - - package: azure-appconfiguration - version: 1.5.0 - - name: azure-communication - service: communication - common_root: azure/communication - in_batch: ${{ parameters.release_azure_communication }} - checkout: - - package: azure-communication-chat - version: 1.2.0 - - package: azure-communication-email - version: 1.0.0 - - package: azure-communication-identity - version: 1.5.0 - - package: azure-communication-phonenumbers - version: 1.1.0 - - package: azure-communication-sms - version: 1.0.1 - - package: azure-communication-rooms - version: 1.0.0 - - package: azure-communication-jobrouter - version: 1.0.0 - - package: azure-communication-callautomation - version: 1.1.0 - - name: azure-confidentialledger - service: confidentialledger - in_batch: ${{ parameters.release_azure_confidentialledger }} - checkout: - - package: azure-confidentialledger - version: 1.1.1 - - name: azure-containerregistry - service: containerregistry - in_batch: ${{ parameters.release_azure_containerregistry }} - checkout: - - package: azure-containerregistry - version: 1.2.0 - - name: azure-cosmos - service: cosmos - in_batch: ${{ parameters.release_azure_cosmos }} - checkout: - - package: azure-cosmos - version: 4.5.1 - - name: azure-data-tables - service: tables - in_batch: ${{ parameters.release_azure_data_tables }} - checkout: - - package: azure-data-tables - version: 12.5.0 - - name: azure-developer-loadtesting - service: loadtesting - in_batch: ${{ parameters.release_azure_developer_loadtesting }} - checkout: - - package: azure-developer-loadtesting - version: 1.0.0 - - name: azure-digitaltwins-core - service: digitaltwins - in_batch: ${{ parameters.release_azure_digitaltwins_core }} - checkout: - - package: azure-digitaltwins-core - version: 1.2.0 - - name: azure-eventgrid - service: eventgrid - in_batch: ${{ parameters.release_azure_eventgrid }} - checkout: - - package: azure-eventgrid - version: 4.17.0 - - name: azure-eventhub - service: eventhub - common_root: azure/eventhub - in_batch: ${{ parameters.release_azure_eventhub }} - checkout: - - package: azure-eventhub-checkpointstoreblob - version: 1.1.4 - - package: azure-eventhub-checkpointstoreblob-aio - version: 1.1.4 - - package: azure-eventhub - version: 5.11.6 - - name: azure-iot-deviceupdate - service: deviceupdate - in_batch: ${{ parameters.release_azure_iot_deviceupdate }} - checkout: - - package: azure-iot-deviceupdate - version: 1.0.0 - - name: azure-keyvault - service: keyvault - common_root: azure/keyvault - in_batch: ${{ parameters.release_azure_keyvault }} - checkout: - - package: azure-keyvault-administration - version: 4.4.0 - - package: azure-keyvault-certificates - version: 4.8.0 - - package: azure-keyvault-keys - version: 4.9.0 - - package: azure-keyvault-secrets - version: 4.8.0 - - name: azure-messaging-webpubsubservice - service: webpubsub - in_batch: ${{ parameters.release_azure_messaging_webpubsubservice }} - checkout: - - package: azure-messaging-webpubsubservice - version: 1.0.1 - - name : azure-monitor-ingestion - service: monitor - in_batch: ${{ parameters.release_azure_monitor_ingestion }} - checkout: - - package: azure-monitor-ingestion - version: 1.0.3 - - name: azure-monitor-query - service: monitor - in_batch: ${{ parameters.release_azure_monitor_query }} - checkout: - - package: azure-monitor-query - version: 1.2.1 - - name: azure-schemaregistry - service: schemaregistry - common_root: azure/schemaregistry - in_batch: ${{ parameters.release_azure_schemaregistry }} - checkout: - - package: azure-schemaregistry - version: 1.2.0 - - package: azure-schemaregistry-avroencoder - version: 1.0.0 - - name: azure-search-documents - service: search - in_batch: ${{ parameters.release_azure_search_documents }} - checkout: - - package: azure-search-documents - version: 11.4.0 - - name: azure-security-attestation - service: attestation - in_batch: ${{ parameters.release_azure_security_attestation }} - checkout: - - package: azure-security-attestation - version: 1.0.0 - - name: azure-servicebus - service: servicebus - in_batch: ${{ parameters.release_azure_servicebus }} - checkout: - - package: azure-servicebus - version: 7.11.4 - - name: azure-mgmt - service: mgmt - in_batch: ${{ parameters.release_azure_mgmt }} - common_root: azure/mgmt - checkout: - - package: azure-mgmt-advisor - version: 9.0.0 - - package: azure-mgmt-alertsmanagement - version: 1.0.0 - - package: azure-mgmt-apicenter - version: 1.0.0 - - package: azure-mgmt-apimanagement - version: 4.0.0 - - package: azure-mgmt-appconfiguration - version: 3.0.0 - - package: azure-mgmt-appcontainers - version: 3.0.0 - - package: azure-mgmt-applicationinsights - version: 4.0.0 - - package: azure-mgmt-appplatform - version: 9.0.0 - - package: azure-mgmt-attestation - version: 1.0.0 - - package: azure-mgmt-authorization - version: 4.0.0 - - package: azure-mgmt-automanage - version: 1.0.0 - - package: azure-mgmt-automation - version: 1.0.0 - - package: azure-mgmt-avs - version: 8.0.0 - - package: azure-mgmt-azurearcdata - version: 1.0.0 - - package: azure-mgmt-azurestack - version: 1.0.0 - - package: azure-mgmt-azurestackhci - version: 7.0.0 - - package: azure-mgmt-baremetalinfrastructure - version: 1.0.0 - - package: azure-mgmt-batch - version: 17.2.0 - - package: azure-mgmt-billing - version: 6.0.0 - - package: azure-mgmt-botservice - version: 2.0.0 - - package: azure-mgmt-cdn - version: 13.0.0 - - package: azure-mgmt-changeanalysis - version: 1.0.0 - - package: azure-mgmt-chaos - version: 1.0.0 - - package: azure-mgmt-cognitiveservices - version: 13.5.0 - - package: azure-mgmt-commerce - version: 6.0.0 - - package: azure-mgmt-communication - version: 2.0.0 - - package: azure-mgmt-compute - version: 30.5.0 - - package: azure-mgmt-confidentialledger - version: 1.0.0 - - package: azure-mgmt-confluent - version: 2.0.0 - - package: azure-mgmt-connectedvmware - version: 1.0.0 - - package: azure-mgmt-consumption - version: 10.0.0 - - package: azure-mgmt-containerinstance - version: 10.1.0 - - package: azure-mgmt-containerregistry - version: 10.3.0 - - package: azure-mgmt-containerservice - version: 29.1.0 - - package: azure-mgmt-containerservicefleet - version: 1.0.0 - - package: azure-mgmt-cosmosdb - version: 9.4.0 - - package: azure-mgmt-costmanagement - version: 4.0.1 - - package: azure-mgmt-customproviders - version: 1.0.0 - - package: azure-mgmt-dashboard - version: 1.1.0 - - package: azure-mgmt-databox - version: 2.0.0 - - package: azure-mgmt-databoxedge - version: 1.0.0 - - package: azure-mgmt-databricks - version: 2.0.0 - - package: azure-mgmt-datadog - version: 2.1.0 - - package: azure-mgmt-datafactory - version: 5.0.0 - - package: azure-mgmt-datamigration - version: 10.0.0 - - package: azure-mgmt-dataprotection - version: 1.3.0 - - package: azure-mgmt-datashare - version: 1.0.0 - - package: azure-mgmt-deploymentmanager - version: 1.0.0 - - package: azure-mgmt-desktopvirtualization - version: 1.1.0 - - package: azure-mgmt-devcenter - version: 1.0.0 - - package: azure-mgmt-deviceupdate - version: 1.1.0 - - package: azure-mgmt-devtestlabs - version: 9.0.0 - - package: azure-mgmt-digitaltwins - version: 6.4.0 - - package: azure-mgmt-dns - version: 8.1.0 - - package: azure-mgmt-dnsresolver - version: 1.0.0 - - package: azure-mgmt-dynatrace - version: 2.0.0 - - package: azure-mgmt-edgeorder - version: 1.0.0 - - package: azure-mgmt-elastic - version: 1.0.0 - - package: azure-mgmt-elasticsan - version: 1.0.0 - - package: azure-mgmt-eventgrid - version: 10.2.0 - - package: azure-mgmt-eventhub - version: 11.0.0 - - package: azure-mgmt-extendedlocation - version: 1.1.0 - - package: azure-mgmt-fluidrelay - version: 1.0.0 - - package: azure-mgmt-frontdoor - version: 1.1.0 - - package: azure-mgmt-graphservices - version: 1.0.0 - - package: azure-mgmt-hanaonazure - version: 1.0.0 - - package: azure-mgmt-hdinsight - version: 9.0.0 - - package: azure-mgmt-healthcareapis - version: 2.0.0 - - package: azure-mgmt-hybridcompute - version: 8.0.0 - - package: azure-mgmt-hybridconnectivity - version: 1.0.0 - - package: azure-mgmt-hybridcontainerservice - version: 1.0.0 - - package: azure-mgmt-hybridkubernetes - version: 1.1.0 - - package: azure-mgmt-hybridnetwork - version: 2.0.0 - - package: azure-mgmt-imagebuilder - version: 1.3.0 - - package: azure-mgmt-iothub - version: 3.0.0 - - package: azure-mgmt-iothubprovisioningservices - version: 1.1.0 - - package: azure-mgmt-keyvault - version: 10.3.0 - - package: azure-mgmt-kubernetesconfiguration - version: 3.1.0 - - package: azure-mgmt-kusto - version: 3.3.0 - - package: azure-mgmt-labservices - version: 2.0.0 - - package: azure-mgmt-loadtesting - version: 1.0.0 - - package: azure-mgmt-loganalytics - version: 12.0.0 - - package: azure-mgmt-logic - version: 10.0.0 - - package: azure-mgmt-logz - version: 1.0.0 - - package: azure-mgmt-machinelearningservices - version: 1.0.0 - - package: azure-mgmt-maintenance - version: 2.1.0 - - package: azure-mgmt-managednetworkfabric - version: 1.0.0 - - package: azure-mgmt-managedservices - version: 6.0.0 - - package: azure-mgmt-managementgroups - version: 1.0.0 - - package: azure-mgmt-managementpartner - version: 1.0.0 - - package: azure-mgmt-maps - version: 2.1.0 - - package: azure-mgmt-marketplaceordering - version: 1.1.0 - - package: azure-mgmt-media - version: 10.2.0 - - package: azure-mgmt-mixedreality - version: 1.0.0 - - package: azure-mgmt-mobilenetwork - version: 3.1.0 - - package: azure-mgmt-monitor - version: 6.0.2 - - package: azure-mgmt-msi - version: 7.0.0 - - package: azure-mgmt-netapp - version: 11.0.0 - - package: azure-mgmt-network - version: 25.3.0 - - package: azure-mgmt-networkanalytics - version: 1.0.0 - - package: azure-mgmt-networkcloud - version: 1.0.0 - - package: azure-mgmt-newrelicobservability - version: 1.0.0 - - package: azure-mgmt-nginx - version: 3.0.0 - - package: azure-mgmt-notificationhubs - version: 8.0.0 - - package: azure-mgmt-operationsmanagement - version: 1.0.0 - - package: azure-mgmt-orbital - version: 2.0.0 - - package: azure-mgmt-paloaltonetworksngfw - version: 1.0.0 - - package: azure-mgmt-peering - version: 1.0.0 - - package: azure-mgmt-policyinsights - version: 1.0.0 - - package: azure-mgmt-portal - version: 1.0.0 - - package: azure-mgmt-powerbidedicated - version: 1.0.0 - - package: azure-mgmt-privatedns - version: 1.1.0 - - package: azure-mgmt-purview - version: 1.0.0 - - package: azure-mgmt-qumulo - version: 1.0.0 - - package: azure-mgmt-quota - version: 1.1.0 - - package: azure-mgmt-rdbms - version: 10.1.0 - - package: azure-mgmt-recoveryservices - version: 2.5.0 - - package: azure-mgmt-recoveryservicesbackup - version: 9.0.0 - - package: azure-mgmt-recoveryservicessiterecovery - version: 1.2.0 - - package: azure-mgmt-redhatopenshift - version: 1.4.0 - - package: azure-mgmt-redis - version: 14.3.0 - - package: azure-mgmt-redisenterprise - version: 2.0.0 - - package: azure-mgmt-relay - version: 1.1.0 - - package: azure-mgmt-reservations - version: 2.3.0 - - package: azure-mgmt-resource - version: 23.0.1 - - package: azure-mgmt-resourceconnector - version: 1.0.0 - - package: azure-mgmt-resourcemover - version: 1.1.0 - - package: azure-mgmt-search - version: 9.1.0 - - package: azure-mgmt-security - version: 6.0.0 - - package: azure-mgmt-securityinsight - version: 1.0.0 - - package: azure-mgmt-selfhelp - version: 1.0.0 - - package: azure-mgmt-serialconsole - version: 1.0.0 - - package: azure-mgmt-servicebus - version: 8.2.0 - - package: azure-mgmt-servicefabric - version: 2.1.0 - - package: azure-mgmt-servicelinker - version: 1.1.0 - - package: azure-mgmt-servicenetworking - version: 1.0.0 - - package: azure-mgmt-signalr - version: 1.2.0 - - package: azure-mgmt-sql - version: 3.0.1 - - package: azure-mgmt-storage - version: 21.1.0 - - package: azure-mgmt-storagecache - version: 1.5.0 - - package: azure-mgmt-storagemover - version: 2.0.0 - - package: azure-mgmt-storagepool - version: 1.0.0 - - package: azure-mgmt-storagesync - version: 1.0.0 - - package: azure-mgmt-streamanalytics - version: 1.0.0 - - package: azure-mgmt-subscription - version: 3.1.1 - - package: azure-mgmt-support - version: 6.0.0 - - package: azure-mgmt-synapse - version: 2.0.0 - - package: azure-mgmt-timeseriesinsights - version: 1.0.0 - - package: azure-mgmt-trafficmanager - version: 1.1.0 - - package: azure-mgmt-voiceservices - version: 1.0.0 - - package: azure-mgmt-web - version: 7.2.0 - - package: azure-mgmt-webpubsub - version: 1.1.0 - - package: azure-mgmt-workloads - version: 1.0.0 - # # - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}: - # # - template: archetype-conda-release.yml - # # parameters: - # # DependsOn: Build - # # CondaArtifacts: $(CondaArtifacts) \ No newline at end of file diff --git a/eng/pipelines/templates/stages/platform-matrix.json b/eng/pipelines/templates/stages/platform-matrix.json index c7762d733abf..835d6866b2f2 100644 --- a/eng/pipelines/templates/stages/platform-matrix.json +++ b/eng/pipelines/templates/stages/platform-matrix.json @@ -6,9 +6,9 @@ }, "matrix": { "Agent": { - "windows-2022": { "OSVmImage": "MMS2022", "Pool": "azsdk-pool-mms-win-2022-general" }, - "ubuntu-20.04": { "OSVmImage": "MMSUbuntu20.04", "Pool": "azsdk-pool-mms-ubuntu-2004-general" }, - "macos-11": { "OSVmImage": "macos-11", "Pool": "Azure Pipelines" } + "windows-2022": { "OSVmImage": "env:WINDOWSVMIMAGE", "Pool": "env:WINDOWSPOOL" }, + "ubuntu-20.04": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" }, + "macos-11": { "OSVmImage": "env:MACVMIMAGE", "Pool": "env:MACPOOL" } }, "PythonVersion": [ "3.8", "pypy3.9", "3.11", "3.10" ], "CoverageArg": "--disablecov", @@ -18,8 +18,8 @@ { "CoverageConfig": { "ubuntu2004_39_coverage": { - "OSVmImage": "MMSUbuntu20.04", - "Pool": "azsdk-pool-mms-ubuntu-2004-general", + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL", "PythonVersion": "3.9", "CoverageArg": "", "TestSamples": "false" @@ -29,8 +29,8 @@ { "Config": { "Ubuntu2004_312": { - "OSVmImage": "MMSUbuntu20.04", - "Pool": "azsdk-pool-mms-ubuntu-2004-general", + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL", "PythonVersion": "3.12", "CoverageArg": "--disablecov", "TestSamples": "false" diff --git a/eng/pipelines/templates/stages/regression-job-matrix.json b/eng/pipelines/templates/stages/regression-job-matrix.json index 5c8726774623..7fc9760ec54f 100644 --- a/eng/pipelines/templates/stages/regression-job-matrix.json +++ b/eng/pipelines/templates/stages/regression-job-matrix.json @@ -2,8 +2,9 @@ "matrix": { "Agent": { "ubuntu-20.04": { - "OSVmImage": "MMSUbuntu20.04", - "Pool": "azsdk-pool-mms-ubuntu-2004-general" + "OSVmImage": "azsdk-pool-mms-ubuntu-2004-1espt", + "Pool": "env:LINUXPOOL", + "os": "linux" } }, "DependentService": [ diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index 366cc7529933..1df98e02393a 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -153,7 +153,7 @@ steps: - template: /eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactPath: '$(Build.ArtifactStagingDirectory)/reports' ArtifactName: 'reports' diff --git a/eng/pipelines/templates/steps/build-conda-artifacts.yml b/eng/pipelines/templates/steps/build-conda-artifacts.yml index 1355746f2773..3743a76664c5 100644 --- a/eng/pipelines/templates/steps/build-conda-artifacts.yml +++ b/eng/pipelines/templates/steps/build-conda-artifacts.yml @@ -32,7 +32,7 @@ steps: sdk_build_conda -f "$argFile" ${{ parameters.Arguments }} displayName: Assemble Conda Packages - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactPath: '$(Build.SourcesDirectory)/conda/assembled' ArtifactName: '${{ parameters.ArtifactPrefix }}distributions' @@ -44,7 +44,7 @@ steps: inputs: BuildDropPath: '$(Build.SourcesDirectory)/conda/output' - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactPath: '$(Build.SourcesDirectory)/conda/output' ArtifactName: '${{ parameters.ArtifactPrefix }}conda' diff --git a/eng/pipelines/templates/steps/build-extended-artifacts.yml b/eng/pipelines/templates/steps/build-extended-artifacts.yml index 8532e8e41d90..2516def113a0 100644 --- a/eng/pipelines/templates/steps/build-extended-artifacts.yml +++ b/eng/pipelines/templates/steps/build-extended-artifacts.yml @@ -81,7 +81,7 @@ steps: - ${{ parameters.BeforePublishSteps }} - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactPath: '$(Build.ArtifactStagingDirectory)' ArtifactName: 'packages_extended' @@ -91,20 +91,8 @@ steps: Write-Output "##vso[task.setvariable variable=DirectoryExists]$directoryExists" displayName: Check if docs directory exists - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactPath: '$(Build.SourcesDirectory)/_docs' CustomCondition: and(${{ parameters.BuildDocs }}, eq(variables['DirectoryExists'], 'True')) ArtifactName: 'documentation' - - - ${{if eq(variables['System.TeamProject'], 'internal') }}: - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate BOM' - condition: succeededOrFailed() - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml - parameters: - ArtifactPath: '$(Build.ArtifactStagingDirectory)/_manifest' - ArtifactName: 'extended_manifest' \ No newline at end of file diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index a95421ed792d..c5f4298b76cf 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -31,13 +31,13 @@ parameters: # However, please note that this variable will not ALWAYS be set. If using `build-artifacts.yml`, ensure that this variable is set AT LEAST # to "linux", otherwise the primary tasks will not be run as expected. APIStub, Docs, etc are NOT run on machines that aren't linux. steps: - - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml@self parameters: PackageName: "azure-template" ServiceDirectory: "template" TestPipeline: ${{ parameters.TestPipeline }} - - template: /eng/common/pipelines/templates/steps/set-default-branch.yml + - template: /eng/common/pipelines/templates/steps/set-default-branch.yml@self - script: | echo "##vso[build.addbuildtag]Scheduled" @@ -54,7 +54,7 @@ steps: python -m pip install -r eng/ci_tools.txt displayName: 'Prep Environment' - - template: set-dev-build.yml + - template: set-dev-build.yml@self parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} @@ -94,37 +94,13 @@ steps: - ${{ parameters.BeforePublishSteps }} - ${{ if eq(parameters.ArtifactSuffix, '') }}: - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactPath: '$(Build.ArtifactStagingDirectory)' ArtifactName: 'packages' - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate BOM' - condition: succeededOrFailed() - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml - parameters: - ArtifactPath: '$(Build.ArtifactStagingDirectory)/_manifest' - ArtifactName: 'packages_manifest' - - ${{ if ne(parameters.ArtifactSuffix, '') }}: - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: ArtifactPath: '$(Build.ArtifactStagingDirectory)' ArtifactName: 'packages_${{ parameters.ArtifactSuffix }}' - - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate BOM' - condition: succeededOrFailed() - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml - parameters: - ArtifactPath: '$(Build.ArtifactStagingDirectory)/_manifest' - ArtifactName: 'packages_manifest_${{ parameters.ArtifactSuffix }}' \ No newline at end of file diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index cc88faa6ee52..72da1d91c412 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -89,12 +89,6 @@ steps: - ${{ parameters.AfterTestSteps }} - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml - parameters: - ArtifactPath: '$(Build.SourcesDirectory)/_coverage' - ArtifactName: '$(System.StageName)_$(Agent.JobName)_coverage' - CustomCondition: '${{ parameters.RunCoverage }}' - - pwsh: | python -m pip install coverage==7.2.5 python scripts/devops_tasks/create_coverage.py @@ -112,12 +106,6 @@ steps: --toxenv="samples" env: ${{ parameters.EnvVars }} - - template: /eng/common/pipelines/templates/steps/publish-artifact.yml - parameters: - ArtifactPath: '$(Build.SourcesDirectory)/_tox_logs' - ArtifactName: '$(System.StageName)_$(Agent.JobName)_tox_logs' - CustomCondition: failed() - - task: PublishTestResults@2 condition: always() inputs: diff --git a/eng/pipelines/templates/steps/smoke-test-steps.yml b/eng/pipelines/templates/steps/smoke-test-steps.yml new file mode 100644 index 000000000000..81dc3aa27681 --- /dev/null +++ b/eng/pipelines/templates/steps/smoke-test-steps.yml @@ -0,0 +1,115 @@ +parameters: + - name: Daily + default: true + - name: Artifact + type: object + default: {} + - name: ArtifactName + type: string + default: "not-specified" + +steps: + - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml + parameters: + AgentImage: $(OSVmImage) + + - task: UsePythonVersion@0 + displayName: "Use Python $(PythonVersion)" + inputs: + versionSpec: $(PythonVersion) + + - script: | + python -m pip install pip==20.0.2 + pip --version + displayName: pip --version + + - ${{ if eq(parameters.Daily, false) }}: + - download: current + artifact: ${{ parameters.ArtifactName }} + timeoutInMinutes: 5 + + - pwsh: | + $packages = Get-ChildItem "$(Pipeline.Workspace)/${{ parameters.ArtifactName }}/${{ parameters.Artifact.name }}/*.tar.gz" + Write-Host "Artifacts found:" + $artifacts = $packages | ForEach-Object { + if ($_.Name -match "([a-zA-Z\-]+)\-(.*).tar.gz") { + Write-Host "$($matches[1]): $($matches[2])" + return @{ "name" = $matches[1]; "version" = $matches[2] } + } + } + if ($artifacts.name -notcontains "${{parameters.Artifact.name}}") { + Write-Host "Can't find package ${{parameters.Artifact.name}}" + exit 1 + } + $dependencies = Get-Content $(requirements) | ForEach-Object { + $line = $_ + if ($line -match "([a-zA-Z\-]+)(\W+)(.*)") { + $override = ($artifacts | Where-Object { $_.Name -eq $matches[1] }).Version + if ($override) { + $line = $line -replace '([a-zA-Z\-]+)(\W+)(.*)', ('${1}${2}' + $override) + Write-Host "Overriding dependency to: $line" + } + } + return $line + } + + $dependencies | Out-File $(requirements) + + displayName: Override requirements with pipeline build artifact versions + + # Retry for pip install due to delay in package availability after publish + # The package is expected to be available for download/installation within 10 minutes + - pwsh: | + $ErrorActionPreference = "Continue" + while ($retries++ -lt 15) { + Write-Host "python -m pip install -r $(requirements) --no-deps --upgrade --no-cache-dir" + python -m pip install -r "$(requirements)" --no-deps --upgrade --no-cache-dir + if ($LASTEXITCODE) { + if ($retries -ge 15) { + exit $LASTEXITCODE + } + Write-Host "Installation failed, retrying in 1 minute..." + sleep 60 + } else { + break + } + } + displayName: Install requirements without dependencies + + - ${{ if eq(parameters.Daily, true) }}: + - pwsh: | + python -m pip install -r "$(requirements)" --pre --no-deps --upgrade ` + --index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple + + displayName: Install requirements from dev feed without dependencies + + - pwsh: python -m pip install -r $(Build.SourcesDirectory)/common/smoketest/requirements_async.txt + displayName: "Install requirements_async.txt" + condition: and(succeeded(), ne(variables['SkipAsyncInstall'], true)) + + - pwsh: | + python $(Build.SourcesDirectory)/common/smoketest/dependencies.py -r "$(requirements)" ` + | Out-File $(Build.SourcesDirectory)/common/smoketest/requirements_dependencies.txt + displayName: Create dependency list from installed packages + + - script: python -m pip install -r $(Build.SourcesDirectory)/common/smoketest/requirements_dependencies.txt + displayName: Install package dependencies from PyPI + + - script: python -m pip freeze + displayName: Show installed packages (pip freeze) + + - template: /eng/common/TestResources/deploy-test-resources.yml + parameters: + ServiceDirectory: '$(Build.SourcesDirectory)/common/smoketest/' + SubscriptionConfiguration: $(SubscriptionConfiguration) + Location: $(Location) + ArmTemplateParameters: $(ArmTemplateParameters) + + - script: python $(Build.SourcesDirectory)/common/smoketest/program.py + displayName: Run Smoke Test + + - template: /eng/common/TestResources/remove-test-resources.yml + parameters: + ServiceDirectory: '$(Build.SourcesDirectory)/common/smoketest/' + SubscriptionConfiguration: $(SubscriptionConfiguration) + diff --git a/eng/pipelines/templates/steps/targeting-string-resolve.yml b/eng/pipelines/templates/steps/targeting-string-resolve.yml index 10e95eba9350..268d6d92c3e7 100644 --- a/eng/pipelines/templates/steps/targeting-string-resolve.yml +++ b/eng/pipelines/templates/steps/targeting-string-resolve.yml @@ -9,8 +9,12 @@ steps: # if the variable is not set, it'll just come back as the variable name. otherwise it's set. if ('$(BuildTargetingString)' -ne ('$' + '(BuildTargetingString)')) { + Write-Host "The variable named BuildTargetingString is set to $(BuildTargetingString)" $setting = "$(BuildTargetingString)" } + else { + Write-Host "We are falling back to the parameter value ${{ parameters.BuildTargetingString }}" + } Write-Host "Setting TargetingString to $setting" Write-Host "##vso[task.setvariable variable=TargetingString]$setting" diff --git a/eng/pipelines/templates/variables/globals.yml b/eng/pipelines/templates/variables/globals.yml index c2492b875123..065d73d83102 100644 --- a/eng/pipelines/templates/variables/globals.yml +++ b/eng/pipelines/templates/variables/globals.yml @@ -7,3 +7,7 @@ variables: DisableDockerDetector: true # Disable CodeQL injections except for where we specifically enable it Codeql.SkipTaskAutoInjection: true + BuildId: $(Build.BuildId) + REPOROOT: $(Build.SourcesDirectory) + WINDOWS_OUTPUTROOT: $(REPOROOT)\out + WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' \ No newline at end of file diff --git a/eng/pipelines/templates/variables/image.yml b/eng/pipelines/templates/variables/image.yml new file mode 100644 index 000000000000..56cfb9a57438 --- /dev/null +++ b/eng/pipelines/templates/variables/image.yml @@ -0,0 +1,27 @@ +# Default pool image selection. Set as variable so we can override at pipeline level + +variables: + - name: LINUXPOOL + value: azsdk-pool-mms-ubuntu-2004-general + - name: WINDOWSPOOL + value: azsdk-pool-mms-win-2022-general + - name: MACPOOL + value: Azure Pipelines + + - name: LINUXVMIMAGE + value: azsdk-pool-mms-ubuntu-2004-1espt + - name: LINUXNEXTVMIMAGE + value: ubuntu-22.04 + - name: WINDOWSVMIMAGE + value: azsdk-pool-mms-win-2022-1espt + - name: MACVMIMAGE + value: macos-11 + + # Values required for pool.os field in 1es pipeline templates + - name: LINUXOS + value: linux + - name: WINDOWSOS + value: windows + - name: MACOS + value: macOS + diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index a91092a98181..ccd15555fdff 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -35,7 +35,7 @@ extends: template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: core - BuildTargetingString: '*' + BuildTargetingString: "*" ValidateFormatting: true Artifacts: - name: azure-core diff --git a/sdk/template/tests.yml b/sdk/template/tests.yml index 7f7e4fb0989a..aa7c0a0dcb99 100644 --- a/sdk/template/tests.yml +++ b/sdk/template/tests.yml @@ -1,7 +1,7 @@ trigger: none -stages: - - template: /eng/pipelines/templates/stages/archetype-sdk-tests.yml +extends: + template: /eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: ServiceDirectory: template MatrixReplace: diff --git a/tools/azure-sdk-tools/ci_tools/build.py b/tools/azure-sdk-tools/ci_tools/build.py index 27bdfe734cbb..dabdb11cb021 100644 --- a/tools/azure-sdk-tools/ci_tools/build.py +++ b/tools/azure-sdk-tools/ci_tools/build.py @@ -11,6 +11,7 @@ from ci_tools.versioning.version_set_dev import get_dev_version, format_build_id from ci_tools.logging import initialize_logger, run_logged +logger = initialize_logger("build.py") def build() -> None: parser = argparse.ArgumentParser( @@ -105,6 +106,8 @@ def build() -> None: else: target_dir = repo_root + logger.debug(f"Searching for packages starting from {target_dir} with glob string {args.glob_string} and package filter {args.package_filter_string}") + targeted_packages = discover_targeted_packages( args.glob_string, target_dir, @@ -113,6 +116,7 @@ def build() -> None: compatibility_filter=True, include_inactive=args.inactive, ) + artifact_directory = get_artifact_directory(args.distribution_directory) build_id = format_build_id(args.build_id or DEFAULT_BUILD_ID) @@ -146,12 +150,10 @@ def build_packages( build_apiview_artifact: bool = False, build_id: str = "", ): - logger = initialize_logger("build.py") - logger.log(level=logging.INFO, msg=f"Generating Package Using Python {sys.version}") + logger.log(level=logging.INFO, msg=f"Generating {targeted_packages} using python{sys.version}") for package_root in targeted_packages: setup_parsed = ParsedSetup.from_path(package_root) - package_name_in_artifacts = os.path.join(os.path.basename(package_root)) dist_dir = os.path.join(distribution_directory, package_name_in_artifacts) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index da48cdd16131..400a06663312 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -154,7 +154,7 @@ def generate_difference(original_packages: List[str], filtered_packages: List[st def glob_packages(glob_string: str, target_root_dir: str) -> List[str]: if glob_string: - individual_globs = glob_string.split(",") + individual_globs = [glob_string.strip() for glob_string in glob_string.split(",")] else: individual_globs = "azure-*" collected_top_level_directories = []