diff --git a/doc/dev/conda-builds.md b/doc/dev/conda-builds.md index 152156bf67ad..6034a02f66f1 100644 --- a/doc/dev/conda-builds.md +++ b/doc/dev/conda-builds.md @@ -4,6 +4,8 @@ Follow the instructions [here](https://docs.conda.io/projects/conda-build/en/latest/install-conda-build.html) to install `conda` and `conda-build`. +**The Azure SDK Conda artifacts support `python3.8` and `python3.9` only.** + ## CI Build Process There will be a `CondaArtifact` defined in the `ci.yml` of each service directory. (`sdk/`) @@ -15,9 +17,18 @@ A Conda Artifact defines: - Any other necessary details. ## How to Build an Azure SDK Conda Package Locally +#### If using powershell, you will need to prep your environment before proceeding to the next step + +``` +powershell -ExecutionPolicy ByPass -NoExit -Command "& '\shell\condabin\conda-hook.ps1' ; conda activate '' " +``` + +Afterwards, invoke `conda init powershell` and re-create the pshell session. +By default, your powershell environment will now load `conda`. If you want pure pip, you will need to use explicit invocations of your `python` locations to create virtual envs. ### Set up your conda environment + You will notice that all the azure-sdk conda distributions have the **same** version number and requirement set. This is due to the fact that the azure-sdk team pushes our conda packages out in waves. To support this, all versions are set via a common environment variable `AZURESDK_CONDA_VERSION`. We keep this environment variable set properly across all our builds by using a common `conda_env.yml` when creating our build environment. This environment definition ensures that: diff --git a/eng/conda_env.yml b/eng/conda_env.yml index 960c090a9339..4c674de246a3 100644 --- a/eng/conda_env.yml +++ b/eng/conda_env.yml @@ -1,2 +1,2 @@ variables: - AZURESDK_CONDA_VERSION: '2021.05.01' + AZURESDK_CONDA_VERSION: '2021.05.01b1' diff --git a/eng/conda_test_requirements.txt b/eng/conda_test_requirements.txt new file mode 100644 index 000000000000..b359bbbf5b7c --- /dev/null +++ b/eng/conda_test_requirements.txt @@ -0,0 +1,12 @@ +# install from root of repo +aiohttp>=3.0; python_version >= '3.5' +tools/azure-devtools +tools/azure-sdk-tools +mock; +aiodns>=2.0; python_version >= '3.5' +parameterized>=0.7.3; python_version >= '3.0' +trio; python_version >= '3.5' +typing_extensions>=3.7.2 +futures==3.3.0; python_version <= '2.7' +cryptography +adal \ No newline at end of file diff --git a/eng/pipelines/templates/jobs/ci.conda.tests.yml b/eng/pipelines/templates/jobs/ci.conda.tests.yml new file mode 100644 index 000000000000..dafeada3932a --- /dev/null +++ b/eng/pipelines/templates/jobs/ci.conda.tests.yml @@ -0,0 +1,194 @@ +parameters: + - name: TestPipeline + type: boolean + default: false + - name: ServiceDirectory + type: string + default: '' + - name: CondaArtifacts + type: object + default: [] + - name: TestMarkArgument + type: string + default: '' + - name: PythonVersion + type: string + default: '' + - name: OSVmImage + type: string + default: '' + - name: Matrix + type: string + - name: DependsOn + type: string + default: '' + - name: UsePlatformContainer + type: boolean + default: false + - name: TestTimeoutInMinutes + type: number + default: 0 + - name: CloudConfig + type: object + default: {} + +jobs: + - job: + displayName: 'Test Conda' + condition: | + and( + succeededOrFailed(), + ne(variables['Skip.TestConda'], 'true') + ) + timeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} + + dependsOn: + - ${{ parameters.DependsOn }} + + strategy: + matrix: $[ ${{ parameters.Matrix }} ] + + pool: + name: $(Pool) + vmImage: $(OSVmImage) + + ${{ if eq(parameters.UsePlatformContainer, 'true') }}: + # Add a default so the job doesn't fail when the matrix is empty + container: $[ variables['Container'] ] + + variables: + - template: ../variables/globals.yml + + steps: + - task: DownloadPipelineArtifact@2 + inputs: + artifactName: 'conda' + targetPath: $(Build.ArtifactStagingDirectory) + + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + parameters: + PackageName: "azure-template" + ServiceDirectory: "template" + TestPipeline: ${{ parameters.TestPipeline }} + + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: $(PythonVersion) + + - pwsh: | + # due to faulty deployed scripts/how the path gets manipulated by conda actions on + # ubuntu and mac, we can't rely on bin/scripts being referenced correctly. see + # https://github.com/MicrosoftDocs/azure-devops-docs/issues/3812 + $activateMethod = "source $($env:CONDA)/bin/activate" + + # pypy3 is not a true python executable. in conda-land, we need to call it using pypy3, NOT python + + # on windows, we need to add "--user" as otherwise pip won't successfully install/uninstall due to + # how windows holds reservation on pip.exe. this is unnecessary on ubuntu/mac. + $requirementSuffix = "" + + # we always want to prepend the path with conda bin + Write-Host "##vso[task.prependpath]]$($env:CONDA)/bin" + + if ($IsWindows) { + # powershell does not have an equivalent of call/source, which is necessary when + # using conda in azure devops. Note that we use `activate` natively here, as + # a later path prepend of the /scripts directory actually works. + $activateMethod = "call activate" + $requirementSuffix = " --user" + + # on windows only, need to prepend with the scripts directory as well + Write-Host "##vso[task.prependpath]$($env:CONDA)/Scripts" + } + + if("$(PythonVersion)" -eq "pypy3"){ + Write-Host "##vso[task.setvariable variable=PyVersion]-c conda-forge pypy3.7 pip" + } + else { + Write-Host "##vso[task.setvariable variable=PyVersion]python=$(PythonVersion)" + } + + # we will use these variables extensively later + Write-Host "##vso[task.setvariable variable=activate.method]$activateMethod" + Write-Host "##vso[task.setvariable variable=requirement.suffix]$requirementSuffix" + displayName: 'Evaluate OS Specific PATH and Parameters' + + - ${{ each artifact in parameters.CondaArtifacts }}: + # due to the fact that `pypy3` and `conda-build` conda packages are INCOMPATIBLE, we have to create + # a separate env to install `conda-build` and use that to `conda index` the local file channel + - script: | + echo "conda create --name ${{ artifact.name }} $(PyVersion) --yes" + conda create --name ${{ artifact.name }} $(PyVersion) --yes + + echo "conda create --name index-env --yes" + conda create --name index-env --yes + + echo "conda install --name index-env --yes --quiet conda-build" + conda install --name index-env --yes --quiet conda-build + + echo "$(activate.method) index-env" + $(activate.method) index-env + + echo "conda index $(Build.ArtifactStagingDirectory)/${{ artifact.name }}" + conda index $(Build.ArtifactStagingDirectory)/${{ artifact.name }} + displayName: 'Prepare Conda Environment for Testing ${{ artifact.name }}, Index the Target Local Artifact' + + - script: | + echo "$(activate.method) ${{ artifact.name }}" + $(activate.method) ${{ artifact.name }} + + echo "python -m pip install -r eng/ci_tools.txt $(requirement.suffix)" + python -m pip install -r eng/ci_tools.txt $(requirement.suffix) + displayName: 'Activate Conda Environment and Install General Dependencies ${{ artifact.name }}' + + - pwsh: | + mkdir $(Agent.BuildDirectory)/conda/ + Write-Host "##vso[task.setvariable variable=conda.build]$(Agent.BuildDirectory)/conda_checkout" + displayName: 'Create Conda Working Directory for Testing' + + - script: | + echo "$(activate.method) ${{ artifact.name }}" + $(activate.method) ${{ artifact.name }} + + echo "python -m pip install -r $(Build.SourcesDirectory)/eng/conda_test_requirements.txt" + python -m pip install -r $(Build.SourcesDirectory)/eng/conda_test_requirements.txt + + python -m pip uninstall azure-core -y + displayName: 'Prep Conda Environment w/ Dependencies' + + - script: | + echo "conda install --name ${{ artifact.name }} ${{ artifact.name }} -c $(Build.ArtifactStagingDirectory)/${{ artifact.name }} --yes -c $(AzureSDKCondaChannel)" + conda install --name ${{ artifact.name }} ${{ artifact.name }} -c $(Build.ArtifactStagingDirectory)/${{ artifact.name }} --yes -c $(AzureSDKCondaChannel) + + echo "conda install --name ${{ artifact.name }} azure-identity -c $(Build.ArtifactStagingDirectory)/${{ artifact.name }} -c $(AzureSDKCondaChannel) --yes" + conda install --name ${{ artifact.name }} azure-identity -c $(Build.ArtifactStagingDirectory)/${{ artifact.name }} -c $(AzureSDKCondaChannel) --yes + + echo "$(activate.method) ${{ artifact.name }}" + $(activate.method) ${{ artifact.name }} + python -m pip freeze + displayName: 'Install ${{ artifact.name }} Conda Package' + + - ${{ each checkout in artifact.checkout }}: + - pwsh: + Write-Host "Clean up Conda Build Directory $(conda.build)" + Remove-Item $(conda.build)/* -Recurse -Force + displayName: 'Clean Up Before Testing ${{ artifact.name }}' + + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - "${{ checkout.checkout_path }}" + - "sdk/conftest.py" + - "tools/" + Repositories: + - Name: "Azure/azure-sdk-for-python" + Commitish: "${{ checkout.Package }}_${{ checkout.Version }}" + WorkingDirectory: "$(conda.build)" + SkipDefaultCheckout: true + + - script: | + echo "$(activate.method) ${{ artifact.name }}" + $(activate.method) ${{ artifact.name }} + python -m pytest $(conda.build)/${{ checkout.checkout_path }}/${{ checkout.package }} + displayName: 'Run Tests for ${{ checkout.package }}' diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 04042ad43e4c..71d5b0fcb93f 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -126,6 +126,29 @@ jobs: ToxEnvParallel: ${{ parameters.ToxEnvParallel }} InjectedPackages: ${{ parameters.InjectedPackages }} + - ${{ if gt(length(parameters.CondaArtifacts), 0) }}: + - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml + parameters: + JobTemplatePath: /eng/pipelines/templates/jobs/ci.conda.tests.yml + GenerateJobName: generate_conda_matrix + DependsOn: + - 'Build' + MatrixConfigs: + - Name: Python_ci_conda_envs + Path: eng/pipelines/templates/stages/platform-matrix-conda-support.json + Selection: sparse + GenerateVMJobs: true + MatrixFilters: ${{ parameters.MatrixFilters }} + MatrixReplace: ${{ parameters.MatrixReplace }} + CloudConfig: + Cloud: Public + AdditionalParameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + TestPipeline: ${{ parameters.TestPipeline }} + TestMarkArgument: ${{ parameters.TestMarkArgument }} + TestTimeoutInMinutes: ${{ parameters.TestTimeoutInMinutes }} + CondaArtifacts: ${{ parameters.CondaArtifacts}} + - job: 'RunRegression' condition: and(succeededOrFailed(), or(eq(variables['Run.Regression'], 'true'), and(eq(variables['Build.Reason'], 'Schedule'), eq(variables['System.TeamProject'],'internal')))) displayName: 'Run Regression' diff --git a/eng/pipelines/templates/stages/archetype-conda-release.yml b/eng/pipelines/templates/stages/archetype-conda-release.yml index ff9da7fda525..350beefd5b1c 100644 --- a/eng/pipelines/templates/stages/archetype-conda-release.yml +++ b/eng/pipelines/templates/stages/archetype-conda-release.yml @@ -9,7 +9,7 @@ parameters: stages: - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: - ${{ each artifact in parameters.CondaArtifacts }}: - - stage: Release_${{ replace(artifact.name, '-', '_') }} + - stage: Release_${{ replace(artifact.name, '-', '_') }}_To_Blob displayName: 'Conda Release: ${{artifact.name}}' dependsOn: ${{parameters.DependsOn}} condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr')) diff --git a/eng/pipelines/templates/stages/platform-matrix-conda-support.json b/eng/pipelines/templates/stages/platform-matrix-conda-support.json new file mode 100644 index 000000000000..301bffddee63 --- /dev/null +++ b/eng/pipelines/templates/stages/platform-matrix-conda-support.json @@ -0,0 +1,10 @@ +{ + "matrix": { + "Agent": { + "ubuntu-18.04": { "OSVmImage": "MMSUbuntu18.04", "Pool": "azsdk-pool-mms-ubuntu-1804-general" }, + "windows-2019": { "OSVmImage": "MMS2019", "Pool": "azsdk-pool-mms-win-2019-general" }, + "macOS-10.15": { "OSVmImage": "macOS-10.15", "Pool": "Azure Pipelines" } + }, + "PythonVersion": [ "3.6", "3.8", "3.9" ] + } +} diff --git a/eng/pipelines/templates/steps/build-conda-artifacts.yml b/eng/pipelines/templates/steps/build-conda-artifacts.yml index bcaec17a22e3..3881062f5063 100644 --- a/eng/pipelines/templates/steps/build-conda-artifacts.yml +++ b/eng/pipelines/templates/steps/build-conda-artifacts.yml @@ -74,9 +74,9 @@ steps: - bash: | source activate ${{ artifact.name }} - conda-build . --output-folder "$(Agent.BuildDirectory)/conda/output" -c $(AzureSDKCondaChannel) + conda-build . --output-folder "$(Agent.BuildDirectory)/conda/output/${{ artifact.name }}" -c $(AzureSDKCondaChannel) displayName: 'Activate Conda Environment and Build ${{ artifact.name }}' - workingDirectory: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }} + workingDirectory: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}/conda-recipe - template: /eng/common/pipelines/templates/steps/publish-artifact.yml parameters: diff --git a/scripts/devops_tasks/build_conda_artifacts.py b/scripts/devops_tasks/build_conda_artifacts.py index 090932f56192..6f47c9e86087 100644 --- a/scripts/devops_tasks/build_conda_artifacts.py +++ b/scripts/devops_tasks/build_conda_artifacts.py @@ -33,7 +33,7 @@ VERSION_REGEX = re.compile(r"\s*AZURESDK_CONDA_VERSION\s*:\s*[\'](.*)[\']\s*") -SUMMARY_TEMPLATE = "- Generated from {}." +SUMMARY_TEMPLATE = " - Generated from {}." NAMESPACE_EXTENSION_TEMPLATE = """__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: str """ diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index d3800432f4ba..c569bf00fc25 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -46,7 +46,7 @@ extends: safeName: azurecommon CondaArtifacts: - name: azure-core - meta_source: meta.yaml + meta_source: conda-recipe/meta.yaml common_root: azure checkout: - package: azure-core diff --git a/sdk/core/meta.yaml b/sdk/core/conda-recipe/meta.yaml similarity index 100% rename from sdk/core/meta.yaml rename to sdk/core/conda-recipe/meta.yaml diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index cb7389b84345..57d6db8ab90d 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -58,7 +58,7 @@ extends: safeName: azuremgmtstorageimportexport CondaArtifacts: - name: azure-storage - meta_source: meta.yaml + meta_source: conda-recipe/meta.yaml common_root: azure/storage checkout: - package: azure-storage-blob diff --git a/sdk/storage/meta.yaml b/sdk/storage/conda-recipe/meta.yaml similarity index 100% rename from sdk/storage/meta.yaml rename to sdk/storage/conda-recipe/meta.yaml