Merge pull request #4169 from pypa/feature/vendor-update

Update vendored dependencies, relevant API usage, and CI config

Author: techalchemy

.azure-pipelines/scripts/New-Ramdisk.ps1

@@ -0,0 +1,75 @@
+# Taken from https://github.com/pypa/pip/blob/ceaf75b9ede9a9c25bcee84fe512fa6774889685/.azure-pipelines/scripts/New-RAMDisk.ps1
+[CmdletBinding()]
+param(
+    [Parameter(Mandatory=$true,
+    HelpMessage="Drive letter to use for the RAMDisk")]
+    [String]$drive,
+    [Parameter(HelpMessage="Size to allocate to the RAMDisk")]
+    [UInt64]$size=1GB
+)
+
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version Latest
+
+Write-Output "Installing FS-iSCSITarget-Server"
+Install-WindowsFeature -Name FS-iSCSITarget-Server
+
+Write-Output "Starting MSiSCSI"
+Start-Service MSiSCSI
+$retry = 10
+do {
+    $service = Get-Service MSiSCSI
+    if ($service.Status -eq "Running") {
+        break;
+    }
+    $retry--
+    Start-Sleep -Milliseconds 500
+} until ($retry -eq 0)
+
+$service = Get-Service MSiSCSI
+if ($service.Status -ne "Running") {
+    throw "MSiSCSI is not running"
+}
+
+Write-Output "Configuring Firewall"
+Get-NetFirewallServiceFilter -Service MSiSCSI | Enable-NetFirewallRule
+
+Write-Output "Configuring RAMDisk"
+# Must use external-facing IP address, otherwise New-IscsiTargetPortal is
+# unable to connect.
+$ip = (
+    Get-NetIPAddress -AddressFamily IPv4 |
+    Where-Object {$_.IPAddress -ne "127.0.0.1"}
+)[0].IPAddress
+if (
+    -not (Get-IscsiServerTarget -ComputerName localhost | Where-Object {$_.TargetName -eq "ramdisks"})
+) {
+    New-IscsiServerTarget `
+        -ComputerName localhost `
+        -TargetName ramdisks `
+        -InitiatorId IPAddress:$ip
+}
+
+$newVirtualDisk = New-IscsiVirtualDisk `
+    -ComputerName localhost `
+    -Path ramdisk:local$drive.vhdx `
+    -Size $size
+Add-IscsiVirtualDiskTargetMapping `
+    -ComputerName localhost `
+    -TargetName ramdisks `
+    -Path ramdisk:local$drive.vhdx
+
+Write-Output "Connecting to iSCSI"
+New-IscsiTargetPortal -TargetPortalAddress $ip
+Get-IscsiTarget | Where-Object {!$_.IsConnected} | Connect-IscsiTarget
+
+Write-Output "Configuring disk"
+$newDisk = Get-IscsiConnection |
+    Get-Disk |
+    Where-Object {$_.SerialNumber -eq $newVirtualDisk.SerialNumber}
+
+Set-Disk -InputObject $newDisk -IsOffline $false
+Initialize-Disk -InputObject $newDisk -PartitionStyle MBR
+New-Partition -InputObject $newDisk -UseMaximumSize -DriveLetter $drive
+
+Format-Volume -DriveLetter $drive -NewFileSystemLabel Temp -FileSystem NTFS

.azure-pipelines/steps/build-package.yml

@@ -6,16 +6,17 @@ steps:
     addToPath: true
   displayName: Use Python $(python.version)
 
-- template: install-dependencies.yml
+- bash: |
+    PYTHON_PATH=$(python -c 'import sys; print(sys.executable)')
+    echo "##vso[task.setvariable variable=PY_EXE]$PYTHON_PATH"
+  displayName: Set Python Path
 
 - script: |
     echo '##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]'$(python.version)
   env:
     PYTHON_VERSION: $(python.version)
 
-- template: create-virtualenv.yml
-  parameters:
-    python_version: $(python.version)
+- template: install-dependencies.yml
 
 - script: |
     python -m pip install --upgrade wheel pip setuptools twine readme_renderer[md]

.azure-pipelines/steps/create-virtualenv.yml

@@ -1,5 +1,31 @@
+parameters:
+  python_version: ''
+
 steps:
-- script: 'python -m pip install --upgrade pip setuptools wheel -e .[dev,tests] --upgrade'
+
+- script: |
+    echo "##vso[task.setvariable variable=LANG]C.UTF-8"
+    echo "##vso[task.setvariable variable=PIP_PROCESS_DEPENDENCY_LINKS]1"
+  displayName: Set Environment Variables
+
+- script: |
+    echo "Python path: $(PY_EXE)"
+    echo "GIT_SSL_CAINFO: $(GIT_SSL_CAINFO)"
+    echo "PIPENV PYTHON VERSION: $(python.version)"
+    echo "python_version: ${{ parameters.python_version }}"
+    git submodule sync
+    git submodule update --init --recursive
+    $(PY_EXE) -m pip install --upgrade --upgrade-strategy=eager pip setuptools wheel
+    $(PY_EXE) -m pip install "virtualenv<20"
+  env:
+    PIPENV_DEFAULT_PYTHON_VERSION: ${{ parameters.python_version }}
+    PYTHONWARNINGS: 'ignore:DEPRECATION'
+    PIPENV_NOSPIN: '1'
+  displayName: Make Virtualenv
+
+- script: |
+    $(PY_EXE) -m pip install -e . --upgrade
+    $(PY_EXE) -m pipenv install --deploy --dev --python="$(PY_EXE)"
   displayName: Upgrade Pip & Install Pipenv
   env:
     PYTHONWARNINGS: 'ignore:DEPRECATION'

.azure-pipelines/steps/install-dependencies.yml

@@ -2,13 +2,29 @@ parameters:
   python_version: ''
 
 steps:
+
+- bash: |
+    pip install certifi
+    PYTHON_PATH=$(python -c 'import sys; print(sys.executable)')
+    CERTIFI_CONTENT=$(python -m certifi)
+    echo "##vso[task.setvariable variable=GIT_SSL_CAINFO]$CERTIFI_CONTENT"
+    echo "##vso[task.setvariable variable=PY_EXE]$PYTHON_PATH"
+  displayName: Set Python Path
+  env:
+    PYTHONWARNINGS: 'ignore:DEPRECATION'
+
+- template: install-dependencies.yml
+  parameters:
+    python_version: ${{ parameters.python_version }}
+
 - script: |
     # Fix Git SSL errors
     echo "Using pipenv python version: $(PIPENV_DEFAULT_PYTHON_VERSION)"
     git submodule sync && git submodule update --init --recursive
-    pipenv run pytest --junitxml=test-results.xml
+    pipenv run pytest -n 4 --junitxml=junit/test-results.xml
   displayName: Run integration tests
   env:
     PYTHONWARNINGS: ignore:DEPRECATION
     PIPENV_NOSPIN: '1'
     PIPENV_DEFAULT_PYTHON_VERSION: ${{ parameters.python_version }}
+    GIT_SSH_COMMAND: ssh -o StrictHostKeyChecking=accept-new -o CheckHostIP=no

.azure-pipelines/steps/reinstall-pythons.yml

@@ -1,21 +1,56 @@
 parameters:
   python_version: ''
+  python_architecture: ''
 
 steps:
-- powershell: |
-    subst T: "$env:TEMP"
-    Write-Host "##vso[task.setvariable variable=TEMP]T:\"
-    Write-Host "##vso[task.setvariable variable=TMP]T:\"
-    Write-Host "##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]$env:PYTHON_VERSION"
-    Write-Host "##vso[task.setvariable variable=PIPENV_NOSPIN]1"
-  displayName: Fix Temp Variable
-  env:
-    PYTHON_VERSION: ${{ parameters.python_version }}
+  - task: PowerShell@2
+    inputs:
+      filePath: .azure-pipelines/scripts/New-RAMDisk.ps1
+      arguments: "-Drive R -Size 2GB"
+    displayName: Setup RAMDisk
 
-- script: |
-    git submodule sync && git submodule update --init --recursive
-    pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests
-  displayName: Run integration tests
-  env:
-    PYTHONWARNINGS: 'ignore:DEPRECATION'
-    PIPENV_NOSPIN: '1'
+  - powershell: |
+      mkdir R:\virtualenvs
+      $acl = Get-Acl "R:\"
+      $rule = New-Object System.Security.AccessControl.FileSystemAccessRule(
+          "Everyone", "FullControl", "ContainerInherit,ObjectInherit", "None", "Allow"
+      )
+      $acl.AddAccessRule($rule)
+      Set-Acl "R:\" $acl
+    displayName: Set RAMDisk Permissions
+
+  - powershell: |
+      Write-Host "##vso[task.setvariable variable=TEMP]R:\"
+      Write-Host "##vso[task.setvariable variable=TMP]R:\"
+      Write-Host "##vso[task.setvariable variable=WORKON_HOME]R:\virtualenvs"
+      Write-Host "##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]$env:PYTHON_VERSION"
+      Write-Host "##vso[task.setvariable variable=PIPENV_NOSPIN]1"
+    displayName: Fix Temp Variable
+    env:
+      PYTHON_VERSION: ${{ parameters.python_version }}
+
+  - powershell: |
+      pip install certifi
+      $env:PYTHON_PATH=$(python -c "import sys; print(sys.executable)")
+      $env:CERTIFI_CONTENT=$(python -m certifi)
+      echo "##vso[task.setvariable variable=GIT_SSL_CAINFO]$env:CERTIFI_CONTENT"
+      echo "##vso[task.setvariable variable=PY_EXE]$env:PYTHON_PATH"
+    displayName: Set Python Path
+    env:
+      PYTHONWARNINGS: 'ignore:DEPRECATION'
+
+  - template: install-dependencies.yml
+    parameters:
+      python_version: ${{ parameters.python_version }}
+
+  - script: |
+      git submodule sync
+      git submodule update --init --recursive
+      pipenv run pytest -ra -n 4 --junit-xml=junit/test-results.xml tests/
+    failOnStderr: false
+    displayName: Run integration tests
+    env:
+      TEMP: 'R:\'
+      PYTHONWARNINGS: 'ignore:DEPRECATION'
+      PIPENV_NOSPIN: 1
+      GIT_SSH_COMMAND: ssh -o StrictHostKeyChecking=accept-new -o CheckHostIP=no

.azure-pipelines/steps/run-tests-linux.yml

@@ -6,29 +6,23 @@ steps:
     addToPath: true
   displayName: Use Python $(python.version)
 
-- template: install-dependencies.yml
-
 - script: |
     echo '##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]'$(python.version)
   env:
     PYTHON_VERSION: $(python.version)
 
-- template: create-virtualenv.yml
-  parameters:
-    python_version: $(python.version)
-
-- ${{ if eq(parameters.vmImage, 'windows-2019') }}:
+- ${{ if eq(parameters.vmImage, 'windows-latest') }}:
   - template: run-tests-windows.yml
     parameters:
       python_version: $(python.version)
-- ${{ if ne(parameters.vmImage, 'windows-2019') }}:
+- ${{ if ne(parameters.vmImage, 'windows-latest') }}:
   - template: run-tests-linux.yml
     parameters:
       python_version: $(python.version)
 
 - task: PublishTestResults@2
   displayName: Publish Test Results
   inputs:
-    testResultsFiles: '**/test-results.xml'
+    testResultsFiles: '**/junit/*.xml'
     testRunTitle: 'Python $(python.version)'
   condition: succeededOrFailed()

.azure-pipelines/steps/run-tests-windows.yml

@@ -9,25 +9,24 @@ steps:
     addToPath: true
   displayName: Use Python $(python.version)
 
-- template: install-dependencies.yml
+- bash: |
+    python -m pip install --upgrade --upgrade-strategy=eager pip requests certifi wheel setuptools
+    PYTHON_PATH=$(python -c 'import sys; print(sys.executable)')
+    CERTIFI_CONTENT=$(python -m certifi)
+    echo "##vso[task.setvariable variable=PY_EXE]$PYTHON_PATH"
+    echo "##vso[task.setvariable variable=GIT_SSL_CAINFO]$CERTIFI_CONTENT"
+  displayName: Set Python Path
 
-- script: |
-    echo '##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]'$(python.version)
-  env:
-    PYTHON_VERSION: $(python.version)
-
-- template: create-virtualenv.yml
-  parameters:
-    python_version: $(python.version)
+- template: install-dependencies.yml
 
 - script: |
-    python -m pip install --upgrade invoke requests parver bs4 vistir towncrier pip setuptools wheel --upgrade-strategy=eager
+    python -m pip install --upgrade invoke parver bs4 vistir towncrier --upgrade-strategy=eager
     python -m invoke vendoring.update
   displayName: Run Vendor Scripts
   env:
     PY_EXE: $(PY_EXE)
     GIT_SSL_CAINFO: $(GIT_SSL_CAINFO)
     LANG: $(LANG)
-    PIPENV_DEFAULT_PYTHON_VERSION: '${{ parameters.python_version }}'
+    PIPENV_DEFAULT_PYTHON_VERSION: $(python.version)
     PYTHONWARNINGS: ignore:DEPRECATION
     PIPENV_NOSPIN: '1'