From 5af9e12f1caeca07c295e8c37decde053561b18b Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 16 Apr 2021 17:48:08 -0700 Subject: [PATCH 01/20] Update to CUDA 11.3 on Windows. --- scripts/azure-pipelines/windows/provision-image.txt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/azure-pipelines/windows/provision-image.txt b/scripts/azure-pipelines/windows/provision-image.txt index e3e04be28e0df6..384142f7fc75e3 100644 --- a/scripts/azure-pipelines/windows/provision-image.txt +++ b/scripts/azure-pipelines/windows/provision-image.txt @@ -128,12 +128,13 @@ $WindowsWDKUrl = 'https://go.microsoft.com/fwlink/?linkid=2128854' $MpiUrl = 'https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisetup.exe' -$CudaUrl = 'https://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_426.00_win10.exe' -$CudaFeatures = 'nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 gpu_library_advisor_10.1 memcheck_10.1 ' + ` - 'nvdisasm_10.1 nvprof_10.1 visual_profiler_10.1 visual_studio_integration_10.1 cublas_10.1 cublas_dev_10.1 ' + ` - 'cudart_10.1 cufft_10.1 cufft_dev_10.1 curand_10.1 curand_dev_10.1 cusolver_10.1 cusolver_dev_10.1 cusparse_10.1 ' + ` - 'cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1 ' + ` - 'occupancy_calculator_10.1 fortran_examples_10.1' +$CudaUrl = 'https://developer.download.nvidia.com/compute/cuda/11.3.0/network_installers/cuda_11.3.0_win10_network.exe' + +$CudaFeatures = 'nvcc_11.3 cuobjdump_11.3 nvprune_11.3 cupti_11.3 memcheck_11.3 nvdisasm_11.3 nvprof_11.3 ' + ` + 'visual_studio_integration_11.3 visual_profiler_11.3 visual_profiler_11.3 cublas_11.3 cublas_dev_11.3 ' + ` + 'cudart_11.3 cufft_11.3 cufft_dev_11.3 curand_11.3 curand_dev_11.3 cusolver_11.3 cusolver_dev_11.3 ' + ` + 'cusparse_11.3 cusparse_dev_11.3 npp_11.3 npp_dev_11.3 nvrtc_11.3 nvrtc_dev_11.3 nvml_dev_11.3 ' + ` + 'occupancy_calculator_11.3 ' $PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.1.0/PowerShell-7.1.0-win-x64.msi' From 329e6276053d145207a264607eef2190c2f454e6 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 16 Apr 2021 19:23:06 -0700 Subject: [PATCH 02/20] Update PowerShell to 7.1.3. --- scripts/azure-pipelines/windows/provision-image.txt | 2 +- scripts/vcpkgTools.xml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/azure-pipelines/windows/provision-image.txt b/scripts/azure-pipelines/windows/provision-image.txt index 384142f7fc75e3..7e5533289b3d49 100644 --- a/scripts/azure-pipelines/windows/provision-image.txt +++ b/scripts/azure-pipelines/windows/provision-image.txt @@ -136,7 +136,7 @@ $CudaFeatures = 'nvcc_11.3 cuobjdump_11.3 nvprune_11.3 cupti_11.3 memcheck_11.3 'cusparse_11.3 cusparse_dev_11.3 npp_11.3 npp_dev_11.3 nvrtc_11.3 nvrtc_dev_11.3 nvml_dev_11.3 ' + ` 'occupancy_calculator_11.3 ' -$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.1.0/PowerShell-7.1.0-win-x64.msi' +$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.1.3/PowerShell-7.1.3-win-x64.msi' $ErrorActionPreference = 'Stop' $ProgressPreference = 'SilentlyContinue' diff --git a/scripts/vcpkgTools.xml b/scripts/vcpkgTools.xml index 50e0476fec7515..c4c4b14b123de5 100644 --- a/scripts/vcpkgTools.xml +++ b/scripts/vcpkgTools.xml @@ -134,10 +134,10 @@ ninja-freebsd-1.8.2.zip - 7.1.0 + 7.1.3 pwsh.exe - https://github.com/PowerShell/PowerShell/releases/download/v7.1.0/PowerShell-7.1.0-win-x86.zip - ea6ed619c784af65b09a5e98cdfc0d41333ba7373b7409e488ec70aaf1e0c4e5568e846d2b1748d72c3e3a9c1a4827f0ba29e9f6e93e79b4f49312ca40af6d2a - PowerShell-7.1.0-win-x86.zip + https://github.com/PowerShell/PowerShell/releases/download/v7.1.3/PowerShell-7.1.3-win-x86.zip + 8c2ce510b5c641aad2da6adefc92d47e09bc842d47db3b5d15e14859555a74fe13ad52eaeabf1b2954ca9af737e628b567731c8a3db9bbf0e4aad05279bc1fd8 + PowerShell-7.1.3-win-x86.zip From a060d774ee8c75bb5834368296938607946337e1 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 16 Apr 2021 20:48:00 -0700 Subject: [PATCH 03/20] Update CUDA to 11.3 on Linux. --- scripts/azure-pipelines/linux/provision-image.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh index c04f4b3caac191..37487310b05bef 100755 --- a/scripts/azure-pipelines/linux/provision-image.sh +++ b/scripts/azure-pipelines/linux/provision-image.sh @@ -58,14 +58,12 @@ cd ~ curl -sSL https://get.haskellstack.org/ | sudo sh # Install CUDA -wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_10.2.89-1_amd64.deb -sudo apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub -sudo dpkg -i cuda-repo-ubuntu1804_10.2.89-1_amd64.deb -wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb -sudo dpkg -i nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb +wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-ubuntu1804.pin +sudo mv cuda-ubuntu1804.pin /etc/apt/preferences.d/cuda-repository-pin-600 +sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub +sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /" sudo apt -y update -sudo apt install -y --no-install-recommends cuda-compiler-10-2 cuda-libraries-dev-10-2 cuda-driver-dev-10-2 cuda-cudart-dev-10-2 libcublas10 cuda-curand-dev-10-2 -sudo apt install -y --no-install-recommends libcudnn7-dev +sudo apt install -y --no-install-recommends cuda-compiler-11-3 cuda-libraries-dev-11-3 cuda-driver-dev-11-3 cuda-cudart-dev-11-3 libcublas-11-3 libcurand-dev-11-3 libcudnn8-dev # Install PowerShell wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb From 99e068f27bc18ba81a27e1645d2ee8c3d9e42668 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Sun, 18 Apr 2021 21:07:06 -0700 Subject: [PATCH 04/20] "Explode" VM provisioning scripts for more consistent feedback during deploy. This resolves the deployment script often hanging with no feedback as to why. --- .../azure-pipelines/windows/create-vmss.ps1 | 123 ++++- .../azure-pipelines/windows/deploy-cuda.ps1 | 57 +++ .../azure-pipelines/windows/deploy-mpi.ps1 | 47 ++ .../azure-pipelines/windows/deploy-psexec.ps1 | 8 + .../azure-pipelines/windows/deploy-pwsh.ps1 | 9 + .../windows/deploy-settings.txt | 20 + .../windows/deploy-visual-studio.ps1 | 86 ++++ .../windows/deploy-windows-wdk.ps1 | 47 ++ .../windows/drop-to-admin-user-prefix.ps1 | 27 ++ .../windows/provision-image.txt | 455 ------------------ .../windows/utility-prefix.ps1 | 125 +++++ 11 files changed, 535 insertions(+), 469 deletions(-) create mode 100644 scripts/azure-pipelines/windows/deploy-cuda.ps1 create mode 100644 scripts/azure-pipelines/windows/deploy-mpi.ps1 create mode 100644 scripts/azure-pipelines/windows/deploy-psexec.ps1 create mode 100644 scripts/azure-pipelines/windows/deploy-pwsh.ps1 create mode 100644 scripts/azure-pipelines/windows/deploy-settings.txt create mode 100644 scripts/azure-pipelines/windows/deploy-visual-studio.ps1 create mode 100644 scripts/azure-pipelines/windows/deploy-windows-wdk.ps1 create mode 100644 scripts/azure-pipelines/windows/drop-to-admin-user-prefix.ps1 delete mode 100644 scripts/azure-pipelines/windows/provision-image.txt create mode 100644 scripts/azure-pipelines/windows/utility-prefix.ps1 diff --git a/scripts/azure-pipelines/windows/create-vmss.ps1 b/scripts/azure-pipelines/windows/create-vmss.ps1 index 608465100f6dfe..7e2e42761afd02 100644 --- a/scripts/azure-pipelines/windows/create-vmss.ps1 +++ b/scripts/azure-pipelines/windows/create-vmss.ps1 @@ -43,9 +43,9 @@ $WindowsServerSku = '2019-Datacenter' $ErrorActionPreference = 'Stop' $ProgressActivity = 'Creating Scale Set' -$TotalProgress = 12 +$TotalProgress = 18 if ($Unstable) { - $TotalProgress -= 1 # skipping the archives share part + $TotalProgress -= 2 # skipping 2 archives share parts } $CurrentProgress = 1 @@ -245,30 +245,125 @@ New-AzVm ` #################################################################################################### Write-Progress ` -Activity $ProgressActivity ` - -Status 'Running provisioning script provision-image.txt (as a .ps1) in VM' ` + -Status 'Running provisioning script deploy-psexec.ps1 in VM' ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) -$provisionParameters = @{AdminUserPassword = $AdminPW;} -if (-Not $Unstable) { - $provisionParameters['StorageAccountName'] = $StorageAccountName - $provisionParameters['StorageAccountSasToken'] = $SasToken -} - -$ProvisionImageResult = Invoke-AzVMRunCommand ` +$DeployPsExecResult = Invoke-AzVMRunCommand ` -ResourceGroupName $ResourceGroupName ` -VMName $ProtoVMName ` -CommandId 'RunPowerShellScript' ` - -ScriptPath "$PSScriptRoot\provision-image.txt" ` - -Parameter $provisionParameters + -ScriptPath "$PSScriptRoot\deploy-psexec.ps1" + +Write-Host "deploy-psexec.ps1 output: $($DeployPsExecResult.value.Message)" + +#################################################################################################### +function Invoke-ScriptWithPrefix { + param( + [string]$ScriptName, + [switch]$AddAdminPw + ) + + Write-Progress ` + -Activity $ProgressActivity ` + -Status "Running provisioning script $ScriptName in VM" ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) + + $DropToAdminUserPrefix = Get-Content "$PSScriptRoot\drop-to-admin-user-prefix.ps1" -Encoding utf8NoBOM -Raw + $UtilityPrefixContent = Get-Content "$PSScriptRoot\utility-prefix.ps1" -Encoding utf8NoBOM -Raw + + $tempScriptFilename = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".txt" + try { + $script = Get-Content "$PSScriptRoot\$ScriptName" -Encoding utf8NoBOM -Raw + if ($AddAdminPw) { + $script = $script.Replace('# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1', $DropToAdminUserPrefix) + } + + $script = $script.Replace('# REPLACE WITH UTILITY-PREFIX.ps1', $UtilityPrefixContent); + Set-Content -Path $tempScriptFilename -Value $script -Encoding utf8NoBOM + + $parameter = $null + if ($AddAdminPw) { + $parameter = @{AdminUserPassword = $AdminPW;} + } + + $InvokeResult = Invoke-AzVMRunCommand ` + -ResourceGroupName $ResourceGroupName ` + -VMName $ProtoVMName ` + -CommandId 'RunPowerShellScript' ` + -ScriptPath $tempScriptFilename ` + -Parameter $parameter + + Write-Host "$ScriptName output: $($InvokeResult.value.Message)" + } finally { + Remove-Item $tempScriptFilename -Force + } +} + +Invoke-ScriptWithPrefix -ScriptName 'deploy-visual-studio.ps1' -AddAdminPw +Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName -Write-Host "provision-image.ps1 output: $($ProvisionImageResult.value.Message)" +#################################################################################################### +Invoke-ScriptWithPrefix -ScriptName 'deploy-windows-wdk.ps1' -AddAdminPw +Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName + +#################################################################################################### +Invoke-ScriptWithPrefix -ScriptName 'deploy-mpi.ps1' -AddAdminPw +Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName + +#################################################################################################### +Invoke-ScriptWithPrefix -ScriptName 'deploy-cuda.ps1' -AddAdminPw +Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName + +#################################################################################################### +Invoke-ScriptWithPrefix -ScriptName 'deploy-pwsh.ps1' -AddAdminPw +Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName #################################################################################################### Write-Progress ` -Activity $ProgressActivity ` - -Status 'Restarting VM' ` + -Status 'Running provisioning script deploy-settings.txt (as a .ps1) in VM' ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) +$ProvisionImageResult = Invoke-AzVMRunCommand ` + -ResourceGroupName $ResourceGroupName ` + -VMName $ProtoVMName ` + -CommandId 'RunPowerShellScript' ` + -ScriptPath "$PSScriptRoot\deploy-settings.txt" + +Write-Host "deploy-settings.txt output: $($ProvisionImageResult.value.Message)" + +if (-Not $Unstable) { +#################################################################################################### + Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Deploying SAS token into VM' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) + + $tempScriptFilename = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".txt" + try { + $script = "Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' " ` + + "-Name PROVISIONED_AZURE_STORAGE_NAME " ` + + "-Value '$StorageAccountName'`r`n" ` + + "Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' " ` + + "-Name PROVISIONED_AZURE_STORAGE_SAS_TOKEN " ` + + "-Value '$SasToken'`r`n" + + Write-Host "Script content is:" + Write-Host $script + + Set-Content -Path $tempScriptFilename -Value $script -Encoding utf8NoBOM + $InvokeResult = Invoke-AzVMRunCommand ` + -ResourceGroupName $ResourceGroupName ` + -VMName $ProtoVMName ` + -CommandId 'RunPowerShellScript' ` + -ScriptPath $tempScriptFilename + + Write-Host "Deploy SAS token output: $($InvokeResult.value.Message)" + } finally { + Remove-Item $tempScriptFilename -Force + } +} # -Not $Unstable + Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName #################################################################################################### diff --git a/scripts/azure-pipelines/windows/deploy-cuda.ps1 b/scripts/azure-pipelines/windows/deploy-cuda.ps1 new file mode 100644 index 00000000000000..6f9dd55ffc380b --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-cuda.ps1 @@ -0,0 +1,57 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1 + +# REPLACE WITH UTILITY-PREFIX.ps1 + +$CudaUrl = 'https://developer.download.nvidia.com/compute/cuda/11.3.0/network_installers/cuda_11.3.0_win10_network.exe' + +$CudaFeatures = 'nvcc_11.3 cuobjdump_11.3 nvprune_11.3 cupti_11.3 memcheck_11.3 nvdisasm_11.3 nvprof_11.3 ' + ` + 'visual_studio_integration_11.3 visual_profiler_11.3 visual_profiler_11.3 cublas_11.3 cublas_dev_11.3 ' + ` + 'cudart_11.3 cufft_11.3 cufft_dev_11.3 curand_11.3 curand_dev_11.3 cusolver_11.3 cusolver_dev_11.3 ' + ` + 'cusparse_11.3 cusparse_dev_11.3 npp_11.3 npp_dev_11.3 nvrtc_11.3 nvrtc_dev_11.3 nvml_dev_11.3 ' + ` + 'occupancy_calculator_11.3 ' + +<# +.SYNOPSIS +Installs NVIDIA's CUDA Toolkit. + +.DESCRIPTION +InstallCuda installs the CUDA Toolkit with the features specified as a +space-separated list of strings in $Features. + +.PARAMETER Url +The URL of the CUDA installer. + +.PARAMETER Features +A space-separated list of features to install. +#> +Function InstallCuda { + Param( + [String]$Url, + [String]$Features + ) + + try { + Write-Host 'Downloading CUDA...' + [string]$installerPath = Get-TempFilePath -Extension 'exe' + curl.exe -L -o $installerPath -s -S $Url + Write-Host 'Installing CUDA...' + $proc = Start-Process -FilePath $installerPath -ArgumentList @('-s ' + $Features) -Wait -PassThru + $exitCode = $proc.ExitCode + if ($exitCode -eq 0) { + Write-Host 'Installation successful!' + } + else { + Write-Error "Installation failed! Exited with $exitCode." + throw + } + } + catch { + Write-Error "Failed to install CUDA! $($_.Exception.Message)" + throw + } +} + +InstallCuda -Url $CudaUrl -Features $CudaFeatures diff --git a/scripts/azure-pipelines/windows/deploy-mpi.ps1 b/scripts/azure-pipelines/windows/deploy-mpi.ps1 new file mode 100644 index 00000000000000..ba369f7971d733 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-mpi.ps1 @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1 + +# REPLACE WITH UTILITY-PREFIX.ps1 + +$MpiUrl = 'https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisetup.exe' + +<# +.SYNOPSIS +Installs MPI + +.DESCRIPTION +Downloads the MPI installer located at $Url, and installs it with the +correct flags. + +.PARAMETER Url +The URL of the installer. +#> +Function InstallMpi { + Param( + [String]$Url + ) + + try { + Write-Host 'Downloading MPI...' + [string]$installerPath = Get-TempFilePath -Extension 'exe' + curl.exe -L -o $installerPath -s -S $Url + Write-Host 'Installing MPI...' + $proc = Start-Process -FilePath $installerPath -ArgumentList @('-force', '-unattend') -Wait -PassThru + $exitCode = $proc.ExitCode + if ($exitCode -eq 0) { + Write-Host 'Installation successful!' + } + else { + Write-Error "Installation failed! Exited with $exitCode." + throw + } + } + catch { + Write-Error "Failed to install MPI! $($_.Exception.Message)" + throw + } +} + +InstallMpi -Url $MpiUrl diff --git a/scripts/azure-pipelines/windows/deploy-psexec.ps1 b/scripts/azure-pipelines/windows/deploy-psexec.ps1 new file mode 100644 index 00000000000000..2a1f6fb14a87e4 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-psexec.ps1 @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' +$PsExecPath = 'C:\PsExec64.exe' +Write-Host "Downloading psexec to: $PsExecPath" +& curl.exe -L -o $PsExecPath -s -S https://live.sysinternals.com/PsExec64.exe diff --git a/scripts/azure-pipelines/windows/deploy-pwsh.ps1 b/scripts/azure-pipelines/windows/deploy-pwsh.ps1 new file mode 100644 index 00000000000000..b766385a308511 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-pwsh.ps1 @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1 + +# REPLACE WITH UTILITY-PREFIX.ps1 + +$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.1.3/PowerShell-7.1.3-win-x64.msi' +InstallMSI -Url $PwshUrl -Name 'PowerShell Core' diff --git a/scripts/azure-pipelines/windows/deploy-settings.txt b/scripts/azure-pipelines/windows/deploy-settings.txt new file mode 100644 index 00000000000000..a80ddce399a281 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-settings.txt @@ -0,0 +1,20 @@ +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' + +Write-Host 'Disabling pagefile...' +wmic computersystem set AutomaticManagedPagefile=False +wmic pagefileset delete + +$av = Get-Command Add-MPPreference -ErrorAction SilentlyContinue +if ($null -eq $av) { + Write-Host 'AntiVirus not installed, skipping exclusions.' +} else { + Write-Host 'Configuring AntiVirus exclusions...' + Add-MpPreference -ExclusionPath C:\agent + Add-MPPreference -ExclusionPath D:\ + Add-MPPreference -ExclusionProcess ninja.exe + Add-MPPreference -ExclusionProcess clang-cl.exe + Add-MPPreference -ExclusionProcess cl.exe + Add-MPPreference -ExclusionProcess link.exe + Add-MPPreference -ExclusionProcess python.exe +} diff --git a/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 b/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 new file mode 100644 index 00000000000000..2de2de919556ca --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1 + +# REPLACE WITH UTILITY-PREFIX.ps1 + +$VisualStudioBootstrapperUrl = 'https://aka.ms/vs/16/release/vs_enterprise.exe' +$Workloads = @( + 'Microsoft.VisualStudio.Workload.NativeDesktop', + 'Microsoft.VisualStudio.Workload.Universal', + 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', + 'Microsoft.VisualStudio.Component.VC.Tools.ARM', + 'Microsoft.VisualStudio.Component.VC.Tools.ARM64', + 'Microsoft.VisualStudio.Component.VC.ATL', + 'Microsoft.VisualStudio.Component.VC.ATLMFC', + 'Microsoft.VisualStudio.Component.VC.v141.x86.x64.Spectre', + 'Microsoft.VisualStudio.Component.Windows10SDK.18362', + 'Microsoft.VisualStudio.Component.Windows10SDK.19041', + 'Microsoft.Net.Component.4.8.SDK', + 'Microsoft.Component.NetFX.Native', + 'Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset', + 'Microsoft.VisualStudio.Component.VC.Llvm.Clang', + 'Microsoft.VisualStudio.Component.VC.v141.x86.x64', + 'Microsoft.VisualStudio.Component.VC.140' +) + +<# +.SYNOPSIS +Install Visual Studio. + +.DESCRIPTION +InstallVisualStudio takes the $Workloads array, and installs it with the +installer that's pointed at by $BootstrapperUrl. + +.PARAMETER Workloads +The set of VS workloads to install. + +.PARAMETER BootstrapperUrl +The URL of the Visual Studio installer, i.e. one of vs_*.exe. + +.PARAMETER InstallPath +The path to install Visual Studio at. + +.PARAMETER Nickname +The nickname to give the installation. +#> +Function InstallVisualStudio { + Param( + [String[]]$Workloads, + [String]$BootstrapperUrl, + [String]$InstallPath = $null, + [String]$Nickname = $null + ) + + try { + Write-Host 'Downloading Visual Studio...' + [string]$bootstrapperExe = Get-TempFilePath -Extension 'exe' + curl.exe -L -o $bootstrapperExe -s -S $BootstrapperUrl + Write-Host 'Installing Visual Studio...' + $vsArgs = @('/c', $bootstrapperExe, '--quiet', '--norestart', '--wait', '--nocache') + foreach ($workload in $Workloads) { + $vsArgs += '--add' + $vsArgs += $workload + } + + if (-not ([String]::IsNullOrWhiteSpace($InstallPath))) { + $vsArgs += '--installpath' + $vsArgs += $InstallPath + } + + if (-not ([String]::IsNullOrWhiteSpace($Nickname))) { + $vsArgs += '--nickname' + $vsArgs += $Nickname + } + + $proc = Start-Process -FilePath cmd.exe -ArgumentList $vsArgs -Wait -PassThru + PrintMsiExitCodeMessage $proc.ExitCode + } + catch { + Write-Error "Failed to install Visual Studio! $($_.Exception.Message)" + throw + } +} + +InstallVisualStudio -Workloads $Workloads -BootstrapperUrl $VisualStudioBootstrapperUrl -Nickname 'Stable' diff --git a/scripts/azure-pipelines/windows/deploy-windows-wdk.ps1 b/scripts/azure-pipelines/windows/deploy-windows-wdk.ps1 new file mode 100644 index 00000000000000..d145f79d71a8ec --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-windows-wdk.ps1 @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1 + +# REPLACE WITH UTILITY-PREFIX.ps1 + +$WindowsWDKUrl = 'https://go.microsoft.com/fwlink/?linkid=2128854' + +<# +.SYNOPSIS +Installs Windows WDK version 2004 + +.DESCRIPTION +Downloads the Windows WDK installer located at $Url, and installs it with the +correct flags. + +.PARAMETER Url +The URL of the installer. +#> +Function InstallWindowsWDK { + Param( + [String]$Url + ) + + try { + Write-Host 'Downloading Windows WDK...' + [string]$installerPath = Get-TempFilePath -Extension 'exe' + curl.exe -L -o $installerPath -s -S $Url + Write-Host 'Installing Windows WDK...' + $proc = Start-Process -FilePath $installerPath -ArgumentList @('/features', '+', '/q') -Wait -PassThru + $exitCode = $proc.ExitCode + if ($exitCode -eq 0) { + Write-Host 'Installation successful!' + } + else { + Write-Error "Installation failed! Exited with $exitCode." + throw + } + } + catch { + Write-Error "Failed to install Windows WDK! $($_.Exception.Message)" + throw + } +} + +InstallWindowsWDK -Url $WindowsWDKUrl diff --git a/scripts/azure-pipelines/windows/drop-to-admin-user-prefix.ps1 b/scripts/azure-pipelines/windows/drop-to-admin-user-prefix.ps1 new file mode 100644 index 00000000000000..b4592eabd4eee4 --- /dev/null +++ b/scripts/azure-pipelines/windows/drop-to-admin-user-prefix.ps1 @@ -0,0 +1,27 @@ +param( + [string]$AdminUserPassword = $null +) + +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' +if (-Not [string]::IsNullOrEmpty($AdminUserPassword)) { + $PsExecPath = 'C:\PsExec64.exe' + $PsExecArgs = @( + '-u', + 'AdminUser', + '-p', + $AdminUserPassword, + '-accepteula', + '-i', + '-h', + 'C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe', + '-ExecutionPolicy', + 'Unrestricted', + '-File', + $PSCommandPath + ) + + Write-Host "Executing: $PsExecPath $PsExecArgs" + $proc = Start-Process -FilePath $PsExecPath -ArgumentList $PsExecArgs -Wait -PassThru + exit $proc.ExitCode +} diff --git a/scripts/azure-pipelines/windows/provision-image.txt b/scripts/azure-pipelines/windows/provision-image.txt deleted file mode 100644 index 7e5533289b3d49..00000000000000 --- a/scripts/azure-pipelines/windows/provision-image.txt +++ /dev/null @@ -1,455 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: MIT - -<# -.SYNOPSIS -Sets up a machine to be an image for a scale set. - -.DESCRIPTION -provision-image.ps1 runs on an existing, freshly provisioned virtual machine, -and sets up that virtual machine as a build machine. After this is done, -(outside of this script), we take that machine and make it an image to be copied -for setting up new VMs in the scale set. - -This script must either be run as admin, or one must pass AdminUserPassword; -if the script is run with AdminUserPassword, it runs itself again as an -administrator. - -.PARAMETER AdminUserPassword -The administrator user's password; if this is $null, or not passed, then the -script assumes it's running on an administrator account. - -.PARAMETER StorageAccountName -The name of the storage account. Stored in the environment variable %PROVISIONED_AZURE_STORAGE_NAME%. -Used by the CI system to access the global storage. - -.PARAMETER StorageAccountSasToken -The SAS token to access the storage account. Stored in the environment variable -%PROVISIONED_AZURE_STORAGE_SAS_TOKEN%. -Used by the CI system to access the global storage. -#> -param( - [string]$AdminUserPassword = $null, - [string]$StorageAccountName = $null, - [string]$StorageAccountSasToken = $null -) - -$ErrorActionPreference = 'Stop' - -<# -.SYNOPSIS -Gets a random file path in the temp directory. - -.DESCRIPTION -Get-TempFilePath takes an extension, and returns a path with a random -filename component in the temporary directory with that extension. - -.PARAMETER Extension -The extension to use for the path. -#> -Function Get-TempFilePath { - Param( - [String]$Extension - ) - - if ([String]::IsNullOrWhiteSpace($Extension)) { - throw 'Missing Extension' - } - - $tempPath = [System.IO.Path]::GetTempPath() - $tempName = [System.IO.Path]::GetRandomFileName() + '.' + $Extension - return Join-Path $tempPath $tempName -} - -$TranscriptPath = 'C:\provision-image-transcript.txt' - -if ([string]::IsNullOrEmpty($AdminUserPassword)) { - Start-Transcript -Path $TranscriptPath -} else { - Write-Host 'AdminUser password supplied; switching to AdminUser.' - $PsExecPath = Get-TempFilePath -Extension 'exe' - Write-Host "Downloading psexec to: $PsExecPath" - & curl.exe -L -o $PsExecPath -s -S https://live.sysinternals.com/PsExec64.exe - $PsExecArgs = @( - '-u', - 'AdminUser', - '-p', - $AdminUserPassword, - '-accepteula', - '-h', - 'C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe', - '-ExecutionPolicy', - 'Unrestricted', - '-File', - $PSCommandPath - ) - - if (-Not ([string]::IsNullOrWhiteSpace($StorageAccountName))) { - $PsExecArgs += '-StorageAccountName' - $PsExecArgs += $StorageAccountName - } - - if (-Not ([string]::IsNullOrWhiteSpace($StorageAccountSasToken))) { - $PsExecArgs += '-StorageAccountSasToken' - $PsExecArgs += $StorageAccountSasToken - } - - Write-Host "Executing: $PsExecPath $PsExecArgs" - - $proc = Start-Process -FilePath $PsExecPath -ArgumentList $PsExecArgs -Wait -PassThru - Write-Host 'Reading transcript...' - Get-Content -Path $TranscriptPath - Write-Host 'Cleaning up...' - Remove-Item $PsExecPath - exit $proc.ExitCode -} - -$VisualStudioBootstrapperUrl = 'https://aka.ms/vs/16/release/vs_enterprise.exe' -$Workloads = @( - 'Microsoft.VisualStudio.Workload.NativeDesktop', - 'Microsoft.VisualStudio.Workload.Universal', - 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', - 'Microsoft.VisualStudio.Component.VC.Tools.ARM', - 'Microsoft.VisualStudio.Component.VC.Tools.ARM64', - 'Microsoft.VisualStudio.Component.VC.ATL', - 'Microsoft.VisualStudio.Component.VC.ATLMFC', - 'Microsoft.VisualStudio.Component.VC.v141.x86.x64.Spectre', - 'Microsoft.VisualStudio.Component.Windows10SDK.18362', - 'Microsoft.VisualStudio.Component.Windows10SDK.19041', - 'Microsoft.Net.Component.4.8.SDK', - 'Microsoft.Component.NetFX.Native', - 'Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset', - 'Microsoft.VisualStudio.Component.VC.Llvm.Clang', - 'Microsoft.VisualStudio.Component.VC.v141.x86.x64', - 'Microsoft.VisualStudio.Component.VC.140' -) - -$WindowsWDKUrl = 'https://go.microsoft.com/fwlink/?linkid=2128854' - -$MpiUrl = 'https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisetup.exe' - -$CudaUrl = 'https://developer.download.nvidia.com/compute/cuda/11.3.0/network_installers/cuda_11.3.0_win10_network.exe' - -$CudaFeatures = 'nvcc_11.3 cuobjdump_11.3 nvprune_11.3 cupti_11.3 memcheck_11.3 nvdisasm_11.3 nvprof_11.3 ' + ` - 'visual_studio_integration_11.3 visual_profiler_11.3 visual_profiler_11.3 cublas_11.3 cublas_dev_11.3 ' + ` - 'cudart_11.3 cufft_11.3 cufft_dev_11.3 curand_11.3 curand_dev_11.3 cusolver_11.3 cusolver_dev_11.3 ' + ` - 'cusparse_11.3 cusparse_dev_11.3 npp_11.3 npp_dev_11.3 nvrtc_11.3 nvrtc_dev_11.3 nvml_dev_11.3 ' + ` - 'occupancy_calculator_11.3 ' - -$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.1.3/PowerShell-7.1.3-win-x64.msi' - -$ErrorActionPreference = 'Stop' -$ProgressPreference = 'SilentlyContinue' - -<# -.SYNOPSIS -Writes a message to the screen depending on ExitCode. - -.DESCRIPTION -Since msiexec can return either 0 or 3010 successfully, in both cases -we write that installation succeeded, and which exit code it exited with. -If msiexec returns anything else, we write an error. - -.PARAMETER ExitCode -The exit code that msiexec returned. -#> -Function PrintMsiExitCodeMessage { - Param( - $ExitCode - ) - - # 3010 is probably ERROR_SUCCESS_REBOOT_REQUIRED - if ($ExitCode -eq 0 -or $ExitCode -eq 3010) { - Write-Host "Installation successful! Exited with $ExitCode." - } - else { - Write-Error "Installation failed! Exited with $ExitCode." - throw - } -} - -<# -.SYNOPSIS -Install Visual Studio. - -.DESCRIPTION -InstallVisualStudio takes the $Workloads array, and installs it with the -installer that's pointed at by $BootstrapperUrl. - -.PARAMETER Workloads -The set of VS workloads to install. - -.PARAMETER BootstrapperUrl -The URL of the Visual Studio installer, i.e. one of vs_*.exe. - -.PARAMETER InstallPath -The path to install Visual Studio at. - -.PARAMETER Nickname -The nickname to give the installation. -#> -Function InstallVisualStudio { - Param( - [String[]]$Workloads, - [String]$BootstrapperUrl, - [String]$InstallPath = $null, - [String]$Nickname = $null - ) - - try { - Write-Host 'Downloading Visual Studio...' - [string]$bootstrapperExe = Get-TempFilePath -Extension 'exe' - curl.exe -L -o $bootstrapperExe -s -S $BootstrapperUrl - Write-Host 'Installing Visual Studio...' - $args = @('/c', $bootstrapperExe, '--quiet', '--norestart', '--wait', '--nocache') - foreach ($workload in $Workloads) { - $args += '--add' - $args += $workload - } - - if (-not ([String]::IsNullOrWhiteSpace($InstallPath))) { - $args += '--installpath' - $args += $InstallPath - } - - if (-not ([String]::IsNullOrWhiteSpace($Nickname))) { - $args += '--nickname' - $args += $Nickname - } - - $proc = Start-Process -FilePath cmd.exe -ArgumentList $args -Wait -PassThru - PrintMsiExitCodeMessage $proc.ExitCode - } - catch { - Write-Error "Failed to install Visual Studio! $($_.Exception.Message)" - throw - } -} - -<# -.SYNOPSIS -Install a .msi file. - -.DESCRIPTION -InstallMSI takes a url where an .msi lives, and installs that .msi to the system. - -.PARAMETER Name -The name of the thing to install. - -.PARAMETER Url -The URL at which the .msi lives. -#> -Function InstallMSI { - Param( - [String]$Name, - [String]$Url - ) - - try { - Write-Host "Downloading $Name..." - [string]$msiPath = Get-TempFilePath -Extension 'msi' - curl.exe -L -o $msiPath -s -S $Url - Write-Host "Installing $Name..." - $args = @('/i', $msiPath, '/norestart', '/quiet', '/qn') - $proc = Start-Process -FilePath 'msiexec.exe' -ArgumentList $args -Wait -PassThru - PrintMsiExitCodeMessage $proc.ExitCode - } - catch { - Write-Error "Failed to install $Name! $($_.Exception.Message)" - throw - } -} - -<# -.SYNOPSIS -Unpacks a zip file to $Dir. - -.DESCRIPTION -InstallZip takes a URL of a zip file, and unpacks the zip file to the directory -$Dir. - -.PARAMETER Name -The name of the tool being installed. - -.PARAMETER Url -The URL of the zip file to unpack. - -.PARAMETER Dir -The directory to unpack the zip file to. -#> -Function InstallZip { - Param( - [String]$Name, - [String]$Url, - [String]$Dir - ) - - try { - Write-Host "Downloading $Name..." - [string]$zipPath = Get-TempFilePath -Extension 'zip' - curl.exe -L -o $zipPath -s -S $Url - Write-Host "Installing $Name..." - Expand-Archive -Path $zipPath -DestinationPath $Dir -Force - } - catch { - Write-Error "Failed to install $Name! $($_.Exception.Message)" - throw - } -} - -<# -.SYNOPSIS -Installs Windows WDK version 2004 - -.DESCRIPTION -Downloads the Windows WDK installer located at $Url, and installs it with the -correct flags. - -.PARAMETER Url -The URL of the installer. -#> -Function InstallWindowsWDK { - Param( - [String]$Url - ) - - try { - Write-Host 'Downloading Windows WDK...' - [string]$installerPath = Get-TempFilePath -Extension 'exe' - curl.exe -L -o $installerPath -s -S $Url - Write-Host 'Installing Windows WDK...' - $proc = Start-Process -FilePath $installerPath -ArgumentList @('/features', '+', '/q') -Wait -PassThru - $exitCode = $proc.ExitCode - if ($exitCode -eq 0) { - Write-Host 'Installation successful!' - } - else { - Write-Error "Installation failed! Exited with $exitCode." - throw - } - } - catch { - Write-Error "Failed to install Windows WDK! $($_.Exception.Message)" - throw - } -} - -<# -.SYNOPSIS -Installs MPI - -.DESCRIPTION -Downloads the MPI installer located at $Url, and installs it with the -correct flags. - -.PARAMETER Url -The URL of the installer. -#> -Function InstallMpi { - Param( - [String]$Url - ) - - try { - Write-Host 'Downloading MPI...' - [string]$installerPath = Get-TempFilePath -Extension 'exe' - curl.exe -L -o $installerPath -s -S $Url - Write-Host 'Installing MPI...' - $proc = Start-Process -FilePath $installerPath -ArgumentList @('-force', '-unattend') -Wait -PassThru - $exitCode = $proc.ExitCode - if ($exitCode -eq 0) { - Write-Host 'Installation successful!' - } - else { - Write-Error "Installation failed! Exited with $exitCode." - throw - } - } - catch { - Write-Error "Failed to install MPI! $($_.Exception.Message)" - throw - } -} - -<# -.SYNOPSIS -Installs NVIDIA's CUDA Toolkit. - -.DESCRIPTION -InstallCuda installs the CUDA Toolkit with the features specified as a -space-separated list of strings in $Features. - -.PARAMETER Url -The URL of the CUDA installer. - -.PARAMETER Features -A space-separated list of features to install. -#> -Function InstallCuda { - Param( - [String]$Url, - [String]$Features - ) - - try { - Write-Host 'Downloading CUDA...' - [string]$installerPath = Get-TempFilePath -Extension 'exe' - curl.exe -L -o $installerPath -s -S $Url - Write-Host 'Installing CUDA...' - $proc = Start-Process -FilePath $installerPath -ArgumentList @('-s ' + $Features) -Wait -PassThru - $exitCode = $proc.ExitCode - if ($exitCode -eq 0) { - Write-Host 'Installation successful!' - } - else { - Write-Error "Installation failed! Exited with $exitCode." - throw - } - } - catch { - Write-Error "Failed to install CUDA! $($_.Exception.Message)" - throw - } -} - -Write-Host "AdminUser password not supplied; assuming already running as AdminUser" - -Write-Host 'Disabling pagefile...' -wmic computersystem set AutomaticManagedPagefile=False -wmic pagefileset delete - -$av = Get-Command Add-MPPreference -ErrorAction SilentlyContinue -if ($null -eq $av) { - Write-Host 'AntiVirus not installed, skipping exclusions.' -} else { - Write-Host 'Configuring AntiVirus exclusions...' - Add-MpPreference -ExclusionPath C:\agent - Add-MPPreference -ExclusionPath D:\ - Add-MPPreference -ExclusionProcess ninja.exe - Add-MPPreference -ExclusionProcess clang-cl.exe - Add-MPPreference -ExclusionProcess cl.exe - Add-MPPreference -ExclusionProcess link.exe - Add-MPPreference -ExclusionProcess python.exe -} - -InstallVisualStudio -Workloads $Workloads -BootstrapperUrl $VisualStudioBootstrapperUrl -Nickname 'Stable' -InstallWindowsWDK -Url $WindowsWDKUrl -InstallMpi -Url $MpiUrl -InstallCuda -Url $CudaUrl -Features $CudaFeatures -InstallMSI -Url $PwshUrl -Name 'PowerShell Core' -if ([string]::IsNullOrWhiteSpace($StorageAccountName)) { - Write-Host 'No storage account name configured.' -} else { - Write-Host 'Storing storage account name to environment' - Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' ` - -Name PROVISIONED_AZURE_STORAGE_NAME ` - -Value $StorageAccountName -} -if ([string]::IsNullOrWhiteSpace($StorageAccountSasToken)) { - Write-Host 'No storage account key configured.' -} else { - Write-Host 'Storing storage account key to environment' - Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' ` - -Name PROVISIONED_AZURE_STORAGE_SAS_TOKEN ` - -Value $StorageAccountSasToken -} diff --git a/scripts/azure-pipelines/windows/utility-prefix.ps1 b/scripts/azure-pipelines/windows/utility-prefix.ps1 new file mode 100644 index 00000000000000..8cd0066db60097 --- /dev/null +++ b/scripts/azure-pipelines/windows/utility-prefix.ps1 @@ -0,0 +1,125 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +<# +.SYNOPSIS +Gets a random file path in the temp directory. + +.DESCRIPTION +Get-TempFilePath takes an extension, and returns a path with a random +filename component in the temporary directory with that extension. + +.PARAMETER Extension +The extension to use for the path. +#> +Function Get-TempFilePath { + Param( + [String]$Extension + ) + + if ([String]::IsNullOrWhiteSpace($Extension)) { + throw 'Missing Extension' + } + + $tempPath = [System.IO.Path]::GetTempPath() + $tempName = [System.IO.Path]::GetRandomFileName() + '.' + $Extension + return Join-Path $tempPath $tempName +} + +<# +.SYNOPSIS +Writes a message to the screen depending on ExitCode. + +.DESCRIPTION +Since msiexec can return either 0 or 3010 successfully, in both cases +we write that installation succeeded, and which exit code it exited with. +If msiexec returns anything else, we write an error. + +.PARAMETER ExitCode +The exit code that msiexec returned. +#> +Function PrintMsiExitCodeMessage { + Param( + $ExitCode + ) + + # 3010 is probably ERROR_SUCCESS_REBOOT_REQUIRED + if ($ExitCode -eq 0 -or $ExitCode -eq 3010) { + Write-Host "Installation successful! Exited with $ExitCode." + } + else { + Write-Error "Installation failed! Exited with $ExitCode." + throw + } +} + +<# +.SYNOPSIS +Install a .msi file. + +.DESCRIPTION +InstallMSI takes a url where an .msi lives, and installs that .msi to the system. + +.PARAMETER Name +The name of the thing to install. + +.PARAMETER Url +The URL at which the .msi lives. +#> +Function InstallMSI { + Param( + [String]$Name, + [String]$Url + ) + + try { + Write-Host "Downloading $Name..." + [string]$msiPath = Get-TempFilePath -Extension 'msi' + curl.exe -L -o $msiPath -s -S $Url + Write-Host "Installing $Name..." + $args = @('/i', $msiPath, '/norestart', '/quiet', '/qn') + $proc = Start-Process -FilePath 'msiexec.exe' -ArgumentList $args -Wait -PassThru + PrintMsiExitCodeMessage $proc.ExitCode + } + catch { + Write-Error "Failed to install $Name! $($_.Exception.Message)" + throw + } +} + +<# +.SYNOPSIS +Unpacks a zip file to $Dir. + +.DESCRIPTION +InstallZip takes a URL of a zip file, and unpacks the zip file to the directory +$Dir. + +.PARAMETER Name +The name of the tool being installed. + +.PARAMETER Url +The URL of the zip file to unpack. + +.PARAMETER Dir +The directory to unpack the zip file to. +#> +Function InstallZip { + Param( + [String]$Name, + [String]$Url, + [String]$Dir + ) + + try { + Write-Host "Downloading $Name..." + [string]$zipPath = Get-TempFilePath -Extension 'zip' + curl.exe -L -o $zipPath -s -S $Url + Write-Host "Installing $Name..." + Expand-Archive -Path $zipPath -DestinationPath $Dir -Force + } + catch { + Write-Error "Failed to install $Name! $($_.Exception.Message)" + throw + } +} From f8d24615021ed8f28b26826dfc9316fd20face25 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Mon, 19 Apr 2021 14:52:35 -0700 Subject: [PATCH 05/20] [libdatachannel] Fix use of deprecated result_type typedef. Submitted upstream as https://github.com/paullouisageneau/libdatachannel/pull/413 --- ...ADAPTOR_TYPEDEFS_DEPRECATION_WARNING.patch | 25 +++++++++++++++++++ ports/libdatachannel/portfile.cmake | 5 ++-- ports/libdatachannel/vcpkg.json | 2 +- versions/baseline.json | 2 +- versions/l-/libdatachannel.json | 5 ++++ 5 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 ports/libdatachannel/CXX17_ADAPTOR_TYPEDEFS_DEPRECATION_WARNING.patch diff --git a/ports/libdatachannel/CXX17_ADAPTOR_TYPEDEFS_DEPRECATION_WARNING.patch b/ports/libdatachannel/CXX17_ADAPTOR_TYPEDEFS_DEPRECATION_WARNING.patch new file mode 100644 index 00000000000000..f41945c28af2ba --- /dev/null +++ b/ports/libdatachannel/CXX17_ADAPTOR_TYPEDEFS_DEPRECATION_WARNING.patch @@ -0,0 +1,25 @@ +diff --git a/include/rtc/utils.hpp b/include/rtc/utils.hpp +index f1a82acb..e882042c 100644 +--- a/include/rtc/utils.hpp ++++ b/include/rtc/utils.hpp +@@ -25,6 +25,7 @@ + #include + #include + #include ++#include + + namespace rtc { + +@@ -34,11 +35,10 @@ template overloaded(Ts...) -> overloaded; + // weak_ptr bind helper + template auto weak_bind(F &&f, T *t, Args &&..._args) { + return [bound = std::bind(f, t, _args...), weak_this = t->weak_from_this()](auto &&...args) { +- using result_type = typename decltype(bound)::result_type; + if (auto shared_this = weak_this.lock()) + return bound(args...); + else +- return static_cast(false); ++ return static_cast(false); + }; + } + diff --git a/ports/libdatachannel/portfile.cmake b/ports/libdatachannel/portfile.cmake index 8b1a6c3a9e97ec..563224524f9692 100644 --- a/ports/libdatachannel/portfile.cmake +++ b/ports/libdatachannel/portfile.cmake @@ -1,11 +1,12 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO paullouisageneau/libdatachannel - REF v0.12.0 - SHA512 878d0461255620c21597a6b7132ac0c9d1bcba19e32809cdd007039041eacc1f30a73b1c11c43b8b57a82ca1efeae2090822e6c5e13bd98f5afa086ec147e6e7 + REF 655175d21e58281031c940a94042d5d1fd46efb3 # v 0.12.2 + SHA512 e1e228bf720ef57130fbb9cc33310cebbdbd16c001455cd56e8746b6ee41bac56da5e5a90235e0a826b52711dc3c95b9d9f56d9e406999f9fd384aee2892578d HEAD_REF master PATCHES fix-for-vcpkg.patch + CXX17_ADAPTOR_TYPEDEFS_DEPRECATION_WARNING.patch # submitted upstream as https://github.com/paullouisageneau/libdatachannel/pull/413 ) vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS diff --git a/ports/libdatachannel/vcpkg.json b/ports/libdatachannel/vcpkg.json index 6dddc4dc76c5a3..7ba32db68fba45 100644 --- a/ports/libdatachannel/vcpkg.json +++ b/ports/libdatachannel/vcpkg.json @@ -1,6 +1,6 @@ { "name": "libdatachannel", - "version-semver": "0.12.0", + "version-semver": "0.12.2", "description": "libdatachannel is a standalone implementation of WebRTC Data Channels, WebRTC Media Transport, and WebSockets in C++17 with C bindings for POSIX platforms (including GNU/Linux, Android, and Apple macOS) and Microsoft Windows.", "homepage": "https://github.com/paullouisageneau/libdatachannel", "dependencies": [ diff --git a/versions/baseline.json b/versions/baseline.json index 09f969c6a923ad..d4e7726b1c73ca 100644 --- a/versions/baseline.json +++ b/versions/baseline.json @@ -3009,7 +3009,7 @@ "port-version": 0 }, "libdatachannel": { - "baseline": "0.12.0", + "baseline": "0.12.2", "port-version": 0 }, "libdatrie": { diff --git a/versions/l-/libdatachannel.json b/versions/l-/libdatachannel.json index b85861e672d9dc..2b289cd5d9c467 100644 --- a/versions/l-/libdatachannel.json +++ b/versions/l-/libdatachannel.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "dd3107dd99419236f97e299ce4b61c2379a08f55", + "version-semver": "0.12.2", + "port-version": 0 + }, { "git-tree": "1c0a9c87c0695f1ecc03f342ba87d21ae152f7dd", "version-semver": "0.12.0", From 4cade756105fcc44dccdb22d6be28c60365dee0d Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Mon, 19 Apr 2021 15:21:19 -0700 Subject: [PATCH 06/20] [libvpx] Get the libvpx outputs from the correct place. (Perhaps VS2019 version 16.10 moved where these are placed? I've been defensive and left an attempt to find from the old location in place.) --- ports/libvpx/portfile.cmake | 21 +++++++++++++++------ ports/libvpx/vcpkg.json | 2 +- versions/baseline.json | 2 +- versions/l-/libvpx.json | 5 +++++ 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/ports/libvpx/portfile.cmake b/ports/libvpx/portfile.cmake index e888d7fc412f72..2f3f3680d71f00 100644 --- a/ports/libvpx/portfile.cmake +++ b/ports/libvpx/portfile.cmake @@ -99,14 +99,23 @@ if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) ) # note: pdb file names are hardcoded in the lib file, cannot rename + set(LIBVPX_OUTPUT_PREFIX "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}/${LIBVPX_ARCH_DIR}") if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release") - file(INSTALL "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}/${LIBVPX_ARCH_DIR}/Release/vpx.lib" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") - file(INSTALL "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}/${LIBVPX_ARCH_DIR}/Release/vpx/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Release/vpx.lib" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") + if (EXISTS "${LIBVPX_OUTPUT_PREFIX}/Release/vpx.pdb") + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Release/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") + else() + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Release/vpx/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") + endif() endif() if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - file(INSTALL "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}/${LIBVPX_ARCH_DIR}/Debug/vpx.lib" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") - file(INSTALL "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}/${LIBVPX_ARCH_DIR}/Debug/vpx/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Debug/vpx.lib" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") + if (EXISTS "${LIBVPX_OUTPUT_PREFIX}/Debug/vpx.pdb") + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Debug/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") + else() + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Debug/vpx/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") + endif() endif() if (VCPKG_TARGET_ARCHITECTURE STREQUAL arm64) @@ -130,13 +139,13 @@ else() set(OPTIONS_DEBUG "--enable-debug-libs --enable-debug --prefix=${CURRENT_PACKAGES_DIR}/debug") set(OPTIONS_RELEASE "--prefix=${CURRENT_PACKAGES_DIR}") - + if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") set(OPTIONS "${OPTIONS} --disable-static --enable-shared") else() set(OPTIONS "${OPTIONS} --enable-static --disable-shared") endif() - + if(VCPKG_TARGET_ARCHITECTURE STREQUAL x86) set(LIBVPX_TARGET_ARCH "x86") elseif(VCPKG_TARGET_ARCHITECTURE STREQUAL x64) diff --git a/ports/libvpx/vcpkg.json b/ports/libvpx/vcpkg.json index ce3de03411c26c..f0991561c1233a 100644 --- a/ports/libvpx/vcpkg.json +++ b/ports/libvpx/vcpkg.json @@ -1,7 +1,7 @@ { "name": "libvpx", "version-semver": "1.9.0", - "port-version": 8, + "port-version": 9, "description": "The reference software implementation for the video coding formats VP8 and VP9.", "homepage": "https://github.com/webmproject/libvpx" } diff --git a/versions/baseline.json b/versions/baseline.json index d4e7726b1c73ca..1093bb29caafd5 100644 --- a/versions/baseline.json +++ b/versions/baseline.json @@ -3606,7 +3606,7 @@ }, "libvpx": { "baseline": "1.9.0", - "port-version": 8 + "port-version": 9 }, "libwandio": { "baseline": "4.2.1", diff --git a/versions/l-/libvpx.json b/versions/l-/libvpx.json index 0d90bed5fa44d3..65efd934664f63 100644 --- a/versions/l-/libvpx.json +++ b/versions/l-/libvpx.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "dd62b59825999b4a4136fb9e92b393b5600596d1", + "version-semver": "1.9.0", + "port-version": 9 + }, { "git-tree": "b3c25e1e8a48e31e14fb824fd7b8b618266093b9", "version-semver": "1.9.0", From e6e641f85571ef481f9bc8d14d9b2bea80e7b061 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Mon, 19 Apr 2021 16:46:40 -0700 Subject: [PATCH 07/20] [chromaprint] Support implementations where lrintf is an intrinsic. --- ports/chromaprint/fix_lrintf_detection.patch | 23 ++++++++++++++++++++ ports/chromaprint/portfile.cmake | 2 ++ ports/chromaprint/vcpkg.json | 3 ++- versions/baseline.json | 2 +- versions/c-/chromaprint.json | 5 +++++ 5 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 ports/chromaprint/fix_lrintf_detection.patch diff --git a/ports/chromaprint/fix_lrintf_detection.patch b/ports/chromaprint/fix_lrintf_detection.patch new file mode 100644 index 00000000000000..27248f0c802236 --- /dev/null +++ b/ports/chromaprint/fix_lrintf_detection.patch @@ -0,0 +1,23 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 3a0eecb..4fa5e8d 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -17,6 +17,7 @@ set_property(CACHE FFT_LIB PROPERTY STRINGS avfft fftw3 fftw3f kissfft vdsp) + + include(CMakePushCheckState) + include(CheckFunctionExists) ++include(CheckSymbolExists) + include(CheckCXXCompilerFlag) + + find_package(Threads) +@@ -37,8 +38,8 @@ endif() + + cmake_push_check_state(RESET) + set(CMAKE_REQUIRED_LIBRARIES -lm) +-check_function_exists(lrintf HAVE_LRINTF) +-check_function_exists(round HAVE_ROUND) ++check_symbol_exists(lrintf math.h HAVE_LRINTF) ++check_symbol_exists(round math.h HAVE_ROUND) + cmake_pop_check_state() + + add_definitions( diff --git a/ports/chromaprint/portfile.cmake b/ports/chromaprint/portfile.cmake index da430faa5b67f3..286edaeb38ae4c 100644 --- a/ports/chromaprint/portfile.cmake +++ b/ports/chromaprint/portfile.cmake @@ -3,6 +3,8 @@ vcpkg_from_github( REPO acoustid/chromaprint REF v1.5.0 SHA512 333114949928abdf5d4b11aba1db6ec487eebe526324c68d903b3fa80a3af87a28d942af765a2f873e63a1bf222b658b6438cd10cde4446f61b26ea91f537469 + PATCHES + fix_lrintf_detection.patch # submitted upstream as https://github.com/acoustid/chromaprint/pull/85 ) vcpkg_configure_cmake( diff --git a/ports/chromaprint/vcpkg.json b/ports/chromaprint/vcpkg.json index 4808008c97af67..271532380431c5 100644 --- a/ports/chromaprint/vcpkg.json +++ b/ports/chromaprint/vcpkg.json @@ -1,6 +1,7 @@ { "name": "chromaprint", - "version-string": "1.5.0", + "version": "1.5.0", + "port-version": 1, "description": "C library for generating audio fingerprints used by AcoustID", "homepage": "https://github.com/acoustid/chromaprint", "license": "LGPL-2.1-or-later", diff --git a/versions/baseline.json b/versions/baseline.json index 1093bb29caafd5..1fbe890352a954 100644 --- a/versions/baseline.json +++ b/versions/baseline.json @@ -1206,7 +1206,7 @@ }, "chromaprint": { "baseline": "1.5.0", - "port-version": 0 + "port-version": 1 }, "chromium-base": { "baseline": "86.0.4199.1", diff --git a/versions/c-/chromaprint.json b/versions/c-/chromaprint.json index 5fc772975e891d..a97cedf9ced3e9 100644 --- a/versions/c-/chromaprint.json +++ b/versions/c-/chromaprint.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "d841d451ed68f08043a27b2ec0beb24d3a276c81", + "version": "1.5.0", + "port-version": 1 + }, { "git-tree": "1f3fe321540c3d87ad63fe85efa5b48bd9c55ac8", "version-string": "1.5.0", From a3fe1a5d0d5ebf0ced0e3eb57d36c8c57c34b982 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Mon, 19 Apr 2021 19:06:40 -0700 Subject: [PATCH 08/20] Add provision-entire-image script. --- .../windows/provision-entire-image.ps1 | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 scripts/azure-pipelines/windows/provision-entire-image.ps1 diff --git a/scripts/azure-pipelines/windows/provision-entire-image.ps1 b/scripts/azure-pipelines/windows/provision-entire-image.ps1 new file mode 100644 index 00000000000000..3bfb5dbd9e2a69 --- /dev/null +++ b/scripts/azure-pipelines/windows/provision-entire-image.ps1 @@ -0,0 +1,16 @@ +# This script runs all the scripts we run on Azure machines to deploy prerequisites, +# and assumes it is being run as an admin user. + +. "$PSScriptRoot\utility-prefix.ps1" + +. "$PSScriptRoot\deploy-visual-studio.ps1" +. "$PSScriptRoot\deploy-windows-wdk.ps1" +. "$PSScriptRoot\deploy-mpi.ps1" +. "$PSScriptRoot\deploy-cuda.ps1" +. "$PSScriptRoot\deploy-pwsh.ps1" +try { + Copy-Item "$PSScriptRoot\deploy-settings.txt" "$PSScriptRoot\deploy-settings.ps1" + . "$PSScriptRoot\deploy-settings.ps1" +} finally { + Remove-Item "$PSScriptRoot\deploy-settings.ps1" +} From 379c232e3856d8721f2a39e28e0df239c4e46c91 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Mon, 19 Apr 2021 23:15:43 -0700 Subject: [PATCH 09/20] [cudnn] Disable download-on-the-fly due to licensing concerns. --- ports/cudnn/CONTROL | 2 +- ports/cudnn/portfile.cmake | 73 +------ .../azure-pipelines/windows/create-vmss.ps1 | 188 +++++++++++------- .../azure-pipelines/windows/deploy-cuda.ps1 | 79 ++++---- versions/baseline.json | 2 +- versions/c-/cudnn.json | 5 + 6 files changed, 166 insertions(+), 183 deletions(-) diff --git a/ports/cudnn/CONTROL b/ports/cudnn/CONTROL index ebb8143495e477..db910a7bcc68c4 100644 --- a/ports/cudnn/CONTROL +++ b/ports/cudnn/CONTROL @@ -1,6 +1,6 @@ Source: cudnn Version: 7.6.5 -Port-Version: 1 +Port-Version: 2 Description: NVIDIA's cuDNN deep neural network acceleration library Build-Depends: cuda Supports: (windows|linux)&x64 diff --git a/ports/cudnn/portfile.cmake b/ports/cudnn/portfile.cmake index 1c1f0641e560aa..6a008313085ff7 100644 --- a/ports/cudnn/portfile.cmake +++ b/ports/cudnn/portfile.cmake @@ -7,7 +7,6 @@ if(NOT VCPKG_TARGET_ARCHITECTURE STREQUAL "x64") endif() vcpkg_check_linkage(ONLY_DYNAMIC_LIBRARY ONLY_DYNAMIC_CRT) -set(VCPKG_POLICY_MISMATCHED_NUMBER_OF_BINARIES enabled) # only release bits are provided set(MINIMUM_CUDNN_VERSION "7.6.5") @@ -64,74 +63,10 @@ if(CUDNN_HEADER_CONTENTS) endif() if (CUDNN_INCLUDE_DIR AND CUDNN_LIBRARY AND _CUDNN_VERSION VERSION_GREATER_EQUAL MINIMUM_CUDNN_VERSION) - set(CUDNN_FOUND TRUE) -else() - set(CUDNN_FOUND FALSE) -endif() - -# Download CUDNN if not found -if (CUDNN_FOUND) message(STATUS "Found CUDNN ${_CUDNN_VERSION} located on system: (include ${CUDNN_INCLUDE_DIR} lib: ${CUDNN_LIBRARY})") set(VCPKG_POLICY_EMPTY_PACKAGE enabled) -elseif(CUDA_VERSION VERSION_EQUAL "10.1" OR CUDA_VERSION VERSION_EQUAL "10.2") - message(STATUS "CUDNN not found on system - downloading a version compatible with your CUDA v${CUDA_VERSION}...") - if(${CUDA_VERSION} VERSION_EQUAL "10.1") - set(CUDNN_VERSION "7.6.5") - set(CUDNN_VERSION_MAJOR "7") - set(CUDNN_FULL_VERSION "7.6.5-cuda10.1_0") - if(VCPKG_TARGET_IS_WINDOWS) - set(CUDNN_DOWNLOAD_LINK "https://anaconda.org/anaconda/cudnn/${CUDNN_VERSION}/download/win-64/cudnn-${CUDNN_FULL_VERSION}.tar.bz2") - set(SHA512_CUDNN "0fb18dd49de877ad6bae24b53ffe007a99915cc9601697a556897e018cc6d99d3aa68716ea99248cf6a9dfaeeb1a551453c606d04e8bbb3e9315bf768184f15b") - set(CUDNN_OS "windows") - elseif(VCPKG_TARGET_IS_LINUX) - set(CUDNN_DOWNLOAD_LINK "https://anaconda.org/anaconda/cudnn/${CUDNN_VERSION}/download/linux-64/cudnn-${CUDNN_FULL_VERSION}.tar.bz2") - set(SHA512_CUDNN "4d901d96ee8f37e3977240b9e6e6eeecb33848388db953a789be47de8f357d815c3a106ceab04297c4df0d8ed9c2795b2a22304e93cd1e53322307d3f3cd668e") - set(CUDNN_OS "linux") - endif() - elseif(${CUDA_VERSION} VERSION_EQUAL "10.2") - set(CUDNN_VERSION "7.6.5") - set(CUDNN_VERSION_MAJOR "7") - set(CUDNN_FULL_VERSION "7.6.5-cuda10.2_0") - if(VCPKG_TARGET_IS_WINDOWS) - set(CUDNN_DOWNLOAD_LINK "https://anaconda.org/anaconda/cudnn/${CUDNN_VERSION}/download/win-64/cudnn-${CUDNN_FULL_VERSION}.tar.bz2") - set(SHA512_CUDNN "86ca2f5f510d4fbeb37548d0bcab42474a1c4041be2cf96c4964f1e51c3641dc4bf25e8434cd5ff99fac9f53946e5f0e83bd845613144731d136cd60913d4aaa") - set(CUDNN_OS "windows") - elseif(VCPKG_TARGET_IS_LINUX) - set(CUDNN_DOWNLOAD_LINK "https://anaconda.org/anaconda/cudnn/${CUDNN_VERSION}/download/linux-64/cudnn-${CUDNN_FULL_VERSION}.tar.bz2") - set(SHA512_CUDNN "b15b554c2ec004105cec8ee2a99f33fab0f9aed12128522730be4fa6204a5b2dff29e42901b5c4840b5ebf35607e8a54f35eb30525885067165b05dd95aa391b") - set(CUDNN_OS "linux") - endif() - endif() - - vcpkg_download_distfile(ARCHIVE - URLS ${CUDNN_DOWNLOAD_LINK} - FILENAME "cudnn-${CUDNN_FULL_VERSION}-${CUDNN_OS}.tar.bz2" - SHA512 ${SHA512_CUDNN} - ) - - vcpkg_extract_source_archive_ex( - OUT_SOURCE_PATH SOURCE_PATH - ARCHIVE ${ARCHIVE} - NO_REMOVE_ONE_LEVEL - ) - - if(VCPKG_TARGET_IS_WINDOWS) - file(INSTALL "${SOURCE_PATH}/Library/include/cudnn.h" DESTINATION ${CURRENT_PACKAGES_DIR}/include) - file(INSTALL "${SOURCE_PATH}/Library/lib/x64/cudnn.lib" DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - file(INSTALL "${SOURCE_PATH}/Library/bin/cudnn64_${CUDNN_VERSION_MAJOR}.dll" DESTINATION ${CURRENT_PACKAGES_DIR}/bin) - set(LICENSE_PATH "${SOURCE_PATH}/info/LICENSE.txt") - elseif(VCPKG_TARGET_IS_LINUX) - file(INSTALL "${SOURCE_PATH}/include/cudnn.h" DESTINATION ${CURRENT_PACKAGES_DIR}/include) - file(INSTALL "${SOURCE_PATH}/lib/libcudnn.so.${CUDNN_VERSION}" DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - file(INSTALL "${SOURCE_PATH}/lib/libcudnn.so.${CUDNN_VERSION_MAJOR}" DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - file(INSTALL "${SOURCE_PATH}/lib/libcudnn.so" DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - set(LICENSE_PATH "${SOURCE_PATH}/info/licenses/NVIDIA_SLA_cuDNN_Support.txt") - endif() - - file(INSTALL "${LICENSE_PATH}" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright) - file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/FindCUDNN.cmake" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT}) - file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT}) - -else() # CUDNN NOT FOUND AND NOT AUTO-DOWNLOADABLE - message(FATAL_ERROR "Please install manually cuDNN for your CUDA v${CUDA_VERSION}") +elseif(VCPKG_TARGET_IS_WINDOWS) + message(FATAL_ERROR "Please download CUDNN from official sources (such as https://developer.nvidia.com/rdp/cudnn-download ) and extract the zip into your CUDA_TOOLKIT_ROOT (${CUDA_TOOLKIT_ROOT}). (For example: tar.exe -xvf cudnn-11.2-windows-x64-v8.1.1.33.zip --strip 1 --directory \"${CUDA_TOOLKIT_ROOT}\"") +else() + message(FATAL_ERROR "Please install CUDNN using your system package manager (the same way you installed CUDA). For example: apt install libcudnn8-dev.") endif() diff --git a/scripts/azure-pipelines/windows/create-vmss.ps1 b/scripts/azure-pipelines/windows/create-vmss.ps1 index 7e2e42761afd02..fddba0eec73af4 100644 --- a/scripts/azure-pipelines/windows/create-vmss.ps1 +++ b/scripts/azure-pipelines/windows/create-vmss.ps1 @@ -21,11 +21,18 @@ the compiler rather than for testing vcpkg. Differences: * The machine prefix is changed to VcpkgUnstable instead of PrWin. * No storage account or "archives" share is provisioned. * The firewall is not opened to allow communication with Azure Storage. + +.PARAMETER CudnnPath +The path to a CUDNN zip file downloaded from NVidia official sources +(e.g. https://developer.nvidia.com/compute/machine-learning/cudnn/secure/8.1.1.33/11.2_20210301/cudnn-11.2-windows-x64-v8.1.1.33.zip +downloaded in a browser with an NVidia account logged in.) #> [CmdLetBinding()] Param( - [switch]$Unstable = $false + [switch]$Unstable = $false, + [parameter(Mandatory=$true)] + [string]$CudnnPath ) $Location = 'westus2' @@ -44,14 +51,15 @@ $ErrorActionPreference = 'Stop' $ProgressActivity = 'Creating Scale Set' $TotalProgress = 18 -if ($Unstable) { - $TotalProgress -= 2 # skipping 2 archives share parts -} - $CurrentProgress = 1 Import-Module "$PSScriptRoot/../create-vmss-helpers.psm1" -DisableNameChecking +if (-Not $CudnnPath.EndsWith('.zip')) { + Write-Error 'Expected CudnnPath to be a zip file.' + return +} + #################################################################################################### Write-Progress ` -Activity $ProgressActivity ` @@ -155,57 +163,84 @@ $VirtualNetwork = New-AzVirtualNetwork ` -Subnet $Subnet #################################################################################################### -if (-Not $Unstable) { - Write-Progress ` - -Activity $ProgressActivity ` - -Status 'Creating archives storage account' ` - -PercentComplete (100 / $TotalProgress * $CurrentProgress++) +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Creating storage account' ` + -CurrentOperation 'Initial setup' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) - $StorageAccountName = Sanitize-Name $ResourceGroupName +$StorageAccountName = Sanitize-Name $ResourceGroupName - New-AzStorageAccount ` - -ResourceGroupName $ResourceGroupName ` - -Location $Location ` - -Name $StorageAccountName ` - -SkuName 'Standard_LRS' ` - -Kind StorageV2 +New-AzStorageAccount ` + -ResourceGroupName $ResourceGroupName ` + -Location $Location ` + -Name $StorageAccountName ` + -SkuName 'Standard_LRS' ` + -Kind StorageV2 - $StorageAccountKeys = Get-AzStorageAccountKey ` - -ResourceGroupName $ResourceGroupName ` - -Name $StorageAccountName +$StorageAccountKeys = Get-AzStorageAccountKey ` + -ResourceGroupName $ResourceGroupName ` + -Name $StorageAccountName - $StorageAccountKey = $StorageAccountKeys[0].Value +$StorageAccountKey = $StorageAccountKeys[0].Value - $StorageContext = New-AzStorageContext ` - -StorageAccountName $StorageAccountName ` - -StorageAccountKey $StorageAccountKey +$StorageContext = New-AzStorageContext ` + -StorageAccountName $StorageAccountName ` + -StorageAccountKey $StorageAccountKey - New-AzStorageContainer -Name archives -Context $StorageContext -Permission Off - $StartTime = [DateTime]::Now - $ExpiryTime = $StartTime.AddMonths(6) +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Creating storage account' ` + -CurrentOperation 'Uploading cudnn.zip' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress) # note no ++ - $SasToken = New-AzStorageAccountSASToken ` - -Service Blob ` - -Permission "racwdlup" ` - -Context $StorageContext ` - -StartTime $StartTime ` - -ExpiryTime $ExpiryTime ` - -ResourceType Service,Container,Object ` - -Protocol HttpsOnly +New-AzStorageContainer -Name setup -Context $storageContext -Permission blob - $SasToken = $SasToken.Substring(1) # strip leading ? +Set-AzStorageBlobContent -File $CudnnPath ` + -Container 'setup' ` + -Blob 'cudnn.zip' ` + -Context $StorageContext - # Note that we put the storage account into the firewall after creating the above SAS token or we - # would be denied since the person running this script isn't one of the VMs we're creating here. - Set-AzStorageAccount ` - -ResourceGroupName $ResourceGroupName ` - -AccountName $StorageAccountName ` - -NetworkRuleSet ( ` - @{bypass="AzureServices"; ` - virtualNetworkRules=( ` - @{VirtualNetworkResourceId=$VirtualNetwork.Subnets[0].Id;Action="allow"}); ` - defaultAction="Deny"}) -} +$CudnnBlobUrl = "https://$StorageAccountName.blob.core.windows.net/setup/cudnn.zip" + +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Creating storage account' ` + -CurrentOperation 'Creating archives container' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress) # note no ++ + +New-AzStorageContainer -Name archives -Context $StorageContext -Permission Off + +$StartTime = [DateTime]::Now +$ExpiryTime = $StartTime.AddMonths(6) + +$SasToken = New-AzStorageAccountSASToken ` + -Service Blob ` + -Permission "racwdlup" ` + -Context $StorageContext ` + -StartTime $StartTime ` + -ExpiryTime $ExpiryTime ` + -ResourceType Service,Container,Object ` + -Protocol HttpsOnly + +$SasToken = $SasToken.Substring(1) # strip leading ? + +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Creating storage account' ` + -CurrentOperation 'Locking down network' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress) # note no ++ + +# Note that we put the storage account into the firewall after creating the above SAS token or we +# would be denied since the person running this script isn't one of the VMs we're creating here. +Set-AzStorageAccount ` + -ResourceGroupName $ResourceGroupName ` + -AccountName $StorageAccountName ` + -NetworkRuleSet ( ` + @{bypass="AzureServices"; ` + virtualNetworkRules=( ` + @{VirtualNetworkResourceId=$VirtualNetwork.Subnets[0].Id;Action="allow"}); ` + defaultAction="Deny"}) #################################################################################################### Write-Progress ` @@ -260,7 +295,8 @@ Write-Host "deploy-psexec.ps1 output: $($DeployPsExecResult.value.Message)" function Invoke-ScriptWithPrefix { param( [string]$ScriptName, - [switch]$AddAdminPw + [switch]$AddAdminPw, + [switch]$AddCudnnUrl ) Write-Progress ` @@ -278,6 +314,10 @@ function Invoke-ScriptWithPrefix { $script = $script.Replace('# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1', $DropToAdminUserPrefix) } + if ($AddCudnnUrl) { + $script = $script.Replace('# REPLACE WITH $CudnnUrl', "`$CudnnUrl = '$CudnnBlobUrl'") + } + $script = $script.Replace('# REPLACE WITH UTILITY-PREFIX.ps1', $UtilityPrefixContent); Set-Content -Path $tempScriptFilename -Value $script -Encoding utf8NoBOM @@ -311,7 +351,7 @@ Invoke-ScriptWithPrefix -ScriptName 'deploy-mpi.ps1' -AddAdminPw Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName #################################################################################################### -Invoke-ScriptWithPrefix -ScriptName 'deploy-cuda.ps1' -AddAdminPw +Invoke-ScriptWithPrefix -ScriptName 'deploy-cuda.ps1' -AddAdminPw -AddCudnnUrl Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName #################################################################################################### @@ -332,37 +372,35 @@ $ProvisionImageResult = Invoke-AzVMRunCommand ` Write-Host "deploy-settings.txt output: $($ProvisionImageResult.value.Message)" -if (-Not $Unstable) { #################################################################################################### - Write-Progress ` - -Activity $ProgressActivity ` - -Status 'Deploying SAS token into VM' ` - -PercentComplete (100 / $TotalProgress * $CurrentProgress++) +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Deploying SAS token into VM' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) - $tempScriptFilename = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".txt" - try { - $script = "Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' " ` - + "-Name PROVISIONED_AZURE_STORAGE_NAME " ` - + "-Value '$StorageAccountName'`r`n" ` - + "Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' " ` - + "-Name PROVISIONED_AZURE_STORAGE_SAS_TOKEN " ` - + "-Value '$SasToken'`r`n" +$tempScriptFilename = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".txt" +try { + $script = "Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' " ` + + "-Name PROVISIONED_AZURE_STORAGE_NAME " ` + + "-Value '$StorageAccountName'`r`n" ` + + "Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' " ` + + "-Name PROVISIONED_AZURE_STORAGE_SAS_TOKEN " ` + + "-Value '$SasToken'`r`n" - Write-Host "Script content is:" - Write-Host $script + Write-Host "Script content is:" + Write-Host $script - Set-Content -Path $tempScriptFilename -Value $script -Encoding utf8NoBOM - $InvokeResult = Invoke-AzVMRunCommand ` - -ResourceGroupName $ResourceGroupName ` - -VMName $ProtoVMName ` - -CommandId 'RunPowerShellScript' ` - -ScriptPath $tempScriptFilename + Set-Content -Path $tempScriptFilename -Value $script -Encoding utf8NoBOM + $InvokeResult = Invoke-AzVMRunCommand ` + -ResourceGroupName $ResourceGroupName ` + -VMName $ProtoVMName ` + -CommandId 'RunPowerShellScript' ` + -ScriptPath $tempScriptFilename - Write-Host "Deploy SAS token output: $($InvokeResult.value.Message)" - } finally { - Remove-Item $tempScriptFilename -Force - } -} # -Not $Unstable + Write-Host "Deploy SAS token output: $($InvokeResult.value.Message)" +} finally { + Remove-Item $tempScriptFilename -Force +} Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName diff --git a/scripts/azure-pipelines/windows/deploy-cuda.ps1 b/scripts/azure-pipelines/windows/deploy-cuda.ps1 index 6f9dd55ffc380b..6f63c574a6d865 100644 --- a/scripts/azure-pipelines/windows/deploy-cuda.ps1 +++ b/scripts/azure-pipelines/windows/deploy-cuda.ps1 @@ -5,6 +5,10 @@ # REPLACE WITH UTILITY-PREFIX.ps1 +# REPLACE WITH $CudnnUrl + +$CudnnLocalZipPath = "$PSScriptRoot\cudnn-11.2-windows-x64-v8.1.1.33.zip" + $CudaUrl = 'https://developer.download.nvidia.com/compute/cuda/11.3.0/network_installers/cuda_11.3.0_win10_network.exe' $CudaFeatures = 'nvcc_11.3 cuobjdump_11.3 nvprune_11.3 cupti_11.3 memcheck_11.3 nvdisasm_11.3 nvprof_11.3 ' + ` @@ -13,45 +17,46 @@ $CudaFeatures = 'nvcc_11.3 cuobjdump_11.3 nvprune_11.3 cupti_11.3 memcheck_11.3 'cusparse_11.3 cusparse_dev_11.3 npp_11.3 npp_dev_11.3 nvrtc_11.3 nvrtc_dev_11.3 nvml_dev_11.3 ' + ` 'occupancy_calculator_11.3 ' -<# -.SYNOPSIS -Installs NVIDIA's CUDA Toolkit. - -.DESCRIPTION -InstallCuda installs the CUDA Toolkit with the features specified as a -space-separated list of strings in $Features. - -.PARAMETER Url -The URL of the CUDA installer. - -.PARAMETER Features -A space-separated list of features to install. -#> -Function InstallCuda { - Param( - [String]$Url, - [String]$Features - ) - - try { - Write-Host 'Downloading CUDA...' - [string]$installerPath = Get-TempFilePath -Extension 'exe' - curl.exe -L -o $installerPath -s -S $Url - Write-Host 'Installing CUDA...' - $proc = Start-Process -FilePath $installerPath -ArgumentList @('-s ' + $Features) -Wait -PassThru - $exitCode = $proc.ExitCode - if ($exitCode -eq 0) { - Write-Host 'Installation successful!' - } - else { - Write-Error "Installation failed! Exited with $exitCode." - throw - } +$destination = "$env:ProgramFiles\NVIDIA GPU Computing Toolkit\CUDA\v11.3" + +try { + Write-Host 'Downloading CUDA...' + [string]$installerPath = Get-TempFilePath -Extension 'exe' + curl.exe -L -o $installerPath -s -S $CudaUrl + Write-Host 'Installing CUDA...' + $proc = Start-Process -FilePath $installerPath -ArgumentList @('-s ' + $CudaFeatures) -Wait -PassThru + $exitCode = $proc.ExitCode + if ($exitCode -eq 0) { + Write-Host 'Installation successful!' } - catch { - Write-Error "Failed to install CUDA! $($_.Exception.Message)" + else { + Write-Error "Installation failed! Exited with $exitCode." throw } } +catch { + Write-Error "Failed to install CUDA! $($_.Exception.Message)" + throw +} -InstallCuda -Url $CudaUrl -Features $CudaFeatures +try { + if ([string]::IsNullOrWhiteSpace($CudnnUrl)) { + if (-Not (Test-Path $CudnnLocalZipPath)) { + throw "CUDNN zip ($CudnnLocalZipPath) was missing, please download from NVidia and place next to this script." + } + + $cudnnZipPath = $CudnnLocalZipPath + } else { + Write-Host 'Downloading CUDNN...' + $cudnnZipPath = Get-TempFilePath -Extension 'zip' + curl.exe -L -o $cudnnZipPath -s -S $CudnnUrl + } + + Write-Host "Installing CUDNN to $destination..." + tar.exe -xvf "$cudnnZipPath" --strip 1 --directory "$destination" + Write-Host 'Installation successful!' +} +catch { + Write-Error "Failed to install CUDNN! $($_.Exception.Message)" + throw +} diff --git a/versions/baseline.json b/versions/baseline.json index 1fbe890352a954..d5ea8f638cd120 100644 --- a/versions/baseline.json +++ b/versions/baseline.json @@ -1526,7 +1526,7 @@ }, "cudnn": { "baseline": "7.6.5", - "port-version": 1 + "port-version": 2 }, "cunit": { "baseline": "2.1.3-6", diff --git a/versions/c-/cudnn.json b/versions/c-/cudnn.json index 4930853d40409d..cc7e05d0ce712b 100644 --- a/versions/c-/cudnn.json +++ b/versions/c-/cudnn.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "5581791a9ad35390ed30629985b3863cf8b85bba", + "version-string": "7.6.5", + "port-version": 2 + }, { "git-tree": "495cc586b4bfdb28c3c339d3f471b9fbfc8bc481", "version-string": "7.6.5", From 09b03c94ebcac9800ac712ec98b06f720b05cb56 Mon Sep 17 00:00:00 2001 From: Billy Robert ONeal III Date: Wed, 21 Apr 2021 18:12:59 -0700 Subject: [PATCH 10/20] Add libnccl to Linux VMs. --- scripts/azure-pipelines/linux/provision-image.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh index 37487310b05bef..6663baed838808 100755 --- a/scripts/azure-pipelines/linux/provision-image.sh +++ b/scripts/azure-pipelines/linux/provision-image.sh @@ -63,7 +63,8 @@ sudo mv cuda-ubuntu1804.pin /etc/apt/preferences.d/cuda-repository-pin-600 sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /" sudo apt -y update -sudo apt install -y --no-install-recommends cuda-compiler-11-3 cuda-libraries-dev-11-3 cuda-driver-dev-11-3 cuda-cudart-dev-11-3 libcublas-11-3 libcurand-dev-11-3 libcudnn8-dev +sudo apt install -y --no-install-recommends cuda-compiler-11-3 cuda-libraries-dev-11-3 cuda-driver-dev-11-3 \ + cuda-cudart-dev-11-3 libcublas-11-3 libcurand-dev-11-3 libcudnn8-dev libnccl2 libnccl-dev # Install PowerShell wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb From f05d049b83583a5a1bfe221782f1d32af5a86242 Mon Sep 17 00:00:00 2001 From: Billy Robert ONeal III Date: Thu, 22 Apr 2021 14:27:10 -0700 Subject: [PATCH 11/20] [wangle] Disable x64-windows due to an ICE. --- scripts/ci.baseline.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index 01b33f2d1e66c9..3425aec993f58f 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -1784,7 +1784,6 @@ clapack:x64-uwp=skip lapack-reference:arm64-windows=skip lapack-reference:arm-uwp=skip - # failures for x64-windows-static-md ace:x64-windows-static-md=fail activemq-cpp:x64-windows-static-md=fail @@ -1819,3 +1818,9 @@ zyre:x64-windows-static-md=fail usbmuxd:x64-windows-static-md=fail workflow:x64-uwp=fail workflow:arm-uwp=fail + +# wangle triggers an internal compiler error +# https://devdiv.visualstudio.com/DefaultCollection/DevDiv/_workitems/edit/1269468 +wangle:x64-windows=fail +wangle:x64-windows-static=fail +wangle:x64-windows-static-md=fail From b0e4f60a216fe334c64027e35bb51e1f0a092fb9 Mon Sep 17 00:00:00 2001 From: Billy Robert ONeal III Date: Thu, 22 Apr 2021 15:41:18 -0700 Subject: [PATCH 12/20] [cmake] Update cmake to 3.20.1 to avoid https://gitlab.kitware.com/cmake/cmake/-/issues/21571 race --- scripts/vcpkgTools.xml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/scripts/vcpkgTools.xml b/scripts/vcpkgTools.xml index c4c4b14b123de5..3d48d52948764e 100644 --- a/scripts/vcpkgTools.xml +++ b/scripts/vcpkgTools.xml @@ -8,25 +8,25 @@ python-3.9.2-embed-win32.zip - 3.19.2 - cmake-3.19.2-win32-x86\bin\cmake.exe - https://github.com/Kitware/CMake/releases/download/v3.19.2/cmake-3.19.2-win32-x86.zip - 1ed3706d8dd5c5244ceb4c22443f006fcb9ffa76cb6d9ebdbbb8f11a50deb3f92a9a4080ff3fd125f79c0143f553ea0c02a6a2d27521438705c2fea4dd97d239 - cmake-3.19.2-win32-x86.zip + 3.20.1 + cmake-3.20.1-windows-i386\bin\cmake.exe + https://github.com/Kitware/CMake/releases/download/v3.20.1/cmake-3.20.1-windows-i386.zip + bc0be4e32bbffe102b2cb970591118bf2132d6afd6c97469cf46a4f772627fa81a8a57e4069dc32a93c0ffcce176e562d9d008e6780f053876e5d5eb97051554 + cmake-3.20.1-windows-i386.zip - 3.19.2 - cmake-3.19.2-macos-universal/CMake.app/Contents/bin/cmake - https://github.com/Kitware/CMake/releases/download/v3.19.2/cmake-3.19.2-macos-universal.tar.gz - 031196439a26d6ff2a1f42830ca26b5159752dc2c2c889a4e304327c858157b68088905b10727090918c3605fe7fe429aa19d46e68ed30c17d4d89b3d209aae8 - cmake-3.19.2-macos-universal.tar.gz + 3.20.1 + cmake-3.20.1-macos-universal/CMake.app/Contents/bin/cmake + https://github.com/Kitware/CMake/releases/download/v3.20.1/cmake-3.20.1-macos-universal.tar.gz + fd17d09a0639833a2aa5f30045cfcbbb30e0cfa5092d2f22f0bc502ea361a458d26be96437fbfb6b5e68b73954050c0256653af7e2e20bb253a878e206992135 + cmake-3.20.1-macos-universal.tar.gz - 3.19.2 - cmake-3.19.2-Linux-x86_64/bin/cmake - https://github.com/Kitware/CMake/releases/download/v3.19.2/cmake-3.19.2-Linux-x86_64.tar.gz - 67ea7ee428e6c93039646d6d0cbe84c0be337a2b878487635fee71ec2c8992a9078a660e6b793ef11ce13530def44a16b716af96e10a97675d014a26caa0aed4 - cmake-3.19.2-Linux-x86_64.tar.gz + 3.20.1 + cmake-3.20.1-linux-x86_64/bin/cmake + https://github.com/Kitware/CMake/releases/download/v3.20.1/cmake-3.20.1-Linux-x86_64.tar.gz + e14b6cbc03abfbae074c36e34f90eea94af02e727e75ed597f748203e2ce7483ffe693bd173cb02a1de7ac55254a891da5505c43b92528adc918eff9e8b024e6 + cmake-3.20.1-linux-x86_64.tar.gz 3.12.4 From c0ed618f8f1f66667ae6d38a8d3fc5e922b7d619 Mon Sep 17 00:00:00 2001 From: Billy Robert ONeal III Date: Fri, 23 Apr 2021 09:50:49 -0700 Subject: [PATCH 13/20] [libudis86] Fix passing a bogus working directory which fails on CMake 3.20.x --- ports/libudis86/CONTROL | 4 ---- ports/libudis86/portfile.cmake | 10 +++++----- ports/libudis86/vcpkg.json | 7 +++++++ versions/baseline.json | 2 +- versions/l-/libudis86.json | 5 +++++ 5 files changed, 18 insertions(+), 10 deletions(-) delete mode 100644 ports/libudis86/CONTROL create mode 100644 ports/libudis86/vcpkg.json diff --git a/ports/libudis86/CONTROL b/ports/libudis86/CONTROL deleted file mode 100644 index 8e67b89cc4261d..00000000000000 --- a/ports/libudis86/CONTROL +++ /dev/null @@ -1,4 +0,0 @@ -Source: libudis86 -Version: 2018-01-28-56ff6c87 -Homepage: https://github.com/vmt/udis86 -Description: Disassembler Library for x86 and x86-64 diff --git a/ports/libudis86/portfile.cmake b/ports/libudis86/portfile.cmake index a1ae6e539c1e50..42d99d38a1e976 100644 --- a/ports/libudis86/portfile.cmake +++ b/ports/libudis86/portfile.cmake @@ -11,8 +11,8 @@ file(COPY ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt DESTINATION ${SOURCE_PATH}) vcpkg_find_acquire_program(PYTHON2) vcpkg_execute_required_process( - COMMAND ${PYTHON2} ${SOURCE_PATH}/scripts/ud_itab.py ${SOURCE_PATH}/docs/x86/optable.xml ${SOURCE_PATH}/libudis86/ - WORKING_DIRECTORY SOURCE_PATH + COMMAND "${PYTHON2}" "${SOURCE_PATH}/scripts/ud_itab.py" "${SOURCE_PATH}/docs/x86/optable.xml" "${SOURCE_PATH}/libudis86/" + WORKING_DIRECTORY "${SOURCE_PATH}" LOGNAME python-${TARGET_TRIPLET}-generate-sources ) @@ -20,12 +20,12 @@ vcpkg_configure_cmake( SOURCE_PATH ${SOURCE_PATH} PREFER_NINJA OPTIONS_DEBUG - -DDISABLE_INSTALL_HEADERS=ON - -DDISABLE_INSTALL_TOOLS=ON + -DDISABLE_INSTALL_HEADERS=ON + -DDISABLE_INSTALL_TOOLS=ON ) vcpkg_install_cmake() vcpkg_copy_pdbs() vcpkg_copy_tool_dependencies(${CURRENT_PACKAGES_DIR}/tools/libudis86) -file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/libudis86 RENAME copyright) +file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright) diff --git a/ports/libudis86/vcpkg.json b/ports/libudis86/vcpkg.json new file mode 100644 index 00000000000000..a687a41045a274 --- /dev/null +++ b/ports/libudis86/vcpkg.json @@ -0,0 +1,7 @@ +{ + "name": "libudis86", + "version-string": "2018-01-28-56ff6c87", + "port-version": 1, + "description": "Disassembler Library for x86 and x86-64", + "homepage": "https://github.com/vmt/udis86" +} diff --git a/versions/baseline.json b/versions/baseline.json index d5ea8f638cd120..4b550178279872 100644 --- a/versions/baseline.json +++ b/versions/baseline.json @@ -3558,7 +3558,7 @@ }, "libudis86": { "baseline": "2018-01-28-56ff6c87", - "port-version": 0 + "port-version": 1 }, "libudns": { "baseline": "0.4-2", diff --git a/versions/l-/libudis86.json b/versions/l-/libudis86.json index 62fefa67deb84f..0e098416d243c0 100644 --- a/versions/l-/libudis86.json +++ b/versions/l-/libudis86.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "e8ab6af2cfbbb52901433f92f0c028bc766ce3ae", + "version-string": "2018-01-28-56ff6c87", + "port-version": 1 + }, { "git-tree": "55fbd94b278b0b3d753a2db345afa5f20b95c350", "version-string": "2018-01-28-56ff6c87", From d89c87b8c47990071eff6ca7da03d36c8a85f741 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Fri, 23 Apr 2021 18:15:31 +0000 Subject: [PATCH 14/20] [dartsim] Disable unit tests, examples, and tutorials, some of which have CMake authoring errors rejected by 3.20.1. --- ...le_unit_tests_examples_and_tutorials.patch | 20 ++++++++++++++ ports/dartsim/portfile.cmake | 26 ++++++++++--------- ports/dartsim/vcpkg.json | 3 ++- versions/baseline.json | 2 +- versions/d-/dartsim.json | 5 ++++ 5 files changed, 42 insertions(+), 14 deletions(-) create mode 100644 ports/dartsim/disable_unit_tests_examples_and_tutorials.patch diff --git a/ports/dartsim/disable_unit_tests_examples_and_tutorials.patch b/ports/dartsim/disable_unit_tests_examples_and_tutorials.patch new file mode 100644 index 00000000000000..08119ce39f457d --- /dev/null +++ b/ports/dartsim/disable_unit_tests_examples_and_tutorials.patch @@ -0,0 +1,20 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index e32147d..5c85186 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -279,6 +279,7 @@ add_subdirectory(dart) + + set(DART_IN_SOURCE_BUILD TRUE) + ++if (0) + if(TARGET dart AND NOT DART_BUILD_DARTPY) + + # Add a "tests" target to build unit tests. +@@ -330,6 +331,7 @@ if(TARGET dart AND NOT DART_BUILD_DARTPY) + endif(DART_VERBOSE) + + endif() ++endif() + + if (DART_BUILD_DARTPY) + add_subdirectory(python) diff --git a/ports/dartsim/portfile.cmake b/ports/dartsim/portfile.cmake index 0af5f8e9aa5c91..c208482f1f44fa 100644 --- a/ports/dartsim/portfile.cmake +++ b/ports/dartsim/portfile.cmake @@ -7,20 +7,23 @@ vcpkg_from_github( REF v6.9.4 SHA512 a9e8712a294286772ee1e3c3899aac7d202f1d3c2b9242ebeaddb2a142787b192d5421a6e3f889dd9ff9ca9e9106b1f893a0c8ab2e1656b04fac6d0be13494ba HEAD_REF release-6.9 - PATCHES 1478.patch - 1497.patch + PATCHES + 1478.patch + 1497.patch + disable_unit_tests_examples_and_tutorials.patch ) vcpkg_configure_cmake( SOURCE_PATH ${SOURCE_PATH} - OPTIONS -DDART_VERBOSE=ON - -DDART_MSVC_DEFAULT_OPTIONS=ON - -DDART_SKIP_DOXYGEN=ON - -DDART_SKIP_FLANN=ON - -DDART_SKIP_IPOPT=ON - -DDART_SKIP_NLOPT=ON - -DDART_SKIP_OPENGL=ON - -DDART_SKIP_pagmo=ON + OPTIONS + -DDART_VERBOSE=ON + -DDART_MSVC_DEFAULT_OPTIONS=ON + -DDART_SKIP_DOXYGEN=ON + -DDART_SKIP_FLANN=ON + -DDART_SKIP_IPOPT=ON + -DDART_SKIP_NLOPT=ON + -DDART_SKIP_OPENGL=ON + -DDART_SKIP_pagmo=ON ) vcpkg_install_cmake() @@ -36,5 +39,4 @@ file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include) file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/share) # Handle copyright -file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/dartsim RENAME copyright) - +file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright) diff --git a/ports/dartsim/vcpkg.json b/ports/dartsim/vcpkg.json index 0029e6c3c968f3..5f4c00b1865905 100644 --- a/ports/dartsim/vcpkg.json +++ b/ports/dartsim/vcpkg.json @@ -1,6 +1,7 @@ { "name": "dartsim", - "version-string": "6.9.4", + "version": "6.9.4", + "port-version": 1, "description": "Dynamic Animation and Robotics Toolkit", "homepage": "https://dartsim.github.io/", "dependencies": [ diff --git a/versions/baseline.json b/versions/baseline.json index 4b550178279872..a8b589972576dc 100644 --- a/versions/baseline.json +++ b/versions/baseline.json @@ -1574,7 +1574,7 @@ }, "dartsim": { "baseline": "6.9.4", - "port-version": 0 + "port-version": 1 }, "dataframe": { "baseline": "1.12.0", diff --git a/versions/d-/dartsim.json b/versions/d-/dartsim.json index 4fd122f2f9fe73..f5b4f03e25affe 100644 --- a/versions/d-/dartsim.json +++ b/versions/d-/dartsim.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "104efce0932ef1a1e1e01358a442cfff637b285e", + "version": "6.9.4", + "port-version": 1 + }, { "git-tree": "e9a9e2df385efc1d41aeea022f88f7e6a07f81ba", "version-string": "6.9.4", From 46ecf9822b0fff1393ff9190d86d76a8365739d6 Mon Sep 17 00:00:00 2001 From: Billy Robert ONeal III Date: Fri, 23 Apr 2021 12:46:15 -0700 Subject: [PATCH 15/20] Add thrust to the cuda installees. --- scripts/azure-pipelines/windows/deploy-cuda.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/azure-pipelines/windows/deploy-cuda.ps1 b/scripts/azure-pipelines/windows/deploy-cuda.ps1 index 6f63c574a6d865..ab1cc1c54bd371 100644 --- a/scripts/azure-pipelines/windows/deploy-cuda.ps1 +++ b/scripts/azure-pipelines/windows/deploy-cuda.ps1 @@ -15,7 +15,7 @@ $CudaFeatures = 'nvcc_11.3 cuobjdump_11.3 nvprune_11.3 cupti_11.3 memcheck_11.3 'visual_studio_integration_11.3 visual_profiler_11.3 visual_profiler_11.3 cublas_11.3 cublas_dev_11.3 ' + ` 'cudart_11.3 cufft_11.3 cufft_dev_11.3 curand_11.3 curand_dev_11.3 cusolver_11.3 cusolver_dev_11.3 ' + ` 'cusparse_11.3 cusparse_dev_11.3 npp_11.3 npp_dev_11.3 nvrtc_11.3 nvrtc_dev_11.3 nvml_dev_11.3 ' + ` - 'occupancy_calculator_11.3 ' + 'occupancy_calculator_11.3 thrust_11.3 ' $destination = "$env:ProgramFiles\NVIDIA GPU Computing Toolkit\CUDA\v11.3" From 33d5f1a8d34420bb11dfd0cdcd5fe4313deaa5fe Mon Sep 17 00:00:00 2001 From: Billy Robert ONeal III Date: Sat, 24 Apr 2021 03:52:43 -0700 Subject: [PATCH 16/20] [tensorflow] Put .bzl in CURRENT_BUILDTREES_DIR to avoid running out of disk space in CI and to respect --clean-after-build. --- ports/tensorflow-common/tensorflow-common.cmake | 4 ++-- ports/tensorflow-common/vcpkg.json | 1 + versions/baseline.json | 2 +- versions/t-/tensorflow-common.json | 5 +++++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ports/tensorflow-common/tensorflow-common.cmake b/ports/tensorflow-common/tensorflow-common.cmake index 18011257e849fb..4746fcda4f9eca 100644 --- a/ports/tensorflow-common/tensorflow-common.cmake +++ b/ports/tensorflow-common/tensorflow-common.cmake @@ -45,7 +45,7 @@ set(ENV{PYTHON_LIB_PATH} "${PYTHON_LIB_PATH}") vcpkg_execute_required_process(COMMAND ${PYTHON3} -c "import numpy" WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-numpy-${TARGET_TRIPLET}) # tensorflow has long file names, which will not work on windows -set(ENV{TEST_TMPDIR} ${BUILDTREES_DIR}/.bzl) +set(ENV{TEST_TMPDIR} "${CURRENT_BUILDTREES_DIR}/.bzl") set(ENV{USE_DEFAULT_PYTHON_LIB_PATH} 1) set(ENV{TF_NEED_KAFKA} 0) @@ -439,4 +439,4 @@ else() endif() endif() -message(STATUS "You may want to delete ${CURRENT_BUILDTREES_DIR} and ${BUILDTREES_DIR}/.bzl to free diskspace.") +message(STATUS "You may want to delete ${CURRENT_BUILDTREES_DIR} to free diskspace.") diff --git a/ports/tensorflow-common/vcpkg.json b/ports/tensorflow-common/vcpkg.json index 7477461b330107..447d665b583b7e 100644 --- a/ports/tensorflow-common/vcpkg.json +++ b/ports/tensorflow-common/vcpkg.json @@ -1,6 +1,7 @@ { "name": "tensorflow-common", "version-semver": "2.4.1", + "port-version": 1, "description": "This meta package holds common files for the C [tensorflow] and the C++ [tensorflow-cc] API version of TensorFlow but is not installable on its own.", "homepage": "https://github.com/tensorflow/tensorflow" } diff --git a/versions/baseline.json b/versions/baseline.json index a8b589972576dc..d574bb6b529b17 100644 --- a/versions/baseline.json +++ b/versions/baseline.json @@ -5922,7 +5922,7 @@ }, "tensorflow-common": { "baseline": "2.4.1", - "port-version": 0 + "port-version": 1 }, "termcolor": { "baseline": "2.0.0", diff --git a/versions/t-/tensorflow-common.json b/versions/t-/tensorflow-common.json index a42b859be96edc..fc8ec6df0901fe 100644 --- a/versions/t-/tensorflow-common.json +++ b/versions/t-/tensorflow-common.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "c2a11af14488e643cf0a07f676d08319a0f70f08", + "version-semver": "2.4.1", + "port-version": 1 + }, { "git-tree": "566f036d2265f6c960014ceb526ed0d143f65c60", "version-semver": "2.4.1", From bb66d8623f9219838dd0bbc5c8af47611787efd6 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Sun, 25 Apr 2021 00:02:30 -0700 Subject: [PATCH 17/20] [dimcli] Skip port broken by changes in VS2019 project system. --- scripts/ci.baseline.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index 3425aec993f58f..9ced1bd96d47db 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -1824,3 +1824,9 @@ workflow:arm-uwp=fail wangle:x64-windows=fail wangle:x64-windows-static=fail wangle:x64-windows-static-md=fail + +# VS2019 version 16.9.4's project system changes where PDBs are placed in a way that breaks the +# upstream build script of this port. +# See https://developercommunity.visualstudio.com/t/Toolset-169-regression-vcxproj-producin/1356639 +dimcli:x64-windows-static-md=fail +dimcli:x64-windows-static=fail From b0b81b185c65650b4de78feb27d7ec4264a9a882 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Sun, 25 Apr 2021 00:09:24 -0700 Subject: [PATCH 18/20] [upb] Disable an additional warning. --- ports/upb/fix-uwp.patch | 2 +- ports/upb/vcpkg.json | 1 + versions/baseline.json | 2 +- versions/u-/upb.json | 5 +++++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ports/upb/fix-uwp.patch b/ports/upb/fix-uwp.patch index cbd62f602098e6..c681fa336a7a88 100644 --- a/ports/upb/fix-uwp.patch +++ b/ports/upb/fix-uwp.patch @@ -7,7 +7,7 @@ index a6b432f..003615e 100644 endif() +if (MSVC) -+ add_compile_options(/wd4146 -D_CRT_SECURE_NO_WARNINGS) ++ add_compile_options(/wd4146 /wd4703 -D_CRT_SECURE_NO_WARNINGS) +endif() + enable_testing() diff --git a/ports/upb/vcpkg.json b/ports/upb/vcpkg.json index 3213d796fc5754..058833064dd99f 100644 --- a/ports/upb/vcpkg.json +++ b/ports/upb/vcpkg.json @@ -1,6 +1,7 @@ { "name": "upb", "version-date": "2020-12-19", + "port-version": 1, "description": "μpb (often written 'upb') is a small protobuf implementation written in C.", "homepage": "https://github.com/protocolbuffers/upb/" } diff --git a/versions/baseline.json b/versions/baseline.json index d574bb6b529b17..0e17318b5b9340 100644 --- a/versions/baseline.json +++ b/versions/baseline.json @@ -6198,7 +6198,7 @@ }, "upb": { "baseline": "2020-12-19", - "port-version": 0 + "port-version": 1 }, "urdfdom": { "baseline": "1.0.4", diff --git a/versions/u-/upb.json b/versions/u-/upb.json index bbe7a8b07d12c7..6d0406e44aee57 100644 --- a/versions/u-/upb.json +++ b/versions/u-/upb.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "df35bcef11c276d5cc7280edeca76eeff3dad223", + "version-date": "2020-12-19", + "port-version": 1 + }, { "git-tree": "8c7a75bb8754caa918eddfd0e6f9457834e81f64", "version-date": "2020-12-19", From 5d3d500bd62274596376302d30ecb83f33470077 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Sun, 25 Apr 2021 00:15:44 -0700 Subject: [PATCH 19/20] [libhv] Fix typo DISABLE_PARALLEL => DISABLE_PARALLEL_CONFIGURE --- ports/libhv/portfile.cmake | 2 +- ports/libhv/vcpkg.json | 2 +- versions/baseline.json | 2 +- versions/l-/libhv.json | 5 +++++ 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ports/libhv/portfile.cmake b/ports/libhv/portfile.cmake index 01c6f83a50d3a9..f54897107b1452 100644 --- a/ports/libhv/portfile.cmake +++ b/ports/libhv/portfile.cmake @@ -14,7 +14,7 @@ string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "dynamic" BUILD_SHARED) vcpkg_configure_cmake( SOURCE_PATH ${SOURCE_PATH} PREFER_NINJA - DISABLE_PARALLEL + DISABLE_PARALLEL_CONFIGURE OPTIONS -DBUILD_EXAMPLES=OFF -DBUILD_UNITTEST=OFF diff --git a/ports/libhv/vcpkg.json b/ports/libhv/vcpkg.json index c6e1a50f73aa27..d16fbeccfb6a52 100644 --- a/ports/libhv/vcpkg.json +++ b/ports/libhv/vcpkg.json @@ -1,7 +1,7 @@ { "name": "libhv", "version": "1.0.0", - "port-version": 1, + "port-version": 2, "description": "Libhv is a C/C++ network library similar to libevent/libuv.", "homepage": "https://github.com/ithewei/libhv", "supports": "!(arm | uwp)" diff --git a/versions/baseline.json b/versions/baseline.json index 0e17318b5b9340..52e92cec17bbe6 100644 --- a/versions/baseline.json +++ b/versions/baseline.json @@ -3146,7 +3146,7 @@ }, "libhv": { "baseline": "1.0.0", - "port-version": 1 + "port-version": 2 }, "libhydrogen": { "baseline": "2019-08-11", diff --git a/versions/l-/libhv.json b/versions/l-/libhv.json index 8d373a2611a846..9fae80665216e1 100644 --- a/versions/l-/libhv.json +++ b/versions/l-/libhv.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "3a460f29d525c6c1ce8154593238960415a24d80", + "version": "1.0.0", + "port-version": 2 + }, { "git-tree": "82104ab053e797fdcaa93d64b11fe05d37e5e14c", "version": "1.0.0", From ff00f6f1e020df994098a6de2bb407885dbb8acf Mon Sep 17 00:00:00 2001 From: Billy Robert ONeal III Date: Fri, 23 Apr 2021 14:55:00 -0700 Subject: [PATCH 20/20] Update pools --- scripts/azure-pipelines/azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index 27db6a93d9a5c4..9f6526cb795ea6 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -2,8 +2,8 @@ # SPDX-License-Identifier: MIT # variables: - windows-pool: 'PrWin-2020-12-17' - linux-pool: 'PrLin-2020-12-16' + windows-pool: 'PrWin-2021-04-23' + linux-pool: 'PrLin-2021-04-25' osx-pool: 'PrOsx-2020-09-28' stages: