From e495cb00db4abb51bd04a5d449eab2f3169c4608 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Tue, 28 Apr 2020 22:16:12 -0700 Subject: [PATCH 1/7] [vcpkg] Onboard Linux to VMSS, open 'git' port, and switch back to Azure Spot * Adds scripts to generate scale sets for testing Linux. * Note workaround for https://github.com/microsoft/azure-pipelines-agent/pull/2929 * Switches Windows validation to 'Spot' VMs. * Opens the git port 9418. * Removes provisioning of the no longer used 'logs' file share. * Changes Azure region to 'westus2', which is cheaper. * Adds +x to all the scripts in scripts/azure-pipelines. * Use 'xml-results' for all platforms instead of 'raw xml results' on Windows. --- README.md | 4 +- .../azure-pipelines/analyze-test-results.ps1 | 0 scripts/azure-pipelines/azure-pipelines.yml | 9 +- .../azure-pipelines/generate-skip-list.ps1 | 0 .../azure-pipelines/linux/azure-pipelines.yml | 39 ++ scripts/azure-pipelines/linux/create-vmss.ps1 | 408 ++++++++++++++++++ .../linux/initialize-environment.sh | 15 + .../azure-pipelines/linux/provision-image.sh | 75 ++++ .../{macos => osx}/azure-pipelines.yml | 9 +- .../windows/azure-pipelines.yml | 18 +- .../azure-pipelines/windows/create-vmss.ps1 | 35 +- .../windows/initialize-environment.ps1 | 5 - scripts/ci.baseline.txt | 30 +- 13 files changed, 604 insertions(+), 43 deletions(-) mode change 100644 => 100755 scripts/azure-pipelines/analyze-test-results.ps1 mode change 100644 => 100755 scripts/azure-pipelines/generate-skip-list.ps1 create mode 100644 scripts/azure-pipelines/linux/azure-pipelines.yml create mode 100755 scripts/azure-pipelines/linux/create-vmss.ps1 create mode 100755 scripts/azure-pipelines/linux/initialize-environment.sh create mode 100755 scripts/azure-pipelines/linux/provision-image.sh rename scripts/azure-pipelines/{macos => osx}/azure-pipelines.yml (91%) diff --git a/README.md b/README.md index f3aee11d34c192..21c48385461f36 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,7 @@ For short description of available commands, run `vcpkg help`. * Slack: [https://cppalliance.org/slack/](https://cppalliance.org/slack/), the #vcpkg channel * Docs: [Documentation](docs/index.md) -| Windows (x86, x64, arm, uwp) | MacOS | Linux | -| ------------- | ------------- | ------------- | -| [![Build Status](https://dev.azure.com/vcpkg/public/_apis/build/status/vcpkg-Windows-master-CI?branchName=master)](https://dev.azure.com/vcpkg/public/_build/latest?definitionId=9&branchName=master) | [![Build Status](https://dev.azure.com/vcpkg/public/_apis/build/status/vcpkg-osx-master-CI?branchName=master)](https://dev.azure.com/vcpkg/public/_build/latest?definitionId=11&branchName=master) | [![Build Status](https://dev.azure.com/vcpkg/public/_apis/build/status/vcpkg-Linux-master-CI?branchName=master)](https://dev.azure.com/vcpkg/public/_build/latest?definitionId=6&branchName=master) | +[![Build Status](https://dev.azure.com/vcpkg/public/_apis/build/status/microsoft.vcpkg?branchName=master)](https://dev.azure.com/vcpkg/public/_build/latest?definitionId=27&branchName=master) ## Quick Start Prerequisites: diff --git a/scripts/azure-pipelines/analyze-test-results.ps1 b/scripts/azure-pipelines/analyze-test-results.ps1 old mode 100644 new mode 100755 diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index f810a2e6a23a1a..7b6274845bfd76 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -2,7 +2,8 @@ # SPDX-License-Identifier: MIT # variables: - windows-pool: 'PrWin-2020-04-22' + windows-pool: 'PrWin-2020-04-28' + linux-pool: 'PrLin-2020-04-30' jobs: - template: windows/azure-pipelines.yml @@ -41,4 +42,8 @@ jobs: jobName: arm_uwp poolName: $(windows-pool) - - template: macos/azure-pipelines.yml + - template: osx/azure-pipelines.yml + + - template: linux/azure-pipelines.yml + parameters: + poolName: $(linux-pool) diff --git a/scripts/azure-pipelines/generate-skip-list.ps1 b/scripts/azure-pipelines/generate-skip-list.ps1 old mode 100644 new mode 100755 diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml new file mode 100644 index 00000000000000..c975626b4bfa46 --- /dev/null +++ b/scripts/azure-pipelines/linux/azure-pipelines.yml @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +# + +jobs: +- job: x64_linux + pool: + name: ${{ parameters.poolName }} + + timeoutInMinutes: 1440 # 1 day + + steps: + - task: Bash@3 + displayName: 'Initialize Environment' + inputs: + filePath: scripts/azure-pipelines/linux/initialize-environment.sh + - task: Bash@3 + displayName: 'Build vcpkg' + inputs: + filePath: bootstrap-vcpkg.sh + - powershell: | + $skipList = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-linux" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt" + mkdir $(System.ArtifactsDirectory)/xml-results + mkdir /ci/home + $env:HOME = '/ci/home' + ./vcpkg ci x64-linux --x-xunit="$(System.ArtifactsDirectory)/xml-results/x64-linux.xml" --exclude=$skipList --binarycaching + displayName: '** Test Modified Ports **' + - task: PowerShell@2 + displayName: 'Analyze results and prepare test logs' + inputs: + failOnStderr: true + filePath: 'scripts/azure-pipelines/analyze-test-results.ps1' + arguments: '-baselineFile $(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt -logDir "$(System.ArtifactsDirectory)/xml-results" -outputDir "$(Build.ArtifactStagingDirectory)" -failurelogDir "archives/fail" -triplets "x64-linux" -errorOnRegression' + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: x86-linux Build Failure Logs' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/failureLogs' + ArtifactName: 'x64-linux port build failure logs' + condition: failed() diff --git a/scripts/azure-pipelines/linux/create-vmss.ps1 b/scripts/azure-pipelines/linux/create-vmss.ps1 new file mode 100755 index 00000000000000..dfe2102d914218 --- /dev/null +++ b/scripts/azure-pipelines/linux/create-vmss.ps1 @@ -0,0 +1,408 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +# + +<# +.SYNOPSIS +Creates a Linux virtual machine scale set, set up for vcpkg's CI. + +.DESCRIPTION +create-vmss.ps1 creates an Azure Linux VM scale set, set up for vcpkg's CI +system. See https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/overview +for more information. + +This script assumes you have installed Azure tools into PowerShell by following the instructions +at https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-3.6.1 +or are running from Azure Cloud Shell. +#> + +$Location = 'westus2' +$Prefix = 'PrLin-' + (Get-Date -Format 'yyyy-MM-dd') +$VMSize = 'Standard_F16s_v2' +$ProtoVMName = 'PROTOTYPE' +$LiveVMPrefix = 'BUILD' +$InstalledDiskSizeInGB = 1024 +$ErrorActionPreference = 'Stop' + +$ProgressActivity = 'Creating Scale Set' +$TotalProgress = 10 +$CurrentProgress = 1 + +<# +.SYNOPSIS +Returns whether there's a name collision in the resource group. + +.DESCRIPTION +Find-ResourceGroupNameCollision takes a list of resources, and checks if $Test +collides names with any of the resources. + +.PARAMETER Test +The name to test. + +.PARAMETER Resources +The list of resources. +#> +function Find-ResourceGroupNameCollision { + [CmdletBinding()] + Param([string]$Test, $Resources) + + foreach ($resource in $Resources) { + if ($resource.ResourceGroupName -eq $Test) { + return $true + } + } + + return $false +} + +<# +.SYNOPSIS +Attempts to find a name that does not collide with any resources in the resource group. + +.DESCRIPTION +Find-ResourceGroupName takes a set of resources from Get-AzResourceGroup, and finds the +first name in {$Prefix, $Prefix-1, $Prefix-2, ...} such that the name doesn't collide with +any of the resources in the resource group. + +.PARAMETER Prefix +The prefix of the final name; the returned name will be of the form "$Prefix(-[1-9][0-9]*)?" +#> +function Find-ResourceGroupName { + [CmdletBinding()] + Param([string] $Prefix) + + $resources = Get-AzResourceGroup + $result = $Prefix + $suffix = 0 + while (Find-ResourceGroupNameCollision -Test $result -Resources $resources) { + $suffix++ + $result = "$Prefix-$suffix" + } + + return $result +} + +<# +.SYNOPSIS +Creates a randomly generated password. + +.DESCRIPTION +New-Password generates a password, randomly, of length $Length, containing +only alphanumeric characters (both uppercase and lowercase). + +.PARAMETER Length +The length of the returned password. +#> +function New-Password { + Param ([int] $Length = 32) + + $Chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + $result = '' + for ($idx = 0; $idx -lt $Length; $idx++) { + # NOTE: this should probably use RNGCryptoServiceProvider + $result += $Chars[(Get-Random -Minimum 0 -Maximum $Chars.Length)] + } + + return $result +} + +<# +.SYNOPSIS +Sanitizes a name to be used in a storage account. + +.DESCRIPTION +Sanitize-Name takes a string, and removes all of the '-'s and +lowercases the string, since storage account names must have no +'-'s and must be completely lowercase alphanumeric. It then makes +certain that the length of the string is not greater than 24, +since that is invalid. + +.PARAMETER RawName +The name to sanitize. +#> +function Sanitize-Name { + [CmdletBinding()] + Param( + [string]$RawName + ) + + $result = $RawName.Replace('-', '').ToLowerInvariant() + if ($result.Length -gt 24) { + Write-Error 'Sanitized name for storage account $result was too long.' + } + + return $result +} + +#################################################################################################### +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Creating resource group' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) + +$ResourceGroupName = Find-ResourceGroupName $Prefix +$AdminPW = New-Password +New-AzResourceGroup -Name $ResourceGroupName -Location $Location +$AdminPWSecure = ConvertTo-SecureString $AdminPW -AsPlainText -Force +$Credential = New-Object System.Management.Automation.PSCredential ("AdminUser", $AdminPWSecure) + +#################################################################################################### +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Creating virtual network' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) + +$allowHttp = New-AzNetworkSecurityRuleConfig ` + -Name AllowHTTP ` + -Description 'Allow HTTP(S)' ` + -Access Allow ` + -Protocol Tcp ` + -Direction Outbound ` + -Priority 1008 ` + -SourceAddressPrefix * ` + -SourcePortRange * ` + -DestinationAddressPrefix * ` + -DestinationPortRange @(80, 443) + +$allowDns = New-AzNetworkSecurityRuleConfig ` + -Name AllowDNS ` + -Description 'Allow DNS' ` + -Access Allow ` + -Protocol * ` + -Direction Outbound ` + -Priority 1009 ` + -SourceAddressPrefix * ` + -SourcePortRange * ` + -DestinationAddressPrefix * ` + -DestinationPortRange 53 + +$allowGit = New-AzNetworkSecurityRuleConfig ` + -Name AllowGit ` + -Description 'Allow git' ` + -Access Allow ` + -Protocol Tcp ` + -Direction Outbound ` + -Priority 1010 ` + -SourceAddressPrefix * ` + -SourcePortRange * ` + -DestinationAddressPrefix * ` + -DestinationPortRange 9418 + +$allowStorage = New-AzNetworkSecurityRuleConfig ` + -Name AllowStorage ` + -Description 'Allow Storage' ` + -Access Allow ` + -Protocol * ` + -Direction Outbound ` + -Priority 1011 ` + -SourceAddressPrefix VirtualNetwork ` + -SourcePortRange * ` + -DestinationAddressPrefix Storage ` + -DestinationPortRange * + +$denyEverythingElse = New-AzNetworkSecurityRuleConfig ` + -Name DenyElse ` + -Description 'Deny everything else' ` + -Access Deny ` + -Protocol * ` + -Direction Outbound ` + -Priority 1012 ` + -SourceAddressPrefix * ` + -SourcePortRange * ` + -DestinationAddressPrefix * ` + -DestinationPortRange * + +$NetworkSecurityGroupName = $ResourceGroupName + 'NetworkSecurity' +$NetworkSecurityGroup = New-AzNetworkSecurityGroup ` + -Name $NetworkSecurityGroupName ` + -ResourceGroupName $ResourceGroupName ` + -Location $Location ` + -SecurityRules @($allowHttp, $allowDns, $allowGit, $allowStorage, $denyEverythingElse) + +$SubnetName = $ResourceGroupName + 'Subnet' +$Subnet = New-AzVirtualNetworkSubnetConfig ` + -Name $SubnetName ` + -AddressPrefix "10.0.0.0/16" ` + -NetworkSecurityGroup $NetworkSecurityGroup + +$VirtualNetworkName = $ResourceGroupName + 'Network' +$VirtualNetwork = New-AzVirtualNetwork ` + -Name $VirtualNetworkName ` + -ResourceGroupName $ResourceGroupName ` + -Location $Location ` + -AddressPrefix "10.0.0.0/16" ` + -Subnet $Subnet + +#################################################################################################### +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Creating archives storage account' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) + +$StorageAccountName = Sanitize-Name $ResourceGroupName + +New-AzStorageAccount ` + -ResourceGroupName $ResourceGroupName ` + -Location $Location ` + -Name $StorageAccountName ` + -SkuName 'Standard_LRS' ` + -Kind StorageV2 + +$StorageAccountKeys = Get-AzStorageAccountKey ` + -ResourceGroupName $ResourceGroupName ` + -Name $StorageAccountName + +$StorageAccountKey = $StorageAccountKeys[0].Value + +$StorageContext = New-AzStorageContext ` + -StorageAccountName $StorageAccountName ` + -StorageAccountKey $StorageAccountKey + +New-AzStorageShare -Name 'archives' -Context $StorageContext +Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 5120 + +#################################################################################################### +Write-Progress ` + -Activity 'Creating prototype VM' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) + +$NicName = $ResourceGroupName + 'NIC' +$Nic = New-AzNetworkInterface ` + -Name $NicName ` + -ResourceGroupName $ResourceGroupName ` + -Location $Location ` + -Subnet $VirtualNetwork.Subnets[0] + +$VM = New-AzVMConfig -Name $ProtoVMName -VMSize $VMSize +$VM = Set-AzVMOperatingSystem ` + -VM $VM ` + -Linux ` + -ComputerName $ProtoVMName ` + -Credential $Credential + +$VM = Add-AzVMNetworkInterface -VM $VM -Id $Nic.Id +$VM = Set-AzVMSourceImage ` + -VM $VM ` + -PublisherName 'Canonical' ` + -Offer 'UbuntuServer' ` + -Skus '18.04-LTS' ` + -Version latest + +$VM = Set-AzVMBootDiagnostic -VM $VM -Disable +New-AzVm ` + -ResourceGroupName $ResourceGroupName ` + -Location $Location ` + -VM $VM + +#################################################################################################### +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Running provisioning script provision-image.sh in VM' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) + +Invoke-AzVMRunCommand ` + -ResourceGroupName $ResourceGroupName ` + -VMName $ProtoVMName ` + -CommandId 'RunShellScript' ` + -ScriptPath "$PSScriptRoot\provision-image.sh" ` + -Parameter @{StorageAccountName=$StorageAccountName; ` + StorageAccountKey=$StorageAccountKey;} + +#################################################################################################### +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Restarting VM' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) + +Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName + +#################################################################################################### +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Converting VM to Image' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) + +Stop-AzVM ` + -ResourceGroupName $ResourceGroupName ` + -Name $ProtoVMName ` + -Force + +Set-AzVM ` + -ResourceGroupName $ResourceGroupName ` + -Name $ProtoVMName ` + -Generalized + +$VM = Get-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName +$PrototypeOSDiskName = $VM.StorageProfile.OsDisk.Name +$ImageConfig = New-AzImageConfig -Location $Location -SourceVirtualMachineId $VM.ID +$Image = New-AzImage -Image $ImageConfig -ImageName $ProtoVMName -ResourceGroupName $ResourceGroupName + +#################################################################################################### +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Deleting unused VM and disk' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) + +Remove-AzVM -Id $VM.ID -Force +Remove-AzDisk -ResourceGroupName $ResourceGroupName -DiskName $PrototypeOSDiskName -Force + +#################################################################################################### +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Creating scale set' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) + +$VmssIpConfigName = $ResourceGroupName + 'VmssIpConfig' +$VmssIpConfig = New-AzVmssIpConfig -SubnetId $Nic.IpConfigurations[0].Subnet.Id -Primary -Name $VmssIpConfigName +$VmssName = $ResourceGroupName + 'Vmss' +$Vmss = New-AzVmssConfig ` + -Location $Location ` + -SkuCapacity 0 ` + -SkuName $VMSize ` + -SkuTier 'Standard' ` + -Overprovision $false ` + -UpgradePolicyMode Manual ` + -EvictionPolicy Delete ` + -Priority Spot ` + -MaxPrice -1 + +$Vmss = Add-AzVmssNetworkInterfaceConfiguration ` + -VirtualMachineScaleSet $Vmss ` + -Primary $true ` + -IpConfiguration $VmssIpConfig ` + -NetworkSecurityGroupId $NetworkSecurityGroup.Id ` + -Name $NicName + +$Vmss = Set-AzVmssOsProfile ` + -VirtualMachineScaleSet $Vmss ` + -ComputerNamePrefix $LiveVMPrefix ` + -AdminUsername AdminUser ` + -AdminPassword $AdminPW ` + -CustomData ([Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes("#!/bin/bash`n/etc/provision-disks.sh`n"))) + +$Vmss = Set-AzVmssStorageProfile ` + -VirtualMachineScaleSet $Vmss ` + -OsDiskCreateOption 'FromImage' ` + -OsDiskCaching ReadWrite ` + -ImageReferenceId $Image.Id + +$Vmss = Add-AzVmssDataDisk ` + -VirtualMachineScaleSet $Vmss ` + -Lun 0 ` + -Caching 'ReadWrite' ` + -CreateOption Empty ` + -DiskSizeGB 1024 ` + -StorageAccountType 'StandardSSD_LRS' + +New-AzVmss ` + -ResourceGroupName $ResourceGroupName ` + -Name $VmssName ` + -VirtualMachineScaleSet $Vmss + +#################################################################################################### +Write-Progress -Activity $ProgressActivity -Completed +Write-Host "Location: $Location" +Write-Host "Resource group name: $ResourceGroupName" +Write-Host "User name: AdminUser" +Write-Host "Using generated password: $AdminPW" +Write-Host 'Finished!' diff --git a/scripts/azure-pipelines/linux/initialize-environment.sh b/scripts/azure-pipelines/linux/initialize-environment.sh new file mode 100755 index 00000000000000..82b52bd2e8878b --- /dev/null +++ b/scripts/azure-pipelines/linux/initialize-environment.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +# Cleans up the environment to prevent contamination across builds. +if [ ! -d "archives" ]; then + ln -s /ci/archives archives +fi +if [ ! -d "installed" ]; then + ln -s /ci/installed installed +fi + +rm -rf installed/* +rm -rf buildtrees +rm -rf packages diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh new file mode 100755 index 00000000000000..b281529e1c3047 --- /dev/null +++ b/scripts/azure-pipelines/linux/provision-image.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT +# + +sudo apt -y update +sudo apt -y dist-upgrade +# Install common build dependencies and partitioning tools +sudo apt -y install at curl unzip tar libxt-dev gperf libxaw7-dev cifs-utils build-essential zip libx11-dev libxi-dev libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev libxcursor-dev yasm nasm libnuma1 libnuma-dev python-six python3-six python-yaml flex libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev xutils-dev dh-autoreconf libgles2-mesa-dev ruby-full pkg-config +# Required by qt5-x11extras +sudo apt -y install libxkbcommon-dev libxkbcommon-x11-dev +# Required by libhdfs3 +sudo apt -y install libkrb5-dev +# Required by intel-ipsec +sudo apt -y install nasm + +# Install newer version of nasm than the apt package +mkdir /tmp/nasm +cd /tmp/nasm +curl -O https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/nasm-2.14.02.tar.gz +tar -xf nasm-2.14.02.tar.gz +cd nasm-2.14.02/ +./configure --prefix=/usr && make +sudo make install +cd ~ + +# Install the latest Haskell stack +curl -sSL https://get.haskellstack.org/ | sh + +# Install CUDA +wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_10.2.89-1_amd64.deb +sudo apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub +sudo dpkg -i cuda-repo-ubuntu1804_10.2.89-1_amd64.deb +wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb +sudo dpkg -i nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb +sudo apt -y update +sudo apt install -y --no-install-recommends cuda-compiler-10-2 cuda-libraries-dev-10-2 cuda-driver-dev-10-2 cuda-cudart-dev-10-2 libcublas10 cuda-curand-dev-10-2 +sudo apt install -y --no-install-recommends libcudnn7-dev +sudo ln -s /usr/local/cuda-10.1/lib64/stubs/libcuda.so /usr/local/cuda-10.1/lib64/stubs/libcuda.so.1 + +# Install PowerShell +wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb +sudo dpkg -i packages-microsoft-prod.deb +sudo apt update +sudo add-apt-repository universe +sudo apt install -y powershell + +# Write SMB credentials +sudo mkdir /etc/smbcredentials +smbCredentialFile=/etc/smbcredentials/$StorageAccountName.cred +echo "username=$StorageAccountName" | sudo tee $smbCredentialFile > /dev/null +echo "password=$StorageAccountKey" | sudo tee -a $smbCredentialFile > /dev/null +sudo chmod 600 $smbCredentialFile + +# Write script to provision disks used by cloud-init +echo "if [ ! -d \"/ci\" ]; then" > /etc/provision-disks.sh +echo "sudo parted /dev/sdc mklabel gpt" >> /etc/provision-disks.sh +echo "sudo parted /dev/sdc mkpart cidisk ext4 0% 100%" >> /etc/provision-disks.sh +echo "sudo mkfs -t ext4 /dev/sdc1" >> /etc/provision-disks.sh +echo "sudo mkdir /ci -m=777" >> /etc/provision-disks.sh +echo "sudo mkdir /ci/installed -m=777" >> /etc/provision-disks.sh +echo "sudo mkdir /ci/archives -m=777" >> /etc/provision-disks.sh +echo "echo \"/dev/sdc1 /ci/installed ext4 barrier=0 0 0\" | sudo tee -a /etc/fstab" >> /etc/provision-disks.sh +echo "echo \"//$StorageAccountName.file.core.windows.net/archives /ci/archives cifs nofail,vers=3.0,credentials=$smbCredentialFile,serverino,dir_mode=0777,file_mode=0777 0 0\" | sudo tee -a /etc/fstab" >> /etc/provision-disks.sh +echo "sudo mount -a" >> /etc/provision-disks.sh +echo "fi" >> /etc/provision-disks.sh +sudo chmod 700 /etc/provision-disks.sh + +# Delete /etc/debian_version to prevent Azure Pipelines Scale Set Agents from removing some of the above +sudo rm /etc/debian_version + +# Install dependencies that the Azure Pipelines agent will want later to make launching VMs faster +# https://docs.microsoft.com/en-us/dotnet/core/install/dependencies?tabs=netcore31&pivots=os-linux +# (we assume libssl1.0.0 or equivalent is already installed to not accidentially change SSL certs) +apt install -y liblttng-ust0 libkrb5-3 zlib1g libicu60 diff --git a/scripts/azure-pipelines/macos/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml similarity index 91% rename from scripts/azure-pipelines/macos/azure-pipelines.yml rename to scripts/azure-pipelines/osx/azure-pipelines.yml index 68ced441c4dac9..45d9c0a7f0301b 100644 --- a/scripts/azure-pipelines/macos/azure-pipelines.yml +++ b/scripts/azure-pipelines/osx/azure-pipelines.yml @@ -47,10 +47,11 @@ jobs: brew list libtool || brew install libtool displayName: 'Install brew dependencies' - - bash: | - ./bootstrap-vcpkg.sh -allowAppleClang + - task: Bash@3 displayName: 'Build vcpkg' - + inputs: + filePath: bootstrap-vcpkg.sh + arguments: '-allowAppleClang' - powershell: | $skip_list = ./scripts/azure-pipelines/generate-skip-list.ps1 -Triplet "x64-osx" -BaselineFile "$(System.DefaultWorkingDirectory)/scripts/ci.baseline.txt" mkdir $(System.ArtifactsDirectory)/xml-results @@ -68,5 +69,5 @@ jobs: displayName: 'Publish Artifact: x64-osx port build failure logs' inputs: PathtoPublish: '$(Build.ArtifactStagingDirectory)/failureLogs' - ArtifactName: 'Port build failure logs' + ArtifactName: 'x64-osx port build failure logs' condition: failed() diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml index 2e2e38c96191ac..559037e3b9f3ab 100644 --- a/scripts/azure-pipelines/windows/azure-pipelines.yml +++ b/scripts/azure-pipelines/windows/azure-pipelines.yml @@ -7,9 +7,6 @@ jobs: pool: name: ${{ parameters.poolName }} - variables: - triplet: '${{ parameters.triplet }}' - timeoutInMinutes: 1440 # 1 day steps: @@ -20,17 +17,17 @@ jobs: - powershell: | $baselineFile = "$(System.DefaultWorkingDirectory)\scripts\ci.baseline.txt" - $skipList = $(System.DefaultWorkingDirectory)\scripts\azure-pipelines\generate-skip-list.ps1 -Triplet "$(triplet)" -BaselineFile $baselineFile + $skipList = $(System.DefaultWorkingDirectory)\scripts\azure-pipelines\generate-skip-list.ps1 -Triplet "${{ parameters.triplet }}" -BaselineFile $baselineFile Write-Host "baseline file: $baselineFile" Write-Host "skip list: $skipList" - $(System.DefaultWorkingDirectory)\scripts\azure-pipelines\windows\ci-step.ps1 -Triplet "$(triplet)" -ExcludePorts $skipList + $(System.DefaultWorkingDirectory)\scripts\azure-pipelines\windows\ci-step.ps1 -Triplet "${{ parameters.triplet }}" -ExcludePorts $skipList Write-Host "CI test script is complete" errorActionPreference: continue displayName: '** Build vcpkg and Test Modified Ports **' - powershell: | - $baseName = "$(triplet)" - $outputPathRoot = "$(System.ArtifactsDirectory)\raw xml results" + $baseName = "${{ parameters.triplet }}" + $outputPathRoot = "$(System.ArtifactsDirectory)\xml-results" if(-not (Test-Path $outputPathRoot)) { Write-Host "creating $outputPathRoot" @@ -56,11 +53,10 @@ jobs: inputs: failOnStderr: true filePath: 'scripts/azure-pipelines/analyze-test-results.ps1' - arguments: '-baselineFile ''$(System.DefaultWorkingDirectory)\scripts\ci.baseline.txt'' -logDir ''$(System.ArtifactsDirectory)\raw xml results'' -failurelogDir ''archives\fail'' -outputDir ''$(Build.ArtifactStagingDirectory)'' -errorOnRegression -triplets ''$(triplet)''' - + arguments: '-baselineFile "$(System.DefaultWorkingDirectory)\scripts\ci.baseline.txt" -logDir "$(System.ArtifactsDirectory)\xml-results" -failurelogDir "archives\fail" -outputDir "$(Build.ArtifactStagingDirectory)" -errorOnRegression -triplets "${{ parameters.triplet }}"' - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: $(triplet) port build failure logs' + displayName: 'Publish Artifact: ${{ parameters.triplet }} port build failure logs' inputs: PathtoPublish: '$(Build.ArtifactStagingDirectory)\failureLogs' - ArtifactName: '$(triplet) port build failure logs' + ArtifactName: '${{ parameters.triplet }} port build failure logs' condition: failed() diff --git a/scripts/azure-pipelines/windows/create-vmss.ps1 b/scripts/azure-pipelines/windows/create-vmss.ps1 index 099c7dbfb7801e..8c05cd51df5766 100644 --- a/scripts/azure-pipelines/windows/create-vmss.ps1 +++ b/scripts/azure-pipelines/windows/create-vmss.ps1 @@ -1,7 +1,6 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: MIT # -# <# .SYNOPSIS @@ -17,7 +16,7 @@ at https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-3.6 or are running from Azure Cloud Shell. #> -$Location = 'SouthCentralUS' +$Location = 'westus2' $Prefix = 'PrWin-' + (Get-Date -Format 'yyyy-MM-dd') $VMSize = 'Standard_F16s_v2' $ProtoVMName = 'PROTOTYPE' @@ -212,13 +211,25 @@ $allowDns = New-AzNetworkSecurityRuleConfig ` -DestinationAddressPrefix * ` -DestinationPortRange 53 +$allowGit = New-AzNetworkSecurityRuleConfig ` + -Name AllowGit ` + -Description 'Allow git' ` + -Access Allow ` + -Protocol Tcp ` + -Direction Outbound ` + -Priority 1010 ` + -SourceAddressPrefix * ` + -SourcePortRange * ` + -DestinationAddressPrefix * ` + -DestinationPortRange 9418 + $allowStorage = New-AzNetworkSecurityRuleConfig ` -Name AllowStorage ` -Description 'Allow Storage' ` -Access Allow ` -Protocol * ` -Direction Outbound ` - -Priority 1010 ` + -Priority 1011 ` -SourceAddressPrefix VirtualNetwork ` -SourcePortRange * ` -DestinationAddressPrefix Storage ` @@ -230,7 +241,7 @@ $denyEverythingElse = New-AzNetworkSecurityRuleConfig ` -Access Deny ` -Protocol * ` -Direction Outbound ` - -Priority 1011 ` + -Priority 1012 ` -SourceAddressPrefix * ` -SourcePortRange * ` -DestinationAddressPrefix * ` @@ -241,7 +252,7 @@ $NetworkSecurityGroup = New-AzNetworkSecurityGroup ` -Name $NetworkSecurityGroupName ` -ResourceGroupName $ResourceGroupName ` -Location $Location ` - -SecurityRules @($allowHttp, $allowDns, $allowStorage, $denyEverythingElse) + -SecurityRules @($allowHttp, $allowDns, $allowGit, $allowStorage, $denyEverythingElse) $SubnetName = $ResourceGroupName + 'Subnet' $Subnet = New-AzVirtualNetworkSubnetConfig ` @@ -282,10 +293,8 @@ $StorageContext = New-AzStorageContext ` -StorageAccountName $StorageAccountName ` -StorageAccountKey $StorageAccountKey -$ArchivesFiles = New-AzStorageShare -Name 'archives' -Context $StorageContext +New-AzStorageShare -Name 'archives' -Context $StorageContext Set-AzStorageShareQuota -ShareName 'archives' -Context $StorageContext -Quota 5120 -$LogFiles = New-AzStorageShare -Name 'logs' -Context $StorageContext -Set-AzStorageShareQuota -ShareName 'logs' -Context $StorageContext -Quota 64 #################################################################################################### Write-Progress ` @@ -305,8 +314,7 @@ $VM = Set-AzVMOperatingSystem ` -Windows ` -ComputerName $ProtoVMName ` -Credential $Credential ` - -ProvisionVMAgent ` - -EnableAutoUpdate + -ProvisionVMAgent $VM = Add-AzVMNetworkInterface -VM $VM -Id $Nic.Id $VM = Set-AzVMSourceImage ` @@ -417,11 +425,14 @@ $VmssIpConfig = New-AzVmssIpConfig -SubnetId $Nic.IpConfigurations[0].Subnet.Id $VmssName = $ResourceGroupName + 'Vmss' $Vmss = New-AzVmssConfig ` -Location $Location ` - -SkuCapacity 6 ` + -SkuCapacity 0 ` -SkuName $VMSize ` -SkuTier 'Standard' ` -Overprovision $false ` - -UpgradePolicyMode Manual + -UpgradePolicyMode Manual ` + -EvictionPolicy Delete ` + -Priority Spot ` + -MaxPrice -1 $Vmss = Add-AzVmssNetworkInterfaceConfiguration ` -VirtualMachineScaleSet $Vmss ` diff --git a/scripts/azure-pipelines/windows/initialize-environment.ps1 b/scripts/azure-pipelines/windows/initialize-environment.ps1 index b86006a9cf5068..4bbc15665b69aa 100644 --- a/scripts/azure-pipelines/windows/initialize-environment.ps1 +++ b/scripts/azure-pipelines/windows/initialize-environment.ps1 @@ -34,11 +34,6 @@ if (-Not (Test-Path W:)) { net use W: "\\$StorageAccountName.file.core.windows.net\archives" /u:"AZURE\$StorageAccountName" $StorageAccountKey } -Write-Host 'Setting up logs mount' -if (-Not (Test-Path L:)) { - net use L: "\\$StorageAccountName.file.core.windows.net\logs" /u:"AZURE\$StorageAccountName" $StorageAccountKey -} - Write-Host 'Creating downloads directory' mkdir D:\downloads -ErrorAction SilentlyContinue diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index 3dcc559413664d..b8695e15fbd82c 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -340,12 +340,13 @@ dmlc:arm64-windows=ignore dmlc:x64-uwp=fail dmlc:x64-windows-static=ignore dmlc:x86-windows=ignore -dpdk:arm64-windows=fail dpdk:arm-uwp=fail +dpdk:arm64-windows=fail +dpdk:x64-linux=fail dpdk:x64-osx=fail dpdk:x64-uwp=fail -dpdk:x64-windows=fail dpdk:x64-windows-static=fail +dpdk:x64-windows=fail dpdk:x86-windows=fail duckx:arm64-windows = skip duckx:arm-uwp = skip @@ -560,10 +561,15 @@ graphqlparser:x64-uwp=fail grpc:arm-uwp=fail gsl:arm-uwp=fail gsl:x64-uwp=fail -gsoap:arm-uwp=ignore +# https://github.com/microsoft/vcpkg/pull/11048/files +gsoap:arm-uwp=fail +gsoap:arm64-windows=fail gsoap:x64-linux=fail gsoap:x64-osx=fail -gsoap:x86-windows=ignore +gsoap:x64-uwp=fail +gsoap:x64-windows-static=fail +gsoap:x64-windows=fail +gsoap:x86-windows=fail gtk:x64-linux=fail gtk:x86-windows=ignore guetzli:x64-osx=fail @@ -1116,6 +1122,11 @@ mozjpeg:x64-uwp = skip mozjpeg:x64-windows = skip mozjpeg:x64-windows-static = skip mozjpeg:x86-windows = skip +# mp3lame emits error LNK2019: unresolved external symbol __imp_RoInitialize referenced in function +# __scrt_initialize_winrt; likely missing an import library +mp3lame:arm-uwp=fail +mp3lame:arm64-uwp=fail +mp3lame:x64-uwp=fail mpg123:arm-uwp=fail mpg123:x64-uwp=fail mpir:arm64-windows=fail @@ -1346,8 +1357,12 @@ pangomm:x86-windows=ignore pangomm:arm64-windows=fail parmetis:x64-linux=fail parmetis:x64-osx=fail +# https://github.com/microsoft/vcpkg/issues/11050 pbc:arm-uwp=fail +pbc:x86-windows=fail +pbc:x64-windows=fail pbc:x64-uwp=fail +pbc:x64-windows-static=fail pcre2:arm-uwp=fail pdal:x64-linux=fail pdal:x64-osx=fail @@ -1364,6 +1379,9 @@ pfring:x64-windows=fail pfring:x64-windows-static=fail pfring:x86-windows=fail pfring:x64-osx=fail +# pfring on Linux currently fails because its build scripts enable warnings as +# errors, and warnings trigger with the Linux kernel headers in the Azure images. +pfring:x64-linux=fail physfs:arm64-windows=fail physx:arm64-windows=fail physx:x64-linux=fail @@ -1393,12 +1411,13 @@ plibsys:x64-uwp=fail plplot:arm64-windows=fail plplot:arm-uwp=fail plplot:x64-uwp=fail -pmdk:arm64-windows=fail pmdk:arm-uwp=fail +pmdk:arm64-windows=fail pmdk:x64-linux=fail pmdk:x64-osx=fail pmdk:x64-uwp=fail pmdk:x64-windows-static=fail +pmdk:x64-windows=fail pmdk:x86-windows=fail pngwriter:arm-uwp=fail pngwriter:x64-uwp=fail @@ -1550,7 +1569,6 @@ redis-plus-plus:x64-windows-static=fail redis-plus-plus:arm64-windows=fail replxx:arm-uwp=fail replxx:x64-uwp=fail -replxx:x64-windows-static=fail reproc:arm-uwp=fail reproc:x64-uwp=fail restbed:arm-uwp=fail From 4834f641bbda098533107045b826152a863ade68 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 29 Apr 2020 13:06:27 -0700 Subject: [PATCH 2/7] [argh] Patch argh to not take the name 'license'. --- ports/argh/CONTROL | 2 +- ports/argh/portfile.cmake | 2 + .../remove_unnamespaced_license_file.patch | 67 +++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 ports/argh/remove_unnamespaced_license_file.patch diff --git a/ports/argh/CONTROL b/ports/argh/CONTROL index db804708f68e3f..045c96eedb9479 100644 --- a/ports/argh/CONTROL +++ b/ports/argh/CONTROL @@ -1,3 +1,3 @@ Source: argh -Version: 2018-12-18-1 +Version: 2018-12-18-2 Description: Argh! A minimalist argument handler. diff --git a/ports/argh/portfile.cmake b/ports/argh/portfile.cmake index 40e7fbc8411763..eb64117725dbc7 100644 --- a/ports/argh/portfile.cmake +++ b/ports/argh/portfile.cmake @@ -8,6 +8,8 @@ vcpkg_from_github( REF a1edee559757e076e570b8f6c2f555d8d00b373c SHA512 a100c7ff20ef9ed39d53efeac5507a6ed59fb99ccba36ac4b5f8f5aaac6782f8e951b2f26b9b50f6c6fdbc53b5bceaabbb9dd9b9539f968fc1037733342e17a6 HEAD_REF master + PATCHES + remove_unnamespaced_license_file.patch # https://github.com/adishavit/argh/pull/51 ) vcpkg_configure_cmake( diff --git a/ports/argh/remove_unnamespaced_license_file.patch b/ports/argh/remove_unnamespaced_license_file.patch new file mode 100644 index 00000000000000..f6742c86c634b5 --- /dev/null +++ b/ports/argh/remove_unnamespaced_license_file.patch @@ -0,0 +1,67 @@ +From 83f364a2eb1b137453772ae9fcf5d7973770637f Mon Sep 17 00:00:00 2001 +From: Billy Robert O'Neal III +Date: Wed, 29 Apr 2020 13:01:31 -0700 +Subject: [PATCH] Don't install 'LICENSE' or 'README.md' directly to the + installation directory. + +This change unconditionally uses GNUInstallDirs which will correctly 'namespace' these files with the 'argh' project name. + +========= + +The names 'LICENSE' and 'README.md' are extremely common and likely to be taken by other ports. This happened in a recent full vcpkg catalog rebuild: + +Starting package 757/1254: qt-advanced-docking-system:x86-windows +Building package qt-advanced-docking-system[core]:x86-windows... +Could not locate cached archive: C:\agent\_work\1\s\archives\a1\a1507296ead5788137561997571557204aaac78f.zip +-- Downloading https://github.com/githubuser0xFFFF/Qt-Advanced-Docking-System/archive/661d0c4356ab61d155b53718aa1bf6bab86dad13.tar.gz... +-- Extracting source C:/agent/_work/1/s/downloads/githubuser0xFFFF-Qt-Advanced-Docking-System-661d0c4356ab61d155b53718aa1bf6bab86dad13.tar.gz +-- Applying patch config_changes.patch +-- Using source at C:/agent/_work/1/s/buildtrees/qt-advanced-docking-system/src/bab86dad13-3088149a9b +-- Configuring x86-windows +-- Building x86-windows-dbg +-- Building x86-windows-rel +-- Installing: C:/agent/_work/1/s/packages/qt-advanced-docking-system_x86-windows/share/qt-advanced-docking-system/copyright +-- Performing post-build validation +-- Performing post-build validation done +Stored binary cache: C:\agent\_work\1\s\archives\a1\a1507296ead5788137561997571557204aaac78f.zip +Building package qt-advanced-docking-system[core]:x86-windows... done +Installing package qt-advanced-docking-system[core]:x86-windows... +The following files are already installed in C:/agent/_work/1/s/installed/x86-windows and are in conflict with qt-advanced-docking-system:x86-windows + +Installed by argh:x86-windows + license/LICENSE + +Elapsed time for package qt-advanced-docking-system:x86-windows: 11.09 s +--- + CMakeLists.txt | 12 +++++------- + 1 file changed, 5 insertions(+), 7 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 13bc879..f6264f3 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -36,19 +36,17 @@ endif() + + install(TARGETS argh EXPORT arghTargets) + ++include(GNUInstallDirs) ++install(FILES "${CMAKE_CURRENT_LIST_DIR}/argh.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) ++install(FILES "${CMAKE_CURRENT_LIST_DIR}/LICENSE" DESTINATION ${CMAKE_INSTALL_DOCDIR}) ++install(FILES "${CMAKE_CURRENT_LIST_DIR}/README.md" DESTINATION ${CMAKE_INSTALL_DOCDIR}) ++ + if(CMAKE_SYSTEM_NAME STREQUAL Linux) + # this might be a bit too restrictive, since for other (BSD, ...) this might apply also + # but this can be fixed later in extra pull requests from people on the platform +- include(GNUInstallDirs) +- install(FILES "${CMAKE_CURRENT_LIST_DIR}/argh.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +- install(FILES "${CMAKE_CURRENT_LIST_DIR}/LICENSE" DESTINATION ${CMAKE_INSTALL_DOCDIR}) +- install(FILES "${CMAKE_CURRENT_LIST_DIR}/README.md" DESTINATION ${CMAKE_INSTALL_DOCDIR}) + install(FILES argh-config.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/argh) + install(EXPORT arghTargets DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/argh) + else() +- install(FILES "${CMAKE_CURRENT_LIST_DIR}/argh.h" DESTINATION include) +- install(FILES "${CMAKE_CURRENT_LIST_DIR}/LICENSE" DESTINATION license) +- install(FILES "${CMAKE_CURRENT_LIST_DIR}/README.md" DESTINATION .) + install(FILES argh-config.cmake DESTINATION CMake) + install(EXPORT arghTargets DESTINATION CMake) + endif() From ba3bdbfc7b9ca023b9d56bc307707fb2cc08f857 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 29 Apr 2020 13:39:22 -0700 Subject: [PATCH 3/7] [qt-advanced-docking-system] Patch qt-advanced-docking-system to not install LICENSE. --- ports/qt-advanced-docking-system/CONTROL | 4 ++-- .../qt-advanced-docking-system/portfile.cmake | 2 ++ .../skip_license_install.patch | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 ports/qt-advanced-docking-system/skip_license_install.patch diff --git a/ports/qt-advanced-docking-system/CONTROL b/ports/qt-advanced-docking-system/CONTROL index 9b1dda81bc5c79..478551458a3433 100644 --- a/ports/qt-advanced-docking-system/CONTROL +++ b/ports/qt-advanced-docking-system/CONTROL @@ -1,5 +1,5 @@ Source: qt-advanced-docking-system -Version: 3.2.5 +Version: 3.2.5-1 Build-Depends: qt5-base, zlib, bzip2 Description: Create customizable layouts using an advanced window docking system similar to what is found in many popular IDEs such as Visual Studio -Homepage: https://github.com/githubuser0xFFFF/Qt-Advanced-Docking-System \ No newline at end of file +Homepage: https://github.com/githubuser0xFFFF/Qt-Advanced-Docking-System diff --git a/ports/qt-advanced-docking-system/portfile.cmake b/ports/qt-advanced-docking-system/portfile.cmake index 3f9577df97f412..dd70c21e2b92d7 100644 --- a/ports/qt-advanced-docking-system/portfile.cmake +++ b/ports/qt-advanced-docking-system/portfile.cmake @@ -6,6 +6,7 @@ vcpkg_from_github( HEAD_REF master PATCHES config_changes.patch + skip_license_install.patch ) vcpkg_configure_cmake( @@ -17,6 +18,7 @@ vcpkg_configure_cmake( vcpkg_install_cmake() file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright) +file(INSTALL ${SOURCE_PATH}/gnu-lgpl-v2.1.md DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT}) file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include) file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/license) vcpkg_fixup_cmake_targets() diff --git a/ports/qt-advanced-docking-system/skip_license_install.patch b/ports/qt-advanced-docking-system/skip_license_install.patch new file mode 100644 index 00000000000000..e6cfb32a5fac62 --- /dev/null +++ b/ports/qt-advanced-docking-system/skip_license_install.patch @@ -0,0 +1,19 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index a2cb47d..57a1cf5 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -81,14 +81,8 @@ install(FILES ${ads_INSTALL_INCLUDE} + DESTINATION include + COMPONENT headers + ) +-install(FILES +- "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE" +- "${CMAKE_CURRENT_SOURCE_DIR}/gnu-lgpl-v2.1.md" +- DESTINATION license +- COMPONENT license +-) + install(TARGETS qtadvanceddocking + EXPORT qt-advanced-docking-systemConfig + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib From 614e65f7433bdbfd157c82f1bf36f2fb5929ee6c Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 29 Apr 2020 13:44:03 -0700 Subject: [PATCH 4/7] [idevicerestore] Mark as failing because it wants which does not exist on Windows. --- ports/idevicerestore/CONTROL | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ports/idevicerestore/CONTROL b/ports/idevicerestore/CONTROL index ca6cabbd02ed19..326c096f4c306a 100644 --- a/ports/idevicerestore/CONTROL +++ b/ports/idevicerestore/CONTROL @@ -1,4 +1,5 @@ Source: idevicerestore -Version: 1.0.12-3 +Version: 1.0.12-4 Description: Restore/upgrade firmware of iOS devices -Build-Depends: libimobiledevice, curl, libirecovery, libzip \ No newline at end of file +Build-Depends: libimobiledevice, curl, libirecovery, libzip +Supports: !(windows) From 94f2a880930db84c96edf49edabbde645f71c9e2 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 29 Apr 2020 19:33:41 -0700 Subject: [PATCH 5/7] [OpenBLAS] Upgrade to 0.3.9 + a patch to fix AVX512 (which is now present on the VMs) --- ports/openblas/CONTROL | 2 +- ports/openblas/github_2481.patch | 23 +++++++++++++++++++ ports/openblas/portfile.cmake | 5 ++-- .../azure-pipelines/linux/provision-image.sh | 10 ++++---- 4 files changed, 31 insertions(+), 9 deletions(-) create mode 100644 ports/openblas/github_2481.patch diff --git a/ports/openblas/CONTROL b/ports/openblas/CONTROL index 1b1cc5741a96f7..3c2b0b7ebe68d7 100644 --- a/ports/openblas/CONTROL +++ b/ports/openblas/CONTROL @@ -1,5 +1,5 @@ Source: openblas -Version: 0.3.7 +Version: 0.3.9-1 Homepage: https://github.com/xianyi/OpenBLAS Build-Depends: pthread (linux) Description: OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version. diff --git a/ports/openblas/github_2481.patch b/ports/openblas/github_2481.patch new file mode 100644 index 00000000000000..bedffc36cd8ee3 --- /dev/null +++ b/ports/openblas/github_2481.patch @@ -0,0 +1,23 @@ +From 21f6c4b5a972683f7228e5ad446bc940947c2d2b Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=D9=85=D9=87=D8=AF=D9=8A=20=D8=B4=D9=8A=D9=86=D9=88=D9=86?= + =?UTF-8?q?=20=28Mehdi=20Chinoune=29?= +Date: Mon, 2 Mar 2020 17:22:28 +0100 +Subject: [PATCH] fixes #2480 + +--- + cmake/cc.cmake | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/cmake/cc.cmake b/cmake/cc.cmake +index 22217575c..d5551147c 100644 +--- a/cmake/cc.cmake ++++ b/cmake/cc.cmake +@@ -99,7 +99,7 @@ endif () + if (${CORE} STREQUAL "SKYLAKEX") + if (NOT DYNAMIC_ARCH) + if (NOT NO_AVX512) +- set (CCOMMON_OPT = "${CCOMMON_OPT} -march=skylake-avx512") ++ set (CCOMMON_OPT "${CCOMMON_OPT} -march=skylake-avx512") + endif () + endif () + endif () diff --git a/ports/openblas/portfile.cmake b/ports/openblas/portfile.cmake index 17112279ed7b80..2decb588380674 100644 --- a/ports/openblas/portfile.cmake +++ b/ports/openblas/portfile.cmake @@ -3,13 +3,14 @@ include(vcpkg_common_functions) vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO xianyi/OpenBLAS - REF 5f36f18148603facb6c3540e673610d6b24cbfbb # v0.3.7 - SHA512 c1abcfd6fbf6ae3efa28053523ffb3d72aefaa279594347bfd9910c40f25778421b8d1d2d4dc6c934c9def40642942bb74fe76e06c2d96b1898ea61204a95540 + REF v0.3.9 + SHA512 e34da25b3aaf959ec12826ac68c81e739e453d44f2dba28b15e57d7a827edc4d5f42988e9b6d98ac07999940be7b5876246cb3a980e590ae87f77f4c2f12f40a HEAD_REF develop PATCHES uwp.patch fix-space-path.patch fix-redefinition-function.patch + github_2481.patch ) find_program(GIT NAMES git git.cmd) diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh index b281529e1c3047..dbe10cf788afe9 100755 --- a/scripts/azure-pipelines/linux/provision-image.sh +++ b/scripts/azure-pipelines/linux/provision-image.sh @@ -6,26 +6,24 @@ sudo apt -y update sudo apt -y dist-upgrade # Install common build dependencies and partitioning tools -sudo apt -y install at curl unzip tar libxt-dev gperf libxaw7-dev cifs-utils build-essential zip libx11-dev libxi-dev libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev libxcursor-dev yasm nasm libnuma1 libnuma-dev python-six python3-six python-yaml flex libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev xutils-dev dh-autoreconf libgles2-mesa-dev ruby-full pkg-config +sudo apt -y install at curl unzip tar libxt-dev gperf libxaw7-dev cifs-utils build-essential g++ zip libx11-dev libxi-dev libgl1-mesa-dev libglu1-mesa-dev mesa-common-dev libxinerama-dev libxcursor-dev yasm libnuma1 libnuma-dev python-six python3-six python-yaml flex libbison-dev autoconf libudev-dev libncurses5-dev libtool libxrandr-dev xutils-dev dh-autoreconf libgles2-mesa-dev ruby-full pkg-config # Required by qt5-x11extras sudo apt -y install libxkbcommon-dev libxkbcommon-x11-dev # Required by libhdfs3 sudo apt -y install libkrb5-dev -# Required by intel-ipsec -sudo apt -y install nasm -# Install newer version of nasm than the apt package +# Install newer version of nasm than the apt package, required by intel-ipsec mkdir /tmp/nasm cd /tmp/nasm curl -O https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/nasm-2.14.02.tar.gz tar -xf nasm-2.14.02.tar.gz cd nasm-2.14.02/ -./configure --prefix=/usr && make +./configure --prefix=/usr && make -j sudo make install cd ~ # Install the latest Haskell stack -curl -sSL https://get.haskellstack.org/ | sh +curl -sSL https://get.haskellstack.org/ | sudo sh # Install CUDA wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_10.2.89-1_amd64.deb From 771db7130a91d55d40695dd16ae841f521d66a50 Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Wed, 29 Apr 2020 23:44:33 -0700 Subject: [PATCH 6/7] [Ceres] Set LIB_SUFFIX for Ceres to ensure portfile and build agree. --- ports/ceres/portfile.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ports/ceres/portfile.cmake b/ports/ceres/portfile.cmake index 66481fc0934f22..50819162848fec 100644 --- a/ports/ceres/portfile.cmake +++ b/ports/ceres/portfile.cmake @@ -1,5 +1,3 @@ -include(vcpkg_common_functions) - set(MSVC_USE_STATIC_CRT_VALUE OFF) if(VCPKG_CRT_LINKAGE STREQUAL "static") if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") @@ -69,6 +67,7 @@ vcpkg_configure_cmake( -DLAPACK=${LAPACK} -DSUITESPARSE=${SUITESPARSE} -DMSVC_USE_STATIC_CRT=${MSVC_USE_STATIC_CRT_VALUE} + -DLIB_SUFFIX=${LIB_SUFFIX} ) vcpkg_install_cmake() From f39139db626f838cab356932a1eef35ecf624a1b Mon Sep 17 00:00:00 2001 From: Billy Robert O'Neal III Date: Thu, 30 Apr 2020 00:23:48 -0700 Subject: [PATCH 7/7] [glad] Pass path to python explicitly to work on default Ubuntu 18.04 wherein 'python' is python2. --- ports/glad/portfile.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ports/glad/portfile.cmake b/ports/glad/portfile.cmake index 1c14ca77fbfd0c..f94037d26d7bba 100644 --- a/ports/glad/portfile.cmake +++ b/ports/glad/portfile.cmake @@ -10,8 +10,6 @@ vcpkg_from_github( ) vcpkg_find_acquire_program(PYTHON3) -get_filename_component(PYTHON3_DIR "${PYTHON3}" DIRECTORY) -vcpkg_add_to_path("${PYTHON3_DIR}") file(COPY ${CURRENT_INSTALLED_DIR}/include/KHR/khrplatform.h @@ -32,6 +30,7 @@ vcpkg_configure_cmake( -DGLAD_REPRODUCIBLE=ON -DGLAD_SPEC="gl" # {gl,egl,glx,wgl} -DGLAD_PROFILE="compatibility" # {core,compatibility} + -DPYTHON_EXECUTABLE=${PYTHON3} OPTIONS_DEBUG -DGLAD_GENERATOR="c-debug" )