From dab6bfe73e94bbc3c80005679b4b4adfee871a0e Mon Sep 17 00:00:00 2001 From: Sunjay Bhatia Date: Fri, 13 Nov 2020 16:33:31 -0500 Subject: [PATCH 1/5] Bump dependencies in Windows build container LLVM 11 bump may require Bazel 4.0.0 to avoid erronious LIB and INCLUDE path construction, testing whether 3.7.0 can build this toolchain itself, adjusting rules_cc specific to envoy in https://github.com/envoyproxy/envoy/pull/13688 If this fails, we will need to wait till 2nd week December for Bazel 4.0.0 release. Signed-off-by: Sunjay Bhatia Signed-off-by: William A Rowe Jr Co-authored-by: Sunjay Bhatia Co-authored-by: William A Rowe Jr --- build_container/build_container_windows.ps1 | 34 ++++++++++----------- toolchains/regenerate.sh | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/build_container/build_container_windows.ps1 b/build_container/build_container_windows.ps1 index 0bf21a73..e4264968 100644 --- a/build_container/build_container_windows.ps1 +++ b/build_container/build_container_windows.ps1 @@ -75,15 +75,15 @@ mkdir -Force C:\tools # Bazelisk mkdir -Force C:\tools\bazel DownloadAndCheck C:\tools\bazel\bazel.exe ` - https://github.com/bazelbuild/bazelisk/releases/download/v1.5.0/bazelisk-windows-amd64.exe ` - 67149e87d51eb2b34d8b22ee0aa4ae63550919d3f2792f863eaabf9e78826a60 + https://github.com/bazelbuild/bazelisk/releases/download/v1.7.4/bazelisk-windows-amd64.exe ` + ea612bd2e16a793eb9a3d0d5591f08d0a4ff2f4e94540a51a4653f4368b3d019 AddToPath C:\tools\bazel # VS 2019 Build Tools -# Pinned to version 16.7 downloaded on 8/11/2020 via https://aka.ms/vs/16/release/vs_buildtools.exe +# Pinned to version 16.8.1 downloaded on 11/13/2020 via https://aka.ms/vs/16/release/vs_buildtools.exe DownloadAndCheck $env:TEMP\vs_buildtools.exe ` - https://download.visualstudio.microsoft.com/download/pr/e3850c73-59c6-4c05-9db6-a47a74b67daf/dcb113a854b2cb2141755b6a35c4b9aac6f109081bee45649ec2fcc594b0d7a6/vs_BuildTools.exe ` - dcb113a854b2cb2141755b6a35c4b9aac6f109081bee45649ec2fcc594b0d7a6 + https://download.visualstudio.microsoft.com/download/pr/2f4a234d-6e7c-4049-8248-6d9ac0d05c96/ea70cf3199618ad874a552f45262c1b5b06f14a09fcaa0b459746aea0a9ac761/vs_BuildTools.exe ` + ea70cf3199618ad874a552f45262c1b5b06f14a09fcaa0b459746aea0a9ac761 # See: https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2019#c-build-tools # The "Microsoft.VisualStudio.Workload.MSBuildTools" and it's components are added # by the installer and cannot be supressed. @@ -102,38 +102,38 @@ AddToPath (Resolve-Path "C:\Program Files (x86)\Microsoft Visual Studio\2019\Bui # CMake to ensure a 64-bit build of the tool (VS BuildTools ships a 32-bit build) DownloadAndCheck $env:TEMP\cmake.msi ` - https://github.com/Kitware/CMake/releases/download/v3.18.0/cmake-3.18.0-win64-x64.msi ` - 1597eef91b39fe4b34bab506158e34aa3a89490c519c97ac75a7c5d45885e345 + https://github.com/Kitware/CMake/releases/download/v3.18.4/cmake-3.18.4-win64-x64.msi ` + 7e0cf19d4a1ea31de78487306c0126ec6326ab509d95e56dd4d604e9c0b34317 RunAndCheckError "msiexec.exe" @("/i", "$env:TEMP\cmake.msi", "/quiet", "/norestart") $true AddToPath $env:ProgramFiles\CMake\bin # Ninja to ensure a 64-bit build of the tool (VS BuildTools ships a 32-bit build) mkdir -Force C:\tools\ninja DownloadAndCheck $env:TEMP\ninja.zip ` - https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-win.zip ` - 919fd158c16bf135e8a850bb4046ec1ce28a7439ee08b977cd0b7f6b3463d178 + https://github.com/ninja-build/ninja/releases/download/v1.10.1/ninja-win.zip ` + 5d1211ea003ec9760ad7f5d313ebf0b659d4ffafa221187d2b4444bc03714a33 Expand-Archive -Path $env:TEMP\ninja.zip -DestinationPath C:\tools\ninja AddToPath C:\tools\ninja # LLVM to ensure a 64-bit build of the tool (VS BuildTools ships a 32-bit build) DownloadAndCheck $env:TEMP\LLVM-win64.exe ` - https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/LLVM-10.0.0-win64.exe ` - 893f8a12506f8ad29ca464d868fb432fdadd782786a10655b86575fc7fc1a562 + https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/LLVM-11.0.0-win64.exe ` + a773ee3519ecc8d68d91f0ec72ee939cbed8ded483ba8e10899dc19bccba1e22 RunAndCheckError $env:TEMP\LLVM-win64.exe @("/S") $true AddToPath $env:ProgramFiles\LLVM\bin # NASM (preferred by some OSS projects over MS ml.exe, closer to gnu/intel syntax) -$nasmVersion = "2.15.03" +$nasmVersion = "2.15.05" DownloadAndCheck $env:TEMP\nasm-win64.zip ` https://www.nasm.us/pub/nasm/releasebuilds/$nasmVersion/win64/nasm-$nasmVersion-win64.zip ` - e598d1a9c98345f8436750f42d1f7c5d75ba739919eef37cd9ae8406e6a38802 + f5c93c146f52b4f1664fa3ce6579f961a910e869ab0dae431bd871bdd2584ef2 Expand-Archive -Path $env:TEMP\nasm-win64.zip -DestinationPath C:\tools\ AddToPath C:\tools\nasm-$nasmVersion # Python3 (do not install via msys2 or the MS store's flavors, this version follows Win32 semantics) DownloadAndCheck $env:TEMP\python3-installer.exe ` - https://www.python.org/ftp/python/3.8.5/python-3.8.5-amd64.exe ` - cd427c7b17337d7c13761ca20877d2d8be661bd30415ddc17072a31a65a91b64 + https://www.python.org/ftp/python/3.9.0/python-3.9.0-amd64.exe ` + fd2e2c6612d43bb6b213b72fc53f07d73d99059fa72c96e44bde12e7815073ae # python installer needs to be run as an installer with Start-Process RunAndCheckError "$env:TEMP\python3-installer.exe" @("/quiet", "InstallAllUsers=1", "Include_launcher=0", "InstallLauncherAllUsers=0") $true AddToPath $env:ProgramFiles\Python38 @@ -155,8 +155,8 @@ RunAndCheckError "$env:TEMP\7z-installer.exe" @("/S", "/D=$quo$env:TEMP\7z$quo") # msys2 with additional required packages DownloadAndCheck $env:TEMP\msys2.tar.xz ` - https://github.com/msys2/msys2-installer/releases/download/2020-09-03/msys2-base-x86_64-20200903.tar.xz ` - 9d5700a2a8148837bd3869f4940f53d2e46ebfa4148de35c21ca49bf2c239f27 + https://github.com/msys2/msys2-installer/releases/download/2020-11-09/msys2-base-x86_64-20201109.tar.xz ` + ca10a72aa3df219fabeff117aa4b00c1ec700ea93c4febf4cfc03083f4b2cacb RunAndCheckError "$env:TEMP\7z\7z.exe" @("x", "$env:TEMP\msys2.tar.xz", "-o$env:TEMP\msys2.tar", "-y") RunAndCheckError "$env:TEMP\7z\7z.exe" @("x", "$env:TEMP\msys2.tar", "-oC:\tools", "-y") AddToPath C:\tools\msys64\usr\bin diff --git a/toolchains/regenerate.sh b/toolchains/regenerate.sh index 728e970a..3dbadddb 100755 --- a/toolchains/regenerate.sh +++ b/toolchains/regenerate.sh @@ -38,7 +38,7 @@ esac # Bazel query is the right command so bazel won't fail itself. # Keep bazel versions here at most two: current master version, next version -for BAZEL_VERSION in "3.4.1" "3.6.0"; do +for BAZEL_VERSION in "3.7.1" "4.0.0"; do for RBE_BAZEL_TARGET in ${RBE_BAZEL_TARGET_LIST}; do USE_BAZEL_VERSION="${BAZEL_VERSION}" bazel query ${BAZEL_QUERY_OPTIONS} ${RBE_BAZEL_TARGET} done From a2463260f9cf2b2dc090d8bf98275e2fbeeeb4c2 Mon Sep 17 00:00:00 2001 From: Sunjay Bhatia Date: Tue, 17 Nov 2020 13:51:13 -0500 Subject: [PATCH 2/5] Elect 3.6.0 aned 3.7.0 for generation now The anticipated flavors for envoy to build for clang-cl are 3.7.1/4.0.0, these obviously do not exist, yet. Signed-off-by: Sunjay Bhatia Co-authored-by: William A Rowe Jr --- toolchains/regenerate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolchains/regenerate.sh b/toolchains/regenerate.sh index 3dbadddb..9c2cf059 100755 --- a/toolchains/regenerate.sh +++ b/toolchains/regenerate.sh @@ -38,7 +38,7 @@ esac # Bazel query is the right command so bazel won't fail itself. # Keep bazel versions here at most two: current master version, next version -for BAZEL_VERSION in "3.7.1" "4.0.0"; do +for BAZEL_VERSION in "3.6.0" "3.7.0"; do for RBE_BAZEL_TARGET in ${RBE_BAZEL_TARGET_LIST}; do USE_BAZEL_VERSION="${BAZEL_VERSION}" bazel query ${BAZEL_QUERY_OPTIONS} ${RBE_BAZEL_TARGET} done From 0e408b7942aa9230da6edd13e40f6c4c4fd2ae90 Mon Sep 17 00:00:00 2001 From: William A Rowe Jr Date: Tue, 17 Nov 2020 15:27:00 -0500 Subject: [PATCH 3/5] Adjust symlinks and path for Python bump Signed-off-by: William A Rowe Jr --- build_container/build_container_windows.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build_container/build_container_windows.ps1 b/build_container/build_container_windows.ps1 index e4264968..b9c5bcf4 100644 --- a/build_container/build_container_windows.ps1 +++ b/build_container/build_container_windows.ps1 @@ -136,11 +136,11 @@ DownloadAndCheck $env:TEMP\python3-installer.exe ` fd2e2c6612d43bb6b213b72fc53f07d73d99059fa72c96e44bde12e7815073ae # python installer needs to be run as an installer with Start-Process RunAndCheckError "$env:TEMP\python3-installer.exe" @("/quiet", "InstallAllUsers=1", "Include_launcher=0", "InstallLauncherAllUsers=0") $true -AddToPath $env:ProgramFiles\Python38 -AddToPath $env:ProgramFiles\Python38\Scripts +AddToPath $env:ProgramFiles\Python39 +AddToPath $env:ProgramFiles\Python39\Scripts # Add symlinks for canonical executables expected in a Python environment -RunAndCheckError "cmd.exe" @("/c", "mklink", "$env:ProgramFiles\Python38\python3.exe", "$env:ProgramFiles\Python38\python.exe") -RunAndCheckError "cmd.exe" @("/c", "mklink", "$env:ProgramFiles\Python38\python3.8.exe", "$env:ProgramFiles\Python38\python.exe") +RunAndCheckError "cmd.exe" @("/c", "mklink", "$env:ProgramFiles\Python39\python3.exe", "$env:ProgramFiles\Python39\python.exe") +RunAndCheckError "cmd.exe" @("/c", "mklink", "$env:ProgramFiles\Python39\python3.9.exe", "$env:ProgramFiles\Python39\python.exe") # Upgrade pip RunAndCheckError "python.exe" @("-m", "pip", "install", "--upgrade", "pip") # Install wheel so rules_python rules will run From 05da8f6291788250b4f3ec4906c6e6058593e471 Mon Sep 17 00:00:00 2001 From: William A Rowe Jr Date: Tue, 17 Nov 2020 16:50:11 -0500 Subject: [PATCH 4/5] Back down to llvm 10.0.0, as 11 corrupts even msvc-cl builds Signed-off-by: William A Rowe Jr --- build_container/build_container_windows.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_container/build_container_windows.ps1 b/build_container/build_container_windows.ps1 index b9c5bcf4..448f4bac 100644 --- a/build_container/build_container_windows.ps1 +++ b/build_container/build_container_windows.ps1 @@ -117,8 +117,8 @@ AddToPath C:\tools\ninja # LLVM to ensure a 64-bit build of the tool (VS BuildTools ships a 32-bit build) DownloadAndCheck $env:TEMP\LLVM-win64.exe ` - https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/LLVM-11.0.0-win64.exe ` - a773ee3519ecc8d68d91f0ec72ee939cbed8ded483ba8e10899dc19bccba1e22 + https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/LLVM-10.0.0-win64.exe ` + 893f8a12506f8ad29ca464d868fb432fdadd782786a10655b86575fc7fc1a562 RunAndCheckError $env:TEMP\LLVM-win64.exe @("/S") $true AddToPath $env:ProgramFiles\LLVM\bin From 0977c3fd798911c7af591ae1f31b6c5968c2c5cf Mon Sep 17 00:00:00 2001 From: Sunjay Bhatia Date: Wed, 18 Nov 2020 15:20:50 -0500 Subject: [PATCH 5/5] Bump bazelisk for Linux as well Signed-off-by: Sunjay Bhatia --- build_container/build_container_common.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build_container/build_container_common.sh b/build_container/build_container_common.sh index e8902578..be227935 100755 --- a/build_container/build_container_common.sh +++ b/build_container/build_container_common.sh @@ -39,17 +39,17 @@ if [[ "$(uname -m)" == "x86_64" ]]; then chmod +x /usr/local/bin/buildozer # bazelisk - VERSION=1.6.0 + VERSION=1.7.4 download_and_check /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v${VERSION}/bazelisk-linux-amd64 \ - 616f65bcdcfd134a19d5d86c591c35098d7732be26145bf06f02ec9e3e52700c + ab258203db518a54cbd5afa80864d5a3bb366058b95e7a7df4134b0b7765a378 chmod +x /usr/local/bin/bazel fi if [[ "$(uname -m)" == "aarch64" ]]; then # bazelisk - VERSION=1.6.0 + VERSION=1.7.4 download_and_check /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v${VERSION}/bazelisk-linux-arm64 \ - 4a74d233008ec4a59d07fd8ba67cb4acbf7331acb0458aead16f8a4eb1995f9e + aea0ff1036bd4c3703c5c10d07a059d885f2f6ad2f36c2175fc45a1f774ee341 chmod +x /usr/local/bin/bazel fi