diff --git a/ports/chromaprint/fix_lrintf_detection.patch b/ports/chromaprint/fix_lrintf_detection.patch new file mode 100644 index 00000000000000..27248f0c802236 --- /dev/null +++ b/ports/chromaprint/fix_lrintf_detection.patch @@ -0,0 +1,23 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 3a0eecb..4fa5e8d 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -17,6 +17,7 @@ set_property(CACHE FFT_LIB PROPERTY STRINGS avfft fftw3 fftw3f kissfft vdsp) + + include(CMakePushCheckState) + include(CheckFunctionExists) ++include(CheckSymbolExists) + include(CheckCXXCompilerFlag) + + find_package(Threads) +@@ -37,8 +38,8 @@ endif() + + cmake_push_check_state(RESET) + set(CMAKE_REQUIRED_LIBRARIES -lm) +-check_function_exists(lrintf HAVE_LRINTF) +-check_function_exists(round HAVE_ROUND) ++check_symbol_exists(lrintf math.h HAVE_LRINTF) ++check_symbol_exists(round math.h HAVE_ROUND) + cmake_pop_check_state() + + add_definitions( diff --git a/ports/chromaprint/portfile.cmake b/ports/chromaprint/portfile.cmake index da430faa5b67f3..286edaeb38ae4c 100644 --- a/ports/chromaprint/portfile.cmake +++ b/ports/chromaprint/portfile.cmake @@ -3,6 +3,8 @@ vcpkg_from_github( REPO acoustid/chromaprint REF v1.5.0 SHA512 333114949928abdf5d4b11aba1db6ec487eebe526324c68d903b3fa80a3af87a28d942af765a2f873e63a1bf222b658b6438cd10cde4446f61b26ea91f537469 + PATCHES + fix_lrintf_detection.patch # submitted upstream as https://github.com/acoustid/chromaprint/pull/85 ) vcpkg_configure_cmake( diff --git a/ports/chromaprint/vcpkg.json b/ports/chromaprint/vcpkg.json index 4808008c97af67..271532380431c5 100644 --- a/ports/chromaprint/vcpkg.json +++ b/ports/chromaprint/vcpkg.json @@ -1,6 +1,7 @@ { "name": "chromaprint", - "version-string": "1.5.0", + "version": "1.5.0", + "port-version": 1, "description": "C library for generating audio fingerprints used by AcoustID", "homepage": "https://github.com/acoustid/chromaprint", "license": "LGPL-2.1-or-later", diff --git a/ports/cudnn/CONTROL b/ports/cudnn/CONTROL index ebb8143495e477..db910a7bcc68c4 100644 --- a/ports/cudnn/CONTROL +++ b/ports/cudnn/CONTROL @@ -1,6 +1,6 @@ Source: cudnn Version: 7.6.5 -Port-Version: 1 +Port-Version: 2 Description: NVIDIA's cuDNN deep neural network acceleration library Build-Depends: cuda Supports: (windows|linux)&x64 diff --git a/ports/cudnn/portfile.cmake b/ports/cudnn/portfile.cmake index 1c1f0641e560aa..6a008313085ff7 100644 --- a/ports/cudnn/portfile.cmake +++ b/ports/cudnn/portfile.cmake @@ -7,7 +7,6 @@ if(NOT VCPKG_TARGET_ARCHITECTURE STREQUAL "x64") endif() vcpkg_check_linkage(ONLY_DYNAMIC_LIBRARY ONLY_DYNAMIC_CRT) -set(VCPKG_POLICY_MISMATCHED_NUMBER_OF_BINARIES enabled) # only release bits are provided set(MINIMUM_CUDNN_VERSION "7.6.5") @@ -64,74 +63,10 @@ if(CUDNN_HEADER_CONTENTS) endif() if (CUDNN_INCLUDE_DIR AND CUDNN_LIBRARY AND _CUDNN_VERSION VERSION_GREATER_EQUAL MINIMUM_CUDNN_VERSION) - set(CUDNN_FOUND TRUE) -else() - set(CUDNN_FOUND FALSE) -endif() - -# Download CUDNN if not found -if (CUDNN_FOUND) message(STATUS "Found CUDNN ${_CUDNN_VERSION} located on system: (include ${CUDNN_INCLUDE_DIR} lib: ${CUDNN_LIBRARY})") set(VCPKG_POLICY_EMPTY_PACKAGE enabled) -elseif(CUDA_VERSION VERSION_EQUAL "10.1" OR CUDA_VERSION VERSION_EQUAL "10.2") - message(STATUS "CUDNN not found on system - downloading a version compatible with your CUDA v${CUDA_VERSION}...") - if(${CUDA_VERSION} VERSION_EQUAL "10.1") - set(CUDNN_VERSION "7.6.5") - set(CUDNN_VERSION_MAJOR "7") - set(CUDNN_FULL_VERSION "7.6.5-cuda10.1_0") - if(VCPKG_TARGET_IS_WINDOWS) - set(CUDNN_DOWNLOAD_LINK "https://anaconda.org/anaconda/cudnn/${CUDNN_VERSION}/download/win-64/cudnn-${CUDNN_FULL_VERSION}.tar.bz2") - set(SHA512_CUDNN "0fb18dd49de877ad6bae24b53ffe007a99915cc9601697a556897e018cc6d99d3aa68716ea99248cf6a9dfaeeb1a551453c606d04e8bbb3e9315bf768184f15b") - set(CUDNN_OS "windows") - elseif(VCPKG_TARGET_IS_LINUX) - set(CUDNN_DOWNLOAD_LINK "https://anaconda.org/anaconda/cudnn/${CUDNN_VERSION}/download/linux-64/cudnn-${CUDNN_FULL_VERSION}.tar.bz2") - set(SHA512_CUDNN "4d901d96ee8f37e3977240b9e6e6eeecb33848388db953a789be47de8f357d815c3a106ceab04297c4df0d8ed9c2795b2a22304e93cd1e53322307d3f3cd668e") - set(CUDNN_OS "linux") - endif() - elseif(${CUDA_VERSION} VERSION_EQUAL "10.2") - set(CUDNN_VERSION "7.6.5") - set(CUDNN_VERSION_MAJOR "7") - set(CUDNN_FULL_VERSION "7.6.5-cuda10.2_0") - if(VCPKG_TARGET_IS_WINDOWS) - set(CUDNN_DOWNLOAD_LINK "https://anaconda.org/anaconda/cudnn/${CUDNN_VERSION}/download/win-64/cudnn-${CUDNN_FULL_VERSION}.tar.bz2") - set(SHA512_CUDNN "86ca2f5f510d4fbeb37548d0bcab42474a1c4041be2cf96c4964f1e51c3641dc4bf25e8434cd5ff99fac9f53946e5f0e83bd845613144731d136cd60913d4aaa") - set(CUDNN_OS "windows") - elseif(VCPKG_TARGET_IS_LINUX) - set(CUDNN_DOWNLOAD_LINK "https://anaconda.org/anaconda/cudnn/${CUDNN_VERSION}/download/linux-64/cudnn-${CUDNN_FULL_VERSION}.tar.bz2") - set(SHA512_CUDNN "b15b554c2ec004105cec8ee2a99f33fab0f9aed12128522730be4fa6204a5b2dff29e42901b5c4840b5ebf35607e8a54f35eb30525885067165b05dd95aa391b") - set(CUDNN_OS "linux") - endif() - endif() - - vcpkg_download_distfile(ARCHIVE - URLS ${CUDNN_DOWNLOAD_LINK} - FILENAME "cudnn-${CUDNN_FULL_VERSION}-${CUDNN_OS}.tar.bz2" - SHA512 ${SHA512_CUDNN} - ) - - vcpkg_extract_source_archive_ex( - OUT_SOURCE_PATH SOURCE_PATH - ARCHIVE ${ARCHIVE} - NO_REMOVE_ONE_LEVEL - ) - - if(VCPKG_TARGET_IS_WINDOWS) - file(INSTALL "${SOURCE_PATH}/Library/include/cudnn.h" DESTINATION ${CURRENT_PACKAGES_DIR}/include) - file(INSTALL "${SOURCE_PATH}/Library/lib/x64/cudnn.lib" DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - file(INSTALL "${SOURCE_PATH}/Library/bin/cudnn64_${CUDNN_VERSION_MAJOR}.dll" DESTINATION ${CURRENT_PACKAGES_DIR}/bin) - set(LICENSE_PATH "${SOURCE_PATH}/info/LICENSE.txt") - elseif(VCPKG_TARGET_IS_LINUX) - file(INSTALL "${SOURCE_PATH}/include/cudnn.h" DESTINATION ${CURRENT_PACKAGES_DIR}/include) - file(INSTALL "${SOURCE_PATH}/lib/libcudnn.so.${CUDNN_VERSION}" DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - file(INSTALL "${SOURCE_PATH}/lib/libcudnn.so.${CUDNN_VERSION_MAJOR}" DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - file(INSTALL "${SOURCE_PATH}/lib/libcudnn.so" DESTINATION ${CURRENT_PACKAGES_DIR}/lib) - set(LICENSE_PATH "${SOURCE_PATH}/info/licenses/NVIDIA_SLA_cuDNN_Support.txt") - endif() - - file(INSTALL "${LICENSE_PATH}" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright) - file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/FindCUDNN.cmake" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT}) - file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT}) - -else() # CUDNN NOT FOUND AND NOT AUTO-DOWNLOADABLE - message(FATAL_ERROR "Please install manually cuDNN for your CUDA v${CUDA_VERSION}") +elseif(VCPKG_TARGET_IS_WINDOWS) + message(FATAL_ERROR "Please download CUDNN from official sources (such as https://developer.nvidia.com/rdp/cudnn-download ) and extract the zip into your CUDA_TOOLKIT_ROOT (${CUDA_TOOLKIT_ROOT}). (For example: tar.exe -xvf cudnn-11.2-windows-x64-v8.1.1.33.zip --strip 1 --directory \"${CUDA_TOOLKIT_ROOT}\"") +else() + message(FATAL_ERROR "Please install CUDNN using your system package manager (the same way you installed CUDA). For example: apt install libcudnn8-dev.") endif() diff --git a/ports/dartsim/disable_unit_tests_examples_and_tutorials.patch b/ports/dartsim/disable_unit_tests_examples_and_tutorials.patch new file mode 100644 index 00000000000000..08119ce39f457d --- /dev/null +++ b/ports/dartsim/disable_unit_tests_examples_and_tutorials.patch @@ -0,0 +1,20 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index e32147d..5c85186 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -279,6 +279,7 @@ add_subdirectory(dart) + + set(DART_IN_SOURCE_BUILD TRUE) + ++if (0) + if(TARGET dart AND NOT DART_BUILD_DARTPY) + + # Add a "tests" target to build unit tests. +@@ -330,6 +331,7 @@ if(TARGET dart AND NOT DART_BUILD_DARTPY) + endif(DART_VERBOSE) + + endif() ++endif() + + if (DART_BUILD_DARTPY) + add_subdirectory(python) diff --git a/ports/dartsim/portfile.cmake b/ports/dartsim/portfile.cmake index 0af5f8e9aa5c91..c208482f1f44fa 100644 --- a/ports/dartsim/portfile.cmake +++ b/ports/dartsim/portfile.cmake @@ -7,20 +7,23 @@ vcpkg_from_github( REF v6.9.4 SHA512 a9e8712a294286772ee1e3c3899aac7d202f1d3c2b9242ebeaddb2a142787b192d5421a6e3f889dd9ff9ca9e9106b1f893a0c8ab2e1656b04fac6d0be13494ba HEAD_REF release-6.9 - PATCHES 1478.patch - 1497.patch + PATCHES + 1478.patch + 1497.patch + disable_unit_tests_examples_and_tutorials.patch ) vcpkg_configure_cmake( SOURCE_PATH ${SOURCE_PATH} - OPTIONS -DDART_VERBOSE=ON - -DDART_MSVC_DEFAULT_OPTIONS=ON - -DDART_SKIP_DOXYGEN=ON - -DDART_SKIP_FLANN=ON - -DDART_SKIP_IPOPT=ON - -DDART_SKIP_NLOPT=ON - -DDART_SKIP_OPENGL=ON - -DDART_SKIP_pagmo=ON + OPTIONS + -DDART_VERBOSE=ON + -DDART_MSVC_DEFAULT_OPTIONS=ON + -DDART_SKIP_DOXYGEN=ON + -DDART_SKIP_FLANN=ON + -DDART_SKIP_IPOPT=ON + -DDART_SKIP_NLOPT=ON + -DDART_SKIP_OPENGL=ON + -DDART_SKIP_pagmo=ON ) vcpkg_install_cmake() @@ -36,5 +39,4 @@ file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/include) file(REMOVE_RECURSE ${CURRENT_PACKAGES_DIR}/debug/share) # Handle copyright -file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/dartsim RENAME copyright) - +file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright) diff --git a/ports/dartsim/vcpkg.json b/ports/dartsim/vcpkg.json index 0029e6c3c968f3..5f4c00b1865905 100644 --- a/ports/dartsim/vcpkg.json +++ b/ports/dartsim/vcpkg.json @@ -1,6 +1,7 @@ { "name": "dartsim", - "version-string": "6.9.4", + "version": "6.9.4", + "port-version": 1, "description": "Dynamic Animation and Robotics Toolkit", "homepage": "https://dartsim.github.io/", "dependencies": [ diff --git a/ports/libdatachannel/CXX17_ADAPTOR_TYPEDEFS_DEPRECATION_WARNING.patch b/ports/libdatachannel/CXX17_ADAPTOR_TYPEDEFS_DEPRECATION_WARNING.patch new file mode 100644 index 00000000000000..f41945c28af2ba --- /dev/null +++ b/ports/libdatachannel/CXX17_ADAPTOR_TYPEDEFS_DEPRECATION_WARNING.patch @@ -0,0 +1,25 @@ +diff --git a/include/rtc/utils.hpp b/include/rtc/utils.hpp +index f1a82acb..e882042c 100644 +--- a/include/rtc/utils.hpp ++++ b/include/rtc/utils.hpp +@@ -25,6 +25,7 @@ + #include + #include + #include ++#include + + namespace rtc { + +@@ -34,11 +35,10 @@ template overloaded(Ts...) -> overloaded; + // weak_ptr bind helper + template auto weak_bind(F &&f, T *t, Args &&..._args) { + return [bound = std::bind(f, t, _args...), weak_this = t->weak_from_this()](auto &&...args) { +- using result_type = typename decltype(bound)::result_type; + if (auto shared_this = weak_this.lock()) + return bound(args...); + else +- return static_cast(false); ++ return static_cast(false); + }; + } + diff --git a/ports/libdatachannel/portfile.cmake b/ports/libdatachannel/portfile.cmake index 8b1a6c3a9e97ec..563224524f9692 100644 --- a/ports/libdatachannel/portfile.cmake +++ b/ports/libdatachannel/portfile.cmake @@ -1,11 +1,12 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO paullouisageneau/libdatachannel - REF v0.12.0 - SHA512 878d0461255620c21597a6b7132ac0c9d1bcba19e32809cdd007039041eacc1f30a73b1c11c43b8b57a82ca1efeae2090822e6c5e13bd98f5afa086ec147e6e7 + REF 655175d21e58281031c940a94042d5d1fd46efb3 # v 0.12.2 + SHA512 e1e228bf720ef57130fbb9cc33310cebbdbd16c001455cd56e8746b6ee41bac56da5e5a90235e0a826b52711dc3c95b9d9f56d9e406999f9fd384aee2892578d HEAD_REF master PATCHES fix-for-vcpkg.patch + CXX17_ADAPTOR_TYPEDEFS_DEPRECATION_WARNING.patch # submitted upstream as https://github.com/paullouisageneau/libdatachannel/pull/413 ) vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS diff --git a/ports/libdatachannel/vcpkg.json b/ports/libdatachannel/vcpkg.json index 6dddc4dc76c5a3..7ba32db68fba45 100644 --- a/ports/libdatachannel/vcpkg.json +++ b/ports/libdatachannel/vcpkg.json @@ -1,6 +1,6 @@ { "name": "libdatachannel", - "version-semver": "0.12.0", + "version-semver": "0.12.2", "description": "libdatachannel is a standalone implementation of WebRTC Data Channels, WebRTC Media Transport, and WebSockets in C++17 with C bindings for POSIX platforms (including GNU/Linux, Android, and Apple macOS) and Microsoft Windows.", "homepage": "https://github.com/paullouisageneau/libdatachannel", "dependencies": [ diff --git a/ports/libhv/portfile.cmake b/ports/libhv/portfile.cmake index 01c6f83a50d3a9..f54897107b1452 100644 --- a/ports/libhv/portfile.cmake +++ b/ports/libhv/portfile.cmake @@ -14,7 +14,7 @@ string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "dynamic" BUILD_SHARED) vcpkg_configure_cmake( SOURCE_PATH ${SOURCE_PATH} PREFER_NINJA - DISABLE_PARALLEL + DISABLE_PARALLEL_CONFIGURE OPTIONS -DBUILD_EXAMPLES=OFF -DBUILD_UNITTEST=OFF diff --git a/ports/libhv/vcpkg.json b/ports/libhv/vcpkg.json index c6e1a50f73aa27..d16fbeccfb6a52 100644 --- a/ports/libhv/vcpkg.json +++ b/ports/libhv/vcpkg.json @@ -1,7 +1,7 @@ { "name": "libhv", "version": "1.0.0", - "port-version": 1, + "port-version": 2, "description": "Libhv is a C/C++ network library similar to libevent/libuv.", "homepage": "https://github.com/ithewei/libhv", "supports": "!(arm | uwp)" diff --git a/ports/libudis86/CONTROL b/ports/libudis86/CONTROL deleted file mode 100644 index 8e67b89cc4261d..00000000000000 --- a/ports/libudis86/CONTROL +++ /dev/null @@ -1,4 +0,0 @@ -Source: libudis86 -Version: 2018-01-28-56ff6c87 -Homepage: https://github.com/vmt/udis86 -Description: Disassembler Library for x86 and x86-64 diff --git a/ports/libudis86/portfile.cmake b/ports/libudis86/portfile.cmake index a1ae6e539c1e50..42d99d38a1e976 100644 --- a/ports/libudis86/portfile.cmake +++ b/ports/libudis86/portfile.cmake @@ -11,8 +11,8 @@ file(COPY ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt DESTINATION ${SOURCE_PATH}) vcpkg_find_acquire_program(PYTHON2) vcpkg_execute_required_process( - COMMAND ${PYTHON2} ${SOURCE_PATH}/scripts/ud_itab.py ${SOURCE_PATH}/docs/x86/optable.xml ${SOURCE_PATH}/libudis86/ - WORKING_DIRECTORY SOURCE_PATH + COMMAND "${PYTHON2}" "${SOURCE_PATH}/scripts/ud_itab.py" "${SOURCE_PATH}/docs/x86/optable.xml" "${SOURCE_PATH}/libudis86/" + WORKING_DIRECTORY "${SOURCE_PATH}" LOGNAME python-${TARGET_TRIPLET}-generate-sources ) @@ -20,12 +20,12 @@ vcpkg_configure_cmake( SOURCE_PATH ${SOURCE_PATH} PREFER_NINJA OPTIONS_DEBUG - -DDISABLE_INSTALL_HEADERS=ON - -DDISABLE_INSTALL_TOOLS=ON + -DDISABLE_INSTALL_HEADERS=ON + -DDISABLE_INSTALL_TOOLS=ON ) vcpkg_install_cmake() vcpkg_copy_pdbs() vcpkg_copy_tool_dependencies(${CURRENT_PACKAGES_DIR}/tools/libudis86) -file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/libudis86 RENAME copyright) +file(INSTALL ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright) diff --git a/ports/libudis86/vcpkg.json b/ports/libudis86/vcpkg.json new file mode 100644 index 00000000000000..a687a41045a274 --- /dev/null +++ b/ports/libudis86/vcpkg.json @@ -0,0 +1,7 @@ +{ + "name": "libudis86", + "version-string": "2018-01-28-56ff6c87", + "port-version": 1, + "description": "Disassembler Library for x86 and x86-64", + "homepage": "https://github.com/vmt/udis86" +} diff --git a/ports/libvpx/portfile.cmake b/ports/libvpx/portfile.cmake index e888d7fc412f72..2f3f3680d71f00 100644 --- a/ports/libvpx/portfile.cmake +++ b/ports/libvpx/portfile.cmake @@ -99,14 +99,23 @@ if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) ) # note: pdb file names are hardcoded in the lib file, cannot rename + set(LIBVPX_OUTPUT_PREFIX "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}/${LIBVPX_ARCH_DIR}") if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release") - file(INSTALL "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}/${LIBVPX_ARCH_DIR}/Release/vpx.lib" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") - file(INSTALL "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}/${LIBVPX_ARCH_DIR}/Release/vpx/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Release/vpx.lib" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") + if (EXISTS "${LIBVPX_OUTPUT_PREFIX}/Release/vpx.pdb") + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Release/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") + else() + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Release/vpx/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/lib") + endif() endif() if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - file(INSTALL "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}/${LIBVPX_ARCH_DIR}/Debug/vpx.lib" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") - file(INSTALL "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}/${LIBVPX_ARCH_DIR}/Debug/vpx/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Debug/vpx.lib" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") + if (EXISTS "${LIBVPX_OUTPUT_PREFIX}/Debug/vpx.pdb") + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Debug/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") + else() + file(INSTALL "${LIBVPX_OUTPUT_PREFIX}/Debug/vpx/vpx.pdb" DESTINATION "${CURRENT_PACKAGES_DIR}/debug/lib") + endif() endif() if (VCPKG_TARGET_ARCHITECTURE STREQUAL arm64) @@ -130,13 +139,13 @@ else() set(OPTIONS_DEBUG "--enable-debug-libs --enable-debug --prefix=${CURRENT_PACKAGES_DIR}/debug") set(OPTIONS_RELEASE "--prefix=${CURRENT_PACKAGES_DIR}") - + if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") set(OPTIONS "${OPTIONS} --disable-static --enable-shared") else() set(OPTIONS "${OPTIONS} --enable-static --disable-shared") endif() - + if(VCPKG_TARGET_ARCHITECTURE STREQUAL x86) set(LIBVPX_TARGET_ARCH "x86") elseif(VCPKG_TARGET_ARCHITECTURE STREQUAL x64) diff --git a/ports/libvpx/vcpkg.json b/ports/libvpx/vcpkg.json index ce3de03411c26c..f0991561c1233a 100644 --- a/ports/libvpx/vcpkg.json +++ b/ports/libvpx/vcpkg.json @@ -1,7 +1,7 @@ { "name": "libvpx", "version-semver": "1.9.0", - "port-version": 8, + "port-version": 9, "description": "The reference software implementation for the video coding formats VP8 and VP9.", "homepage": "https://github.com/webmproject/libvpx" } diff --git a/ports/tensorflow-common/tensorflow-common.cmake b/ports/tensorflow-common/tensorflow-common.cmake index 18011257e849fb..4746fcda4f9eca 100644 --- a/ports/tensorflow-common/tensorflow-common.cmake +++ b/ports/tensorflow-common/tensorflow-common.cmake @@ -45,7 +45,7 @@ set(ENV{PYTHON_LIB_PATH} "${PYTHON_LIB_PATH}") vcpkg_execute_required_process(COMMAND ${PYTHON3} -c "import numpy" WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-numpy-${TARGET_TRIPLET}) # tensorflow has long file names, which will not work on windows -set(ENV{TEST_TMPDIR} ${BUILDTREES_DIR}/.bzl) +set(ENV{TEST_TMPDIR} "${CURRENT_BUILDTREES_DIR}/.bzl") set(ENV{USE_DEFAULT_PYTHON_LIB_PATH} 1) set(ENV{TF_NEED_KAFKA} 0) @@ -439,4 +439,4 @@ else() endif() endif() -message(STATUS "You may want to delete ${CURRENT_BUILDTREES_DIR} and ${BUILDTREES_DIR}/.bzl to free diskspace.") +message(STATUS "You may want to delete ${CURRENT_BUILDTREES_DIR} to free diskspace.") diff --git a/ports/tensorflow-common/vcpkg.json b/ports/tensorflow-common/vcpkg.json index 7477461b330107..447d665b583b7e 100644 --- a/ports/tensorflow-common/vcpkg.json +++ b/ports/tensorflow-common/vcpkg.json @@ -1,6 +1,7 @@ { "name": "tensorflow-common", "version-semver": "2.4.1", + "port-version": 1, "description": "This meta package holds common files for the C [tensorflow] and the C++ [tensorflow-cc] API version of TensorFlow but is not installable on its own.", "homepage": "https://github.com/tensorflow/tensorflow" } diff --git a/ports/upb/fix-uwp.patch b/ports/upb/fix-uwp.patch index cbd62f602098e6..c681fa336a7a88 100644 --- a/ports/upb/fix-uwp.patch +++ b/ports/upb/fix-uwp.patch @@ -7,7 +7,7 @@ index a6b432f..003615e 100644 endif() +if (MSVC) -+ add_compile_options(/wd4146 -D_CRT_SECURE_NO_WARNINGS) ++ add_compile_options(/wd4146 /wd4703 -D_CRT_SECURE_NO_WARNINGS) +endif() + enable_testing() diff --git a/ports/upb/vcpkg.json b/ports/upb/vcpkg.json index 3213d796fc5754..058833064dd99f 100644 --- a/ports/upb/vcpkg.json +++ b/ports/upb/vcpkg.json @@ -1,6 +1,7 @@ { "name": "upb", "version-date": "2020-12-19", + "port-version": 1, "description": "μpb (often written 'upb') is a small protobuf implementation written in C.", "homepage": "https://github.com/protocolbuffers/upb/" } diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index 27db6a93d9a5c4..9f6526cb795ea6 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -2,8 +2,8 @@ # SPDX-License-Identifier: MIT # variables: - windows-pool: 'PrWin-2020-12-17' - linux-pool: 'PrLin-2020-12-16' + windows-pool: 'PrWin-2021-04-23' + linux-pool: 'PrLin-2021-04-25' osx-pool: 'PrOsx-2020-09-28' stages: diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/scripts/azure-pipelines/linux/provision-image.sh index c04f4b3caac191..6663baed838808 100755 --- a/scripts/azure-pipelines/linux/provision-image.sh +++ b/scripts/azure-pipelines/linux/provision-image.sh @@ -58,14 +58,13 @@ cd ~ curl -sSL https://get.haskellstack.org/ | sudo sh # Install CUDA -wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_10.2.89-1_amd64.deb -sudo apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub -sudo dpkg -i cuda-repo-ubuntu1804_10.2.89-1_amd64.deb -wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb -sudo dpkg -i nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb +wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-ubuntu1804.pin +sudo mv cuda-ubuntu1804.pin /etc/apt/preferences.d/cuda-repository-pin-600 +sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub +sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/ /" sudo apt -y update -sudo apt install -y --no-install-recommends cuda-compiler-10-2 cuda-libraries-dev-10-2 cuda-driver-dev-10-2 cuda-cudart-dev-10-2 libcublas10 cuda-curand-dev-10-2 -sudo apt install -y --no-install-recommends libcudnn7-dev +sudo apt install -y --no-install-recommends cuda-compiler-11-3 cuda-libraries-dev-11-3 cuda-driver-dev-11-3 \ + cuda-cudart-dev-11-3 libcublas-11-3 libcurand-dev-11-3 libcudnn8-dev libnccl2 libnccl-dev # Install PowerShell wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb diff --git a/scripts/azure-pipelines/windows/create-vmss.ps1 b/scripts/azure-pipelines/windows/create-vmss.ps1 index 608465100f6dfe..fddba0eec73af4 100644 --- a/scripts/azure-pipelines/windows/create-vmss.ps1 +++ b/scripts/azure-pipelines/windows/create-vmss.ps1 @@ -21,11 +21,18 @@ the compiler rather than for testing vcpkg. Differences: * The machine prefix is changed to VcpkgUnstable instead of PrWin. * No storage account or "archives" share is provisioned. * The firewall is not opened to allow communication with Azure Storage. + +.PARAMETER CudnnPath +The path to a CUDNN zip file downloaded from NVidia official sources +(e.g. https://developer.nvidia.com/compute/machine-learning/cudnn/secure/8.1.1.33/11.2_20210301/cudnn-11.2-windows-x64-v8.1.1.33.zip +downloaded in a browser with an NVidia account logged in.) #> [CmdLetBinding()] Param( - [switch]$Unstable = $false + [switch]$Unstable = $false, + [parameter(Mandatory=$true)] + [string]$CudnnPath ) $Location = 'westus2' @@ -43,15 +50,16 @@ $WindowsServerSku = '2019-Datacenter' $ErrorActionPreference = 'Stop' $ProgressActivity = 'Creating Scale Set' -$TotalProgress = 12 -if ($Unstable) { - $TotalProgress -= 1 # skipping the archives share part -} - +$TotalProgress = 18 $CurrentProgress = 1 Import-Module "$PSScriptRoot/../create-vmss-helpers.psm1" -DisableNameChecking +if (-Not $CudnnPath.EndsWith('.zip')) { + Write-Error 'Expected CudnnPath to be a zip file.' + return +} + #################################################################################################### Write-Progress ` -Activity $ProgressActivity ` @@ -155,57 +163,84 @@ $VirtualNetwork = New-AzVirtualNetwork ` -Subnet $Subnet #################################################################################################### -if (-Not $Unstable) { - Write-Progress ` - -Activity $ProgressActivity ` - -Status 'Creating archives storage account' ` - -PercentComplete (100 / $TotalProgress * $CurrentProgress++) +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Creating storage account' ` + -CurrentOperation 'Initial setup' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) - $StorageAccountName = Sanitize-Name $ResourceGroupName +$StorageAccountName = Sanitize-Name $ResourceGroupName - New-AzStorageAccount ` - -ResourceGroupName $ResourceGroupName ` - -Location $Location ` - -Name $StorageAccountName ` - -SkuName 'Standard_LRS' ` - -Kind StorageV2 +New-AzStorageAccount ` + -ResourceGroupName $ResourceGroupName ` + -Location $Location ` + -Name $StorageAccountName ` + -SkuName 'Standard_LRS' ` + -Kind StorageV2 - $StorageAccountKeys = Get-AzStorageAccountKey ` - -ResourceGroupName $ResourceGroupName ` - -Name $StorageAccountName +$StorageAccountKeys = Get-AzStorageAccountKey ` + -ResourceGroupName $ResourceGroupName ` + -Name $StorageAccountName - $StorageAccountKey = $StorageAccountKeys[0].Value +$StorageAccountKey = $StorageAccountKeys[0].Value - $StorageContext = New-AzStorageContext ` - -StorageAccountName $StorageAccountName ` - -StorageAccountKey $StorageAccountKey +$StorageContext = New-AzStorageContext ` + -StorageAccountName $StorageAccountName ` + -StorageAccountKey $StorageAccountKey - New-AzStorageContainer -Name archives -Context $StorageContext -Permission Off - $StartTime = [DateTime]::Now - $ExpiryTime = $StartTime.AddMonths(6) +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Creating storage account' ` + -CurrentOperation 'Uploading cudnn.zip' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress) # note no ++ - $SasToken = New-AzStorageAccountSASToken ` - -Service Blob ` - -Permission "racwdlup" ` - -Context $StorageContext ` - -StartTime $StartTime ` - -ExpiryTime $ExpiryTime ` - -ResourceType Service,Container,Object ` - -Protocol HttpsOnly +New-AzStorageContainer -Name setup -Context $storageContext -Permission blob - $SasToken = $SasToken.Substring(1) # strip leading ? +Set-AzStorageBlobContent -File $CudnnPath ` + -Container 'setup' ` + -Blob 'cudnn.zip' ` + -Context $StorageContext - # Note that we put the storage account into the firewall after creating the above SAS token or we - # would be denied since the person running this script isn't one of the VMs we're creating here. - Set-AzStorageAccount ` - -ResourceGroupName $ResourceGroupName ` - -AccountName $StorageAccountName ` - -NetworkRuleSet ( ` - @{bypass="AzureServices"; ` - virtualNetworkRules=( ` - @{VirtualNetworkResourceId=$VirtualNetwork.Subnets[0].Id;Action="allow"}); ` - defaultAction="Deny"}) -} +$CudnnBlobUrl = "https://$StorageAccountName.blob.core.windows.net/setup/cudnn.zip" + +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Creating storage account' ` + -CurrentOperation 'Creating archives container' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress) # note no ++ + +New-AzStorageContainer -Name archives -Context $StorageContext -Permission Off + +$StartTime = [DateTime]::Now +$ExpiryTime = $StartTime.AddMonths(6) + +$SasToken = New-AzStorageAccountSASToken ` + -Service Blob ` + -Permission "racwdlup" ` + -Context $StorageContext ` + -StartTime $StartTime ` + -ExpiryTime $ExpiryTime ` + -ResourceType Service,Container,Object ` + -Protocol HttpsOnly + +$SasToken = $SasToken.Substring(1) # strip leading ? + +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Creating storage account' ` + -CurrentOperation 'Locking down network' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress) # note no ++ + +# Note that we put the storage account into the firewall after creating the above SAS token or we +# would be denied since the person running this script isn't one of the VMs we're creating here. +Set-AzStorageAccount ` + -ResourceGroupName $ResourceGroupName ` + -AccountName $StorageAccountName ` + -NetworkRuleSet ( ` + @{bypass="AzureServices"; ` + virtualNetworkRules=( ` + @{VirtualNetworkResourceId=$VirtualNetwork.Subnets[0].Id;Action="allow"}); ` + defaultAction="Deny"}) #################################################################################################### Write-Progress ` @@ -245,30 +280,128 @@ New-AzVm ` #################################################################################################### Write-Progress ` -Activity $ProgressActivity ` - -Status 'Running provisioning script provision-image.txt (as a .ps1) in VM' ` + -Status 'Running provisioning script deploy-psexec.ps1 in VM' ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) -$provisionParameters = @{AdminUserPassword = $AdminPW;} -if (-Not $Unstable) { - $provisionParameters['StorageAccountName'] = $StorageAccountName - $provisionParameters['StorageAccountSasToken'] = $SasToken +$DeployPsExecResult = Invoke-AzVMRunCommand ` + -ResourceGroupName $ResourceGroupName ` + -VMName $ProtoVMName ` + -CommandId 'RunPowerShellScript' ` + -ScriptPath "$PSScriptRoot\deploy-psexec.ps1" + +Write-Host "deploy-psexec.ps1 output: $($DeployPsExecResult.value.Message)" + +#################################################################################################### +function Invoke-ScriptWithPrefix { + param( + [string]$ScriptName, + [switch]$AddAdminPw, + [switch]$AddCudnnUrl + ) + + Write-Progress ` + -Activity $ProgressActivity ` + -Status "Running provisioning script $ScriptName in VM" ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) + + $DropToAdminUserPrefix = Get-Content "$PSScriptRoot\drop-to-admin-user-prefix.ps1" -Encoding utf8NoBOM -Raw + $UtilityPrefixContent = Get-Content "$PSScriptRoot\utility-prefix.ps1" -Encoding utf8NoBOM -Raw + + $tempScriptFilename = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".txt" + try { + $script = Get-Content "$PSScriptRoot\$ScriptName" -Encoding utf8NoBOM -Raw + if ($AddAdminPw) { + $script = $script.Replace('# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1', $DropToAdminUserPrefix) + } + + if ($AddCudnnUrl) { + $script = $script.Replace('# REPLACE WITH $CudnnUrl', "`$CudnnUrl = '$CudnnBlobUrl'") + } + + $script = $script.Replace('# REPLACE WITH UTILITY-PREFIX.ps1', $UtilityPrefixContent); + Set-Content -Path $tempScriptFilename -Value $script -Encoding utf8NoBOM + + $parameter = $null + if ($AddAdminPw) { + $parameter = @{AdminUserPassword = $AdminPW;} + } + + $InvokeResult = Invoke-AzVMRunCommand ` + -ResourceGroupName $ResourceGroupName ` + -VMName $ProtoVMName ` + -CommandId 'RunPowerShellScript' ` + -ScriptPath $tempScriptFilename ` + -Parameter $parameter + + Write-Host "$ScriptName output: $($InvokeResult.value.Message)" + } finally { + Remove-Item $tempScriptFilename -Force + } } +Invoke-ScriptWithPrefix -ScriptName 'deploy-visual-studio.ps1' -AddAdminPw +Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName + +#################################################################################################### +Invoke-ScriptWithPrefix -ScriptName 'deploy-windows-wdk.ps1' -AddAdminPw +Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName + +#################################################################################################### +Invoke-ScriptWithPrefix -ScriptName 'deploy-mpi.ps1' -AddAdminPw +Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName + +#################################################################################################### +Invoke-ScriptWithPrefix -ScriptName 'deploy-cuda.ps1' -AddAdminPw -AddCudnnUrl +Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName + +#################################################################################################### +Invoke-ScriptWithPrefix -ScriptName 'deploy-pwsh.ps1' -AddAdminPw +Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName + +#################################################################################################### +Write-Progress ` + -Activity $ProgressActivity ` + -Status 'Running provisioning script deploy-settings.txt (as a .ps1) in VM' ` + -PercentComplete (100 / $TotalProgress * $CurrentProgress++) + $ProvisionImageResult = Invoke-AzVMRunCommand ` -ResourceGroupName $ResourceGroupName ` -VMName $ProtoVMName ` -CommandId 'RunPowerShellScript' ` - -ScriptPath "$PSScriptRoot\provision-image.txt" ` - -Parameter $provisionParameters + -ScriptPath "$PSScriptRoot\deploy-settings.txt" -Write-Host "provision-image.ps1 output: $($ProvisionImageResult.value.Message)" +Write-Host "deploy-settings.txt output: $($ProvisionImageResult.value.Message)" #################################################################################################### Write-Progress ` -Activity $ProgressActivity ` - -Status 'Restarting VM' ` + -Status 'Deploying SAS token into VM' ` -PercentComplete (100 / $TotalProgress * $CurrentProgress++) +$tempScriptFilename = [System.IO.Path]::GetTempPath() + [System.IO.Path]::GetRandomFileName() + ".txt" +try { + $script = "Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' " ` + + "-Name PROVISIONED_AZURE_STORAGE_NAME " ` + + "-Value '$StorageAccountName'`r`n" ` + + "Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' " ` + + "-Name PROVISIONED_AZURE_STORAGE_SAS_TOKEN " ` + + "-Value '$SasToken'`r`n" + + Write-Host "Script content is:" + Write-Host $script + + Set-Content -Path $tempScriptFilename -Value $script -Encoding utf8NoBOM + $InvokeResult = Invoke-AzVMRunCommand ` + -ResourceGroupName $ResourceGroupName ` + -VMName $ProtoVMName ` + -CommandId 'RunPowerShellScript' ` + -ScriptPath $tempScriptFilename + + Write-Host "Deploy SAS token output: $($InvokeResult.value.Message)" +} finally { + Remove-Item $tempScriptFilename -Force +} + Restart-AzVM -ResourceGroupName $ResourceGroupName -Name $ProtoVMName #################################################################################################### diff --git a/scripts/azure-pipelines/windows/deploy-cuda.ps1 b/scripts/azure-pipelines/windows/deploy-cuda.ps1 new file mode 100644 index 00000000000000..ab1cc1c54bd371 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-cuda.ps1 @@ -0,0 +1,62 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1 + +# REPLACE WITH UTILITY-PREFIX.ps1 + +# REPLACE WITH $CudnnUrl + +$CudnnLocalZipPath = "$PSScriptRoot\cudnn-11.2-windows-x64-v8.1.1.33.zip" + +$CudaUrl = 'https://developer.download.nvidia.com/compute/cuda/11.3.0/network_installers/cuda_11.3.0_win10_network.exe' + +$CudaFeatures = 'nvcc_11.3 cuobjdump_11.3 nvprune_11.3 cupti_11.3 memcheck_11.3 nvdisasm_11.3 nvprof_11.3 ' + ` + 'visual_studio_integration_11.3 visual_profiler_11.3 visual_profiler_11.3 cublas_11.3 cublas_dev_11.3 ' + ` + 'cudart_11.3 cufft_11.3 cufft_dev_11.3 curand_11.3 curand_dev_11.3 cusolver_11.3 cusolver_dev_11.3 ' + ` + 'cusparse_11.3 cusparse_dev_11.3 npp_11.3 npp_dev_11.3 nvrtc_11.3 nvrtc_dev_11.3 nvml_dev_11.3 ' + ` + 'occupancy_calculator_11.3 thrust_11.3 ' + +$destination = "$env:ProgramFiles\NVIDIA GPU Computing Toolkit\CUDA\v11.3" + +try { + Write-Host 'Downloading CUDA...' + [string]$installerPath = Get-TempFilePath -Extension 'exe' + curl.exe -L -o $installerPath -s -S $CudaUrl + Write-Host 'Installing CUDA...' + $proc = Start-Process -FilePath $installerPath -ArgumentList @('-s ' + $CudaFeatures) -Wait -PassThru + $exitCode = $proc.ExitCode + if ($exitCode -eq 0) { + Write-Host 'Installation successful!' + } + else { + Write-Error "Installation failed! Exited with $exitCode." + throw + } +} +catch { + Write-Error "Failed to install CUDA! $($_.Exception.Message)" + throw +} + +try { + if ([string]::IsNullOrWhiteSpace($CudnnUrl)) { + if (-Not (Test-Path $CudnnLocalZipPath)) { + throw "CUDNN zip ($CudnnLocalZipPath) was missing, please download from NVidia and place next to this script." + } + + $cudnnZipPath = $CudnnLocalZipPath + } else { + Write-Host 'Downloading CUDNN...' + $cudnnZipPath = Get-TempFilePath -Extension 'zip' + curl.exe -L -o $cudnnZipPath -s -S $CudnnUrl + } + + Write-Host "Installing CUDNN to $destination..." + tar.exe -xvf "$cudnnZipPath" --strip 1 --directory "$destination" + Write-Host 'Installation successful!' +} +catch { + Write-Error "Failed to install CUDNN! $($_.Exception.Message)" + throw +} diff --git a/scripts/azure-pipelines/windows/deploy-mpi.ps1 b/scripts/azure-pipelines/windows/deploy-mpi.ps1 new file mode 100644 index 00000000000000..ba369f7971d733 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-mpi.ps1 @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1 + +# REPLACE WITH UTILITY-PREFIX.ps1 + +$MpiUrl = 'https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisetup.exe' + +<# +.SYNOPSIS +Installs MPI + +.DESCRIPTION +Downloads the MPI installer located at $Url, and installs it with the +correct flags. + +.PARAMETER Url +The URL of the installer. +#> +Function InstallMpi { + Param( + [String]$Url + ) + + try { + Write-Host 'Downloading MPI...' + [string]$installerPath = Get-TempFilePath -Extension 'exe' + curl.exe -L -o $installerPath -s -S $Url + Write-Host 'Installing MPI...' + $proc = Start-Process -FilePath $installerPath -ArgumentList @('-force', '-unattend') -Wait -PassThru + $exitCode = $proc.ExitCode + if ($exitCode -eq 0) { + Write-Host 'Installation successful!' + } + else { + Write-Error "Installation failed! Exited with $exitCode." + throw + } + } + catch { + Write-Error "Failed to install MPI! $($_.Exception.Message)" + throw + } +} + +InstallMpi -Url $MpiUrl diff --git a/scripts/azure-pipelines/windows/deploy-psexec.ps1 b/scripts/azure-pipelines/windows/deploy-psexec.ps1 new file mode 100644 index 00000000000000..2a1f6fb14a87e4 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-psexec.ps1 @@ -0,0 +1,8 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' +$PsExecPath = 'C:\PsExec64.exe' +Write-Host "Downloading psexec to: $PsExecPath" +& curl.exe -L -o $PsExecPath -s -S https://live.sysinternals.com/PsExec64.exe diff --git a/scripts/azure-pipelines/windows/deploy-pwsh.ps1 b/scripts/azure-pipelines/windows/deploy-pwsh.ps1 new file mode 100644 index 00000000000000..b766385a308511 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-pwsh.ps1 @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1 + +# REPLACE WITH UTILITY-PREFIX.ps1 + +$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.1.3/PowerShell-7.1.3-win-x64.msi' +InstallMSI -Url $PwshUrl -Name 'PowerShell Core' diff --git a/scripts/azure-pipelines/windows/deploy-settings.txt b/scripts/azure-pipelines/windows/deploy-settings.txt new file mode 100644 index 00000000000000..a80ddce399a281 --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-settings.txt @@ -0,0 +1,20 @@ +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' + +Write-Host 'Disabling pagefile...' +wmic computersystem set AutomaticManagedPagefile=False +wmic pagefileset delete + +$av = Get-Command Add-MPPreference -ErrorAction SilentlyContinue +if ($null -eq $av) { + Write-Host 'AntiVirus not installed, skipping exclusions.' +} else { + Write-Host 'Configuring AntiVirus exclusions...' + Add-MpPreference -ExclusionPath C:\agent + Add-MPPreference -ExclusionPath D:\ + Add-MPPreference -ExclusionProcess ninja.exe + Add-MPPreference -ExclusionProcess clang-cl.exe + Add-MPPreference -ExclusionProcess cl.exe + Add-MPPreference -ExclusionProcess link.exe + Add-MPPreference -ExclusionProcess python.exe +} diff --git a/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 b/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 new file mode 100644 index 00000000000000..2de2de919556ca --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 @@ -0,0 +1,86 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1 + +# REPLACE WITH UTILITY-PREFIX.ps1 + +$VisualStudioBootstrapperUrl = 'https://aka.ms/vs/16/release/vs_enterprise.exe' +$Workloads = @( + 'Microsoft.VisualStudio.Workload.NativeDesktop', + 'Microsoft.VisualStudio.Workload.Universal', + 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', + 'Microsoft.VisualStudio.Component.VC.Tools.ARM', + 'Microsoft.VisualStudio.Component.VC.Tools.ARM64', + 'Microsoft.VisualStudio.Component.VC.ATL', + 'Microsoft.VisualStudio.Component.VC.ATLMFC', + 'Microsoft.VisualStudio.Component.VC.v141.x86.x64.Spectre', + 'Microsoft.VisualStudio.Component.Windows10SDK.18362', + 'Microsoft.VisualStudio.Component.Windows10SDK.19041', + 'Microsoft.Net.Component.4.8.SDK', + 'Microsoft.Component.NetFX.Native', + 'Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset', + 'Microsoft.VisualStudio.Component.VC.Llvm.Clang', + 'Microsoft.VisualStudio.Component.VC.v141.x86.x64', + 'Microsoft.VisualStudio.Component.VC.140' +) + +<# +.SYNOPSIS +Install Visual Studio. + +.DESCRIPTION +InstallVisualStudio takes the $Workloads array, and installs it with the +installer that's pointed at by $BootstrapperUrl. + +.PARAMETER Workloads +The set of VS workloads to install. + +.PARAMETER BootstrapperUrl +The URL of the Visual Studio installer, i.e. one of vs_*.exe. + +.PARAMETER InstallPath +The path to install Visual Studio at. + +.PARAMETER Nickname +The nickname to give the installation. +#> +Function InstallVisualStudio { + Param( + [String[]]$Workloads, + [String]$BootstrapperUrl, + [String]$InstallPath = $null, + [String]$Nickname = $null + ) + + try { + Write-Host 'Downloading Visual Studio...' + [string]$bootstrapperExe = Get-TempFilePath -Extension 'exe' + curl.exe -L -o $bootstrapperExe -s -S $BootstrapperUrl + Write-Host 'Installing Visual Studio...' + $vsArgs = @('/c', $bootstrapperExe, '--quiet', '--norestart', '--wait', '--nocache') + foreach ($workload in $Workloads) { + $vsArgs += '--add' + $vsArgs += $workload + } + + if (-not ([String]::IsNullOrWhiteSpace($InstallPath))) { + $vsArgs += '--installpath' + $vsArgs += $InstallPath + } + + if (-not ([String]::IsNullOrWhiteSpace($Nickname))) { + $vsArgs += '--nickname' + $vsArgs += $Nickname + } + + $proc = Start-Process -FilePath cmd.exe -ArgumentList $vsArgs -Wait -PassThru + PrintMsiExitCodeMessage $proc.ExitCode + } + catch { + Write-Error "Failed to install Visual Studio! $($_.Exception.Message)" + throw + } +} + +InstallVisualStudio -Workloads $Workloads -BootstrapperUrl $VisualStudioBootstrapperUrl -Nickname 'Stable' diff --git a/scripts/azure-pipelines/windows/deploy-windows-wdk.ps1 b/scripts/azure-pipelines/windows/deploy-windows-wdk.ps1 new file mode 100644 index 00000000000000..d145f79d71a8ec --- /dev/null +++ b/scripts/azure-pipelines/windows/deploy-windows-wdk.ps1 @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +# REPLACE WITH DROP-TO-ADMIN-USER-PREFIX.ps1 + +# REPLACE WITH UTILITY-PREFIX.ps1 + +$WindowsWDKUrl = 'https://go.microsoft.com/fwlink/?linkid=2128854' + +<# +.SYNOPSIS +Installs Windows WDK version 2004 + +.DESCRIPTION +Downloads the Windows WDK installer located at $Url, and installs it with the +correct flags. + +.PARAMETER Url +The URL of the installer. +#> +Function InstallWindowsWDK { + Param( + [String]$Url + ) + + try { + Write-Host 'Downloading Windows WDK...' + [string]$installerPath = Get-TempFilePath -Extension 'exe' + curl.exe -L -o $installerPath -s -S $Url + Write-Host 'Installing Windows WDK...' + $proc = Start-Process -FilePath $installerPath -ArgumentList @('/features', '+', '/q') -Wait -PassThru + $exitCode = $proc.ExitCode + if ($exitCode -eq 0) { + Write-Host 'Installation successful!' + } + else { + Write-Error "Installation failed! Exited with $exitCode." + throw + } + } + catch { + Write-Error "Failed to install Windows WDK! $($_.Exception.Message)" + throw + } +} + +InstallWindowsWDK -Url $WindowsWDKUrl diff --git a/scripts/azure-pipelines/windows/drop-to-admin-user-prefix.ps1 b/scripts/azure-pipelines/windows/drop-to-admin-user-prefix.ps1 new file mode 100644 index 00000000000000..b4592eabd4eee4 --- /dev/null +++ b/scripts/azure-pipelines/windows/drop-to-admin-user-prefix.ps1 @@ -0,0 +1,27 @@ +param( + [string]$AdminUserPassword = $null +) + +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' +if (-Not [string]::IsNullOrEmpty($AdminUserPassword)) { + $PsExecPath = 'C:\PsExec64.exe' + $PsExecArgs = @( + '-u', + 'AdminUser', + '-p', + $AdminUserPassword, + '-accepteula', + '-i', + '-h', + 'C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe', + '-ExecutionPolicy', + 'Unrestricted', + '-File', + $PSCommandPath + ) + + Write-Host "Executing: $PsExecPath $PsExecArgs" + $proc = Start-Process -FilePath $PsExecPath -ArgumentList $PsExecArgs -Wait -PassThru + exit $proc.ExitCode +} diff --git a/scripts/azure-pipelines/windows/provision-entire-image.ps1 b/scripts/azure-pipelines/windows/provision-entire-image.ps1 new file mode 100644 index 00000000000000..3bfb5dbd9e2a69 --- /dev/null +++ b/scripts/azure-pipelines/windows/provision-entire-image.ps1 @@ -0,0 +1,16 @@ +# This script runs all the scripts we run on Azure machines to deploy prerequisites, +# and assumes it is being run as an admin user. + +. "$PSScriptRoot\utility-prefix.ps1" + +. "$PSScriptRoot\deploy-visual-studio.ps1" +. "$PSScriptRoot\deploy-windows-wdk.ps1" +. "$PSScriptRoot\deploy-mpi.ps1" +. "$PSScriptRoot\deploy-cuda.ps1" +. "$PSScriptRoot\deploy-pwsh.ps1" +try { + Copy-Item "$PSScriptRoot\deploy-settings.txt" "$PSScriptRoot\deploy-settings.ps1" + . "$PSScriptRoot\deploy-settings.ps1" +} finally { + Remove-Item "$PSScriptRoot\deploy-settings.ps1" +} diff --git a/scripts/azure-pipelines/windows/provision-image.txt b/scripts/azure-pipelines/windows/provision-image.txt deleted file mode 100644 index e3e04be28e0df6..00000000000000 --- a/scripts/azure-pipelines/windows/provision-image.txt +++ /dev/null @@ -1,454 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: MIT - -<# -.SYNOPSIS -Sets up a machine to be an image for a scale set. - -.DESCRIPTION -provision-image.ps1 runs on an existing, freshly provisioned virtual machine, -and sets up that virtual machine as a build machine. After this is done, -(outside of this script), we take that machine and make it an image to be copied -for setting up new VMs in the scale set. - -This script must either be run as admin, or one must pass AdminUserPassword; -if the script is run with AdminUserPassword, it runs itself again as an -administrator. - -.PARAMETER AdminUserPassword -The administrator user's password; if this is $null, or not passed, then the -script assumes it's running on an administrator account. - -.PARAMETER StorageAccountName -The name of the storage account. Stored in the environment variable %PROVISIONED_AZURE_STORAGE_NAME%. -Used by the CI system to access the global storage. - -.PARAMETER StorageAccountSasToken -The SAS token to access the storage account. Stored in the environment variable -%PROVISIONED_AZURE_STORAGE_SAS_TOKEN%. -Used by the CI system to access the global storage. -#> -param( - [string]$AdminUserPassword = $null, - [string]$StorageAccountName = $null, - [string]$StorageAccountSasToken = $null -) - -$ErrorActionPreference = 'Stop' - -<# -.SYNOPSIS -Gets a random file path in the temp directory. - -.DESCRIPTION -Get-TempFilePath takes an extension, and returns a path with a random -filename component in the temporary directory with that extension. - -.PARAMETER Extension -The extension to use for the path. -#> -Function Get-TempFilePath { - Param( - [String]$Extension - ) - - if ([String]::IsNullOrWhiteSpace($Extension)) { - throw 'Missing Extension' - } - - $tempPath = [System.IO.Path]::GetTempPath() - $tempName = [System.IO.Path]::GetRandomFileName() + '.' + $Extension - return Join-Path $tempPath $tempName -} - -$TranscriptPath = 'C:\provision-image-transcript.txt' - -if ([string]::IsNullOrEmpty($AdminUserPassword)) { - Start-Transcript -Path $TranscriptPath -} else { - Write-Host 'AdminUser password supplied; switching to AdminUser.' - $PsExecPath = Get-TempFilePath -Extension 'exe' - Write-Host "Downloading psexec to: $PsExecPath" - & curl.exe -L -o $PsExecPath -s -S https://live.sysinternals.com/PsExec64.exe - $PsExecArgs = @( - '-u', - 'AdminUser', - '-p', - $AdminUserPassword, - '-accepteula', - '-h', - 'C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe', - '-ExecutionPolicy', - 'Unrestricted', - '-File', - $PSCommandPath - ) - - if (-Not ([string]::IsNullOrWhiteSpace($StorageAccountName))) { - $PsExecArgs += '-StorageAccountName' - $PsExecArgs += $StorageAccountName - } - - if (-Not ([string]::IsNullOrWhiteSpace($StorageAccountSasToken))) { - $PsExecArgs += '-StorageAccountSasToken' - $PsExecArgs += $StorageAccountSasToken - } - - Write-Host "Executing: $PsExecPath $PsExecArgs" - - $proc = Start-Process -FilePath $PsExecPath -ArgumentList $PsExecArgs -Wait -PassThru - Write-Host 'Reading transcript...' - Get-Content -Path $TranscriptPath - Write-Host 'Cleaning up...' - Remove-Item $PsExecPath - exit $proc.ExitCode -} - -$VisualStudioBootstrapperUrl = 'https://aka.ms/vs/16/release/vs_enterprise.exe' -$Workloads = @( - 'Microsoft.VisualStudio.Workload.NativeDesktop', - 'Microsoft.VisualStudio.Workload.Universal', - 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', - 'Microsoft.VisualStudio.Component.VC.Tools.ARM', - 'Microsoft.VisualStudio.Component.VC.Tools.ARM64', - 'Microsoft.VisualStudio.Component.VC.ATL', - 'Microsoft.VisualStudio.Component.VC.ATLMFC', - 'Microsoft.VisualStudio.Component.VC.v141.x86.x64.Spectre', - 'Microsoft.VisualStudio.Component.Windows10SDK.18362', - 'Microsoft.VisualStudio.Component.Windows10SDK.19041', - 'Microsoft.Net.Component.4.8.SDK', - 'Microsoft.Component.NetFX.Native', - 'Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset', - 'Microsoft.VisualStudio.Component.VC.Llvm.Clang', - 'Microsoft.VisualStudio.Component.VC.v141.x86.x64', - 'Microsoft.VisualStudio.Component.VC.140' -) - -$WindowsWDKUrl = 'https://go.microsoft.com/fwlink/?linkid=2128854' - -$MpiUrl = 'https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisetup.exe' - -$CudaUrl = 'https://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_426.00_win10.exe' -$CudaFeatures = 'nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 gpu_library_advisor_10.1 memcheck_10.1 ' + ` - 'nvdisasm_10.1 nvprof_10.1 visual_profiler_10.1 visual_studio_integration_10.1 cublas_10.1 cublas_dev_10.1 ' + ` - 'cudart_10.1 cufft_10.1 cufft_dev_10.1 curand_10.1 curand_dev_10.1 cusolver_10.1 cusolver_dev_10.1 cusparse_10.1 ' + ` - 'cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1 ' + ` - 'occupancy_calculator_10.1 fortran_examples_10.1' - -$PwshUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.1.0/PowerShell-7.1.0-win-x64.msi' - -$ErrorActionPreference = 'Stop' -$ProgressPreference = 'SilentlyContinue' - -<# -.SYNOPSIS -Writes a message to the screen depending on ExitCode. - -.DESCRIPTION -Since msiexec can return either 0 or 3010 successfully, in both cases -we write that installation succeeded, and which exit code it exited with. -If msiexec returns anything else, we write an error. - -.PARAMETER ExitCode -The exit code that msiexec returned. -#> -Function PrintMsiExitCodeMessage { - Param( - $ExitCode - ) - - # 3010 is probably ERROR_SUCCESS_REBOOT_REQUIRED - if ($ExitCode -eq 0 -or $ExitCode -eq 3010) { - Write-Host "Installation successful! Exited with $ExitCode." - } - else { - Write-Error "Installation failed! Exited with $ExitCode." - throw - } -} - -<# -.SYNOPSIS -Install Visual Studio. - -.DESCRIPTION -InstallVisualStudio takes the $Workloads array, and installs it with the -installer that's pointed at by $BootstrapperUrl. - -.PARAMETER Workloads -The set of VS workloads to install. - -.PARAMETER BootstrapperUrl -The URL of the Visual Studio installer, i.e. one of vs_*.exe. - -.PARAMETER InstallPath -The path to install Visual Studio at. - -.PARAMETER Nickname -The nickname to give the installation. -#> -Function InstallVisualStudio { - Param( - [String[]]$Workloads, - [String]$BootstrapperUrl, - [String]$InstallPath = $null, - [String]$Nickname = $null - ) - - try { - Write-Host 'Downloading Visual Studio...' - [string]$bootstrapperExe = Get-TempFilePath -Extension 'exe' - curl.exe -L -o $bootstrapperExe -s -S $BootstrapperUrl - Write-Host 'Installing Visual Studio...' - $args = @('/c', $bootstrapperExe, '--quiet', '--norestart', '--wait', '--nocache') - foreach ($workload in $Workloads) { - $args += '--add' - $args += $workload - } - - if (-not ([String]::IsNullOrWhiteSpace($InstallPath))) { - $args += '--installpath' - $args += $InstallPath - } - - if (-not ([String]::IsNullOrWhiteSpace($Nickname))) { - $args += '--nickname' - $args += $Nickname - } - - $proc = Start-Process -FilePath cmd.exe -ArgumentList $args -Wait -PassThru - PrintMsiExitCodeMessage $proc.ExitCode - } - catch { - Write-Error "Failed to install Visual Studio! $($_.Exception.Message)" - throw - } -} - -<# -.SYNOPSIS -Install a .msi file. - -.DESCRIPTION -InstallMSI takes a url where an .msi lives, and installs that .msi to the system. - -.PARAMETER Name -The name of the thing to install. - -.PARAMETER Url -The URL at which the .msi lives. -#> -Function InstallMSI { - Param( - [String]$Name, - [String]$Url - ) - - try { - Write-Host "Downloading $Name..." - [string]$msiPath = Get-TempFilePath -Extension 'msi' - curl.exe -L -o $msiPath -s -S $Url - Write-Host "Installing $Name..." - $args = @('/i', $msiPath, '/norestart', '/quiet', '/qn') - $proc = Start-Process -FilePath 'msiexec.exe' -ArgumentList $args -Wait -PassThru - PrintMsiExitCodeMessage $proc.ExitCode - } - catch { - Write-Error "Failed to install $Name! $($_.Exception.Message)" - throw - } -} - -<# -.SYNOPSIS -Unpacks a zip file to $Dir. - -.DESCRIPTION -InstallZip takes a URL of a zip file, and unpacks the zip file to the directory -$Dir. - -.PARAMETER Name -The name of the tool being installed. - -.PARAMETER Url -The URL of the zip file to unpack. - -.PARAMETER Dir -The directory to unpack the zip file to. -#> -Function InstallZip { - Param( - [String]$Name, - [String]$Url, - [String]$Dir - ) - - try { - Write-Host "Downloading $Name..." - [string]$zipPath = Get-TempFilePath -Extension 'zip' - curl.exe -L -o $zipPath -s -S $Url - Write-Host "Installing $Name..." - Expand-Archive -Path $zipPath -DestinationPath $Dir -Force - } - catch { - Write-Error "Failed to install $Name! $($_.Exception.Message)" - throw - } -} - -<# -.SYNOPSIS -Installs Windows WDK version 2004 - -.DESCRIPTION -Downloads the Windows WDK installer located at $Url, and installs it with the -correct flags. - -.PARAMETER Url -The URL of the installer. -#> -Function InstallWindowsWDK { - Param( - [String]$Url - ) - - try { - Write-Host 'Downloading Windows WDK...' - [string]$installerPath = Get-TempFilePath -Extension 'exe' - curl.exe -L -o $installerPath -s -S $Url - Write-Host 'Installing Windows WDK...' - $proc = Start-Process -FilePath $installerPath -ArgumentList @('/features', '+', '/q') -Wait -PassThru - $exitCode = $proc.ExitCode - if ($exitCode -eq 0) { - Write-Host 'Installation successful!' - } - else { - Write-Error "Installation failed! Exited with $exitCode." - throw - } - } - catch { - Write-Error "Failed to install Windows WDK! $($_.Exception.Message)" - throw - } -} - -<# -.SYNOPSIS -Installs MPI - -.DESCRIPTION -Downloads the MPI installer located at $Url, and installs it with the -correct flags. - -.PARAMETER Url -The URL of the installer. -#> -Function InstallMpi { - Param( - [String]$Url - ) - - try { - Write-Host 'Downloading MPI...' - [string]$installerPath = Get-TempFilePath -Extension 'exe' - curl.exe -L -o $installerPath -s -S $Url - Write-Host 'Installing MPI...' - $proc = Start-Process -FilePath $installerPath -ArgumentList @('-force', '-unattend') -Wait -PassThru - $exitCode = $proc.ExitCode - if ($exitCode -eq 0) { - Write-Host 'Installation successful!' - } - else { - Write-Error "Installation failed! Exited with $exitCode." - throw - } - } - catch { - Write-Error "Failed to install MPI! $($_.Exception.Message)" - throw - } -} - -<# -.SYNOPSIS -Installs NVIDIA's CUDA Toolkit. - -.DESCRIPTION -InstallCuda installs the CUDA Toolkit with the features specified as a -space-separated list of strings in $Features. - -.PARAMETER Url -The URL of the CUDA installer. - -.PARAMETER Features -A space-separated list of features to install. -#> -Function InstallCuda { - Param( - [String]$Url, - [String]$Features - ) - - try { - Write-Host 'Downloading CUDA...' - [string]$installerPath = Get-TempFilePath -Extension 'exe' - curl.exe -L -o $installerPath -s -S $Url - Write-Host 'Installing CUDA...' - $proc = Start-Process -FilePath $installerPath -ArgumentList @('-s ' + $Features) -Wait -PassThru - $exitCode = $proc.ExitCode - if ($exitCode -eq 0) { - Write-Host 'Installation successful!' - } - else { - Write-Error "Installation failed! Exited with $exitCode." - throw - } - } - catch { - Write-Error "Failed to install CUDA! $($_.Exception.Message)" - throw - } -} - -Write-Host "AdminUser password not supplied; assuming already running as AdminUser" - -Write-Host 'Disabling pagefile...' -wmic computersystem set AutomaticManagedPagefile=False -wmic pagefileset delete - -$av = Get-Command Add-MPPreference -ErrorAction SilentlyContinue -if ($null -eq $av) { - Write-Host 'AntiVirus not installed, skipping exclusions.' -} else { - Write-Host 'Configuring AntiVirus exclusions...' - Add-MpPreference -ExclusionPath C:\agent - Add-MPPreference -ExclusionPath D:\ - Add-MPPreference -ExclusionProcess ninja.exe - Add-MPPreference -ExclusionProcess clang-cl.exe - Add-MPPreference -ExclusionProcess cl.exe - Add-MPPreference -ExclusionProcess link.exe - Add-MPPreference -ExclusionProcess python.exe -} - -InstallVisualStudio -Workloads $Workloads -BootstrapperUrl $VisualStudioBootstrapperUrl -Nickname 'Stable' -InstallWindowsWDK -Url $WindowsWDKUrl -InstallMpi -Url $MpiUrl -InstallCuda -Url $CudaUrl -Features $CudaFeatures -InstallMSI -Url $PwshUrl -Name 'PowerShell Core' -if ([string]::IsNullOrWhiteSpace($StorageAccountName)) { - Write-Host 'No storage account name configured.' -} else { - Write-Host 'Storing storage account name to environment' - Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' ` - -Name PROVISIONED_AZURE_STORAGE_NAME ` - -Value $StorageAccountName -} -if ([string]::IsNullOrWhiteSpace($StorageAccountSasToken)) { - Write-Host 'No storage account key configured.' -} else { - Write-Host 'Storing storage account key to environment' - Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment' ` - -Name PROVISIONED_AZURE_STORAGE_SAS_TOKEN ` - -Value $StorageAccountSasToken -} diff --git a/scripts/azure-pipelines/windows/utility-prefix.ps1 b/scripts/azure-pipelines/windows/utility-prefix.ps1 new file mode 100644 index 00000000000000..8cd0066db60097 --- /dev/null +++ b/scripts/azure-pipelines/windows/utility-prefix.ps1 @@ -0,0 +1,125 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: MIT + +<# +.SYNOPSIS +Gets a random file path in the temp directory. + +.DESCRIPTION +Get-TempFilePath takes an extension, and returns a path with a random +filename component in the temporary directory with that extension. + +.PARAMETER Extension +The extension to use for the path. +#> +Function Get-TempFilePath { + Param( + [String]$Extension + ) + + if ([String]::IsNullOrWhiteSpace($Extension)) { + throw 'Missing Extension' + } + + $tempPath = [System.IO.Path]::GetTempPath() + $tempName = [System.IO.Path]::GetRandomFileName() + '.' + $Extension + return Join-Path $tempPath $tempName +} + +<# +.SYNOPSIS +Writes a message to the screen depending on ExitCode. + +.DESCRIPTION +Since msiexec can return either 0 or 3010 successfully, in both cases +we write that installation succeeded, and which exit code it exited with. +If msiexec returns anything else, we write an error. + +.PARAMETER ExitCode +The exit code that msiexec returned. +#> +Function PrintMsiExitCodeMessage { + Param( + $ExitCode + ) + + # 3010 is probably ERROR_SUCCESS_REBOOT_REQUIRED + if ($ExitCode -eq 0 -or $ExitCode -eq 3010) { + Write-Host "Installation successful! Exited with $ExitCode." + } + else { + Write-Error "Installation failed! Exited with $ExitCode." + throw + } +} + +<# +.SYNOPSIS +Install a .msi file. + +.DESCRIPTION +InstallMSI takes a url where an .msi lives, and installs that .msi to the system. + +.PARAMETER Name +The name of the thing to install. + +.PARAMETER Url +The URL at which the .msi lives. +#> +Function InstallMSI { + Param( + [String]$Name, + [String]$Url + ) + + try { + Write-Host "Downloading $Name..." + [string]$msiPath = Get-TempFilePath -Extension 'msi' + curl.exe -L -o $msiPath -s -S $Url + Write-Host "Installing $Name..." + $args = @('/i', $msiPath, '/norestart', '/quiet', '/qn') + $proc = Start-Process -FilePath 'msiexec.exe' -ArgumentList $args -Wait -PassThru + PrintMsiExitCodeMessage $proc.ExitCode + } + catch { + Write-Error "Failed to install $Name! $($_.Exception.Message)" + throw + } +} + +<# +.SYNOPSIS +Unpacks a zip file to $Dir. + +.DESCRIPTION +InstallZip takes a URL of a zip file, and unpacks the zip file to the directory +$Dir. + +.PARAMETER Name +The name of the tool being installed. + +.PARAMETER Url +The URL of the zip file to unpack. + +.PARAMETER Dir +The directory to unpack the zip file to. +#> +Function InstallZip { + Param( + [String]$Name, + [String]$Url, + [String]$Dir + ) + + try { + Write-Host "Downloading $Name..." + [string]$zipPath = Get-TempFilePath -Extension 'zip' + curl.exe -L -o $zipPath -s -S $Url + Write-Host "Installing $Name..." + Expand-Archive -Path $zipPath -DestinationPath $Dir -Force + } + catch { + Write-Error "Failed to install $Name! $($_.Exception.Message)" + throw + } +} diff --git a/scripts/ci.baseline.txt b/scripts/ci.baseline.txt index 01b33f2d1e66c9..9ced1bd96d47db 100644 --- a/scripts/ci.baseline.txt +++ b/scripts/ci.baseline.txt @@ -1784,7 +1784,6 @@ clapack:x64-uwp=skip lapack-reference:arm64-windows=skip lapack-reference:arm-uwp=skip - # failures for x64-windows-static-md ace:x64-windows-static-md=fail activemq-cpp:x64-windows-static-md=fail @@ -1819,3 +1818,15 @@ zyre:x64-windows-static-md=fail usbmuxd:x64-windows-static-md=fail workflow:x64-uwp=fail workflow:arm-uwp=fail + +# wangle triggers an internal compiler error +# https://devdiv.visualstudio.com/DefaultCollection/DevDiv/_workitems/edit/1269468 +wangle:x64-windows=fail +wangle:x64-windows-static=fail +wangle:x64-windows-static-md=fail + +# VS2019 version 16.9.4's project system changes where PDBs are placed in a way that breaks the +# upstream build script of this port. +# See https://developercommunity.visualstudio.com/t/Toolset-169-regression-vcxproj-producin/1356639 +dimcli:x64-windows-static-md=fail +dimcli:x64-windows-static=fail diff --git a/scripts/vcpkgTools.xml b/scripts/vcpkgTools.xml index 50e0476fec7515..3d48d52948764e 100644 --- a/scripts/vcpkgTools.xml +++ b/scripts/vcpkgTools.xml @@ -8,25 +8,25 @@ python-3.9.2-embed-win32.zip - 3.19.2 - cmake-3.19.2-win32-x86\bin\cmake.exe - https://github.com/Kitware/CMake/releases/download/v3.19.2/cmake-3.19.2-win32-x86.zip - 1ed3706d8dd5c5244ceb4c22443f006fcb9ffa76cb6d9ebdbbb8f11a50deb3f92a9a4080ff3fd125f79c0143f553ea0c02a6a2d27521438705c2fea4dd97d239 - cmake-3.19.2-win32-x86.zip + 3.20.1 + cmake-3.20.1-windows-i386\bin\cmake.exe + https://github.com/Kitware/CMake/releases/download/v3.20.1/cmake-3.20.1-windows-i386.zip + bc0be4e32bbffe102b2cb970591118bf2132d6afd6c97469cf46a4f772627fa81a8a57e4069dc32a93c0ffcce176e562d9d008e6780f053876e5d5eb97051554 + cmake-3.20.1-windows-i386.zip - 3.19.2 - cmake-3.19.2-macos-universal/CMake.app/Contents/bin/cmake - https://github.com/Kitware/CMake/releases/download/v3.19.2/cmake-3.19.2-macos-universal.tar.gz - 031196439a26d6ff2a1f42830ca26b5159752dc2c2c889a4e304327c858157b68088905b10727090918c3605fe7fe429aa19d46e68ed30c17d4d89b3d209aae8 - cmake-3.19.2-macos-universal.tar.gz + 3.20.1 + cmake-3.20.1-macos-universal/CMake.app/Contents/bin/cmake + https://github.com/Kitware/CMake/releases/download/v3.20.1/cmake-3.20.1-macos-universal.tar.gz + fd17d09a0639833a2aa5f30045cfcbbb30e0cfa5092d2f22f0bc502ea361a458d26be96437fbfb6b5e68b73954050c0256653af7e2e20bb253a878e206992135 + cmake-3.20.1-macos-universal.tar.gz - 3.19.2 - cmake-3.19.2-Linux-x86_64/bin/cmake - https://github.com/Kitware/CMake/releases/download/v3.19.2/cmake-3.19.2-Linux-x86_64.tar.gz - 67ea7ee428e6c93039646d6d0cbe84c0be337a2b878487635fee71ec2c8992a9078a660e6b793ef11ce13530def44a16b716af96e10a97675d014a26caa0aed4 - cmake-3.19.2-Linux-x86_64.tar.gz + 3.20.1 + cmake-3.20.1-linux-x86_64/bin/cmake + https://github.com/Kitware/CMake/releases/download/v3.20.1/cmake-3.20.1-Linux-x86_64.tar.gz + e14b6cbc03abfbae074c36e34f90eea94af02e727e75ed597f748203e2ce7483ffe693bd173cb02a1de7ac55254a891da5505c43b92528adc918eff9e8b024e6 + cmake-3.20.1-linux-x86_64.tar.gz 3.12.4 @@ -134,10 +134,10 @@ ninja-freebsd-1.8.2.zip - 7.1.0 + 7.1.3 pwsh.exe - https://github.com/PowerShell/PowerShell/releases/download/v7.1.0/PowerShell-7.1.0-win-x86.zip - ea6ed619c784af65b09a5e98cdfc0d41333ba7373b7409e488ec70aaf1e0c4e5568e846d2b1748d72c3e3a9c1a4827f0ba29e9f6e93e79b4f49312ca40af6d2a - PowerShell-7.1.0-win-x86.zip + https://github.com/PowerShell/PowerShell/releases/download/v7.1.3/PowerShell-7.1.3-win-x86.zip + 8c2ce510b5c641aad2da6adefc92d47e09bc842d47db3b5d15e14859555a74fe13ad52eaeabf1b2954ca9af737e628b567731c8a3db9bbf0e4aad05279bc1fd8 + PowerShell-7.1.3-win-x86.zip diff --git a/versions/baseline.json b/versions/baseline.json index 09f969c6a923ad..52e92cec17bbe6 100644 --- a/versions/baseline.json +++ b/versions/baseline.json @@ -1206,7 +1206,7 @@ }, "chromaprint": { "baseline": "1.5.0", - "port-version": 0 + "port-version": 1 }, "chromium-base": { "baseline": "86.0.4199.1", @@ -1526,7 +1526,7 @@ }, "cudnn": { "baseline": "7.6.5", - "port-version": 1 + "port-version": 2 }, "cunit": { "baseline": "2.1.3-6", @@ -1574,7 +1574,7 @@ }, "dartsim": { "baseline": "6.9.4", - "port-version": 0 + "port-version": 1 }, "dataframe": { "baseline": "1.12.0", @@ -3009,7 +3009,7 @@ "port-version": 0 }, "libdatachannel": { - "baseline": "0.12.0", + "baseline": "0.12.2", "port-version": 0 }, "libdatrie": { @@ -3146,7 +3146,7 @@ }, "libhv": { "baseline": "1.0.0", - "port-version": 1 + "port-version": 2 }, "libhydrogen": { "baseline": "2019-08-11", @@ -3558,7 +3558,7 @@ }, "libudis86": { "baseline": "2018-01-28-56ff6c87", - "port-version": 0 + "port-version": 1 }, "libudns": { "baseline": "0.4-2", @@ -3606,7 +3606,7 @@ }, "libvpx": { "baseline": "1.9.0", - "port-version": 8 + "port-version": 9 }, "libwandio": { "baseline": "4.2.1", @@ -5922,7 +5922,7 @@ }, "tensorflow-common": { "baseline": "2.4.1", - "port-version": 0 + "port-version": 1 }, "termcolor": { "baseline": "2.0.0", @@ -6198,7 +6198,7 @@ }, "upb": { "baseline": "2020-12-19", - "port-version": 0 + "port-version": 1 }, "urdfdom": { "baseline": "1.0.4", diff --git a/versions/c-/chromaprint.json b/versions/c-/chromaprint.json index 5fc772975e891d..a97cedf9ced3e9 100644 --- a/versions/c-/chromaprint.json +++ b/versions/c-/chromaprint.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "d841d451ed68f08043a27b2ec0beb24d3a276c81", + "version": "1.5.0", + "port-version": 1 + }, { "git-tree": "1f3fe321540c3d87ad63fe85efa5b48bd9c55ac8", "version-string": "1.5.0", diff --git a/versions/c-/cudnn.json b/versions/c-/cudnn.json index 4930853d40409d..cc7e05d0ce712b 100644 --- a/versions/c-/cudnn.json +++ b/versions/c-/cudnn.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "5581791a9ad35390ed30629985b3863cf8b85bba", + "version-string": "7.6.5", + "port-version": 2 + }, { "git-tree": "495cc586b4bfdb28c3c339d3f471b9fbfc8bc481", "version-string": "7.6.5", diff --git a/versions/d-/dartsim.json b/versions/d-/dartsim.json index 4fd122f2f9fe73..f5b4f03e25affe 100644 --- a/versions/d-/dartsim.json +++ b/versions/d-/dartsim.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "104efce0932ef1a1e1e01358a442cfff637b285e", + "version": "6.9.4", + "port-version": 1 + }, { "git-tree": "e9a9e2df385efc1d41aeea022f88f7e6a07f81ba", "version-string": "6.9.4", diff --git a/versions/l-/libdatachannel.json b/versions/l-/libdatachannel.json index b85861e672d9dc..2b289cd5d9c467 100644 --- a/versions/l-/libdatachannel.json +++ b/versions/l-/libdatachannel.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "dd3107dd99419236f97e299ce4b61c2379a08f55", + "version-semver": "0.12.2", + "port-version": 0 + }, { "git-tree": "1c0a9c87c0695f1ecc03f342ba87d21ae152f7dd", "version-semver": "0.12.0", diff --git a/versions/l-/libhv.json b/versions/l-/libhv.json index 8d373a2611a846..9fae80665216e1 100644 --- a/versions/l-/libhv.json +++ b/versions/l-/libhv.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "3a460f29d525c6c1ce8154593238960415a24d80", + "version": "1.0.0", + "port-version": 2 + }, { "git-tree": "82104ab053e797fdcaa93d64b11fe05d37e5e14c", "version": "1.0.0", diff --git a/versions/l-/libudis86.json b/versions/l-/libudis86.json index 62fefa67deb84f..0e098416d243c0 100644 --- a/versions/l-/libudis86.json +++ b/versions/l-/libudis86.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "e8ab6af2cfbbb52901433f92f0c028bc766ce3ae", + "version-string": "2018-01-28-56ff6c87", + "port-version": 1 + }, { "git-tree": "55fbd94b278b0b3d753a2db345afa5f20b95c350", "version-string": "2018-01-28-56ff6c87", diff --git a/versions/l-/libvpx.json b/versions/l-/libvpx.json index 0d90bed5fa44d3..65efd934664f63 100644 --- a/versions/l-/libvpx.json +++ b/versions/l-/libvpx.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "dd62b59825999b4a4136fb9e92b393b5600596d1", + "version-semver": "1.9.0", + "port-version": 9 + }, { "git-tree": "b3c25e1e8a48e31e14fb824fd7b8b618266093b9", "version-semver": "1.9.0", diff --git a/versions/t-/tensorflow-common.json b/versions/t-/tensorflow-common.json index a42b859be96edc..fc8ec6df0901fe 100644 --- a/versions/t-/tensorflow-common.json +++ b/versions/t-/tensorflow-common.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "c2a11af14488e643cf0a07f676d08319a0f70f08", + "version-semver": "2.4.1", + "port-version": 1 + }, { "git-tree": "566f036d2265f6c960014ceb526ed0d143f65c60", "version-semver": "2.4.1", diff --git a/versions/u-/upb.json b/versions/u-/upb.json index bbe7a8b07d12c7..6d0406e44aee57 100644 --- a/versions/u-/upb.json +++ b/versions/u-/upb.json @@ -1,5 +1,10 @@ { "versions": [ + { + "git-tree": "df35bcef11c276d5cc7280edeca76eeff3dad223", + "version-date": "2020-12-19", + "port-version": 1 + }, { "git-tree": "8c7a75bb8754caa918eddfd0e6f9457834e81f64", "version-date": "2020-12-19",