Skip to content

[cudnn][nccl] Don't download bits if not found #16031

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jacobkahn wants to merge 1 commit into from
Closed

[cudnn][nccl] Don't download bits if not found #16031

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 7 additions & 43 deletions ports/cudnn/portfile.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,15 @@ endif()
vcpkg_check_linkage(ONLY_DYNAMIC_LIBRARY ONLY_DYNAMIC_CRT)
set(VCPKG_POLICY_MISMATCHED_NUMBER_OF_BINARIES enabled) # only release bits are provided

#note: this port must be kept in sync with CUDA port: every time one is upgraded, the other must be too
# note: this port must be kept in sync with CUDA port: every time one is upgraded, the other must be too
# Minimum version to find -- should match the CUDA port's minimum version's corresponding NCCL version
set(CUDNN_VERSION "7.6.0")
set(CUDNN_FULL_VERSION "${CUDNN_VERSION}-cuda10.1_0")
string(REPLACE "." ";" VERSION_LIST ${CUDNN_VERSION})
list(GET VERSION_LIST 0 CUDNN_VERSION_MAJOR)
list(GET VERSION_LIST 1 CUDNN_VERSION_MINOR)
list(GET VERSION_LIST 2 CUDNN_VERSION_PATCH)

# Try to find CUDNN if it exists; only download if it doesn't exist
# Try to find CUDNN if it exists
find_path(CUDNN_INCLUDE_DIR cudnn.h
HINTS ${CUDA_HOME} ${CUDA_TOOLKIT_ROOT_DIR} $ENV{cudnn} $ENV{CUDNN} $ENV{CUDNN_ROOT_DIR}
PATH_SUFFIXES cuda/include include)
Expand Down Expand Up @@ -56,47 +56,11 @@ else()
set(CUDNN_FOUND FALSE)
endif()

# Download CUDNN if not found
if (CUDNN_FOUND)
message(STATUS "Found CUDNN located on system: (include ${CUDNN_INCLUDE_DIR} lib: ${CUDNN_LIBRARY})")
set(VCPKG_POLICY_EMPTY_PACKAGE enabled)
else()
message(STATUS "CUDNN not found on system - downloading...")
if(VCPKG_TARGET_IS_WINDOWS)
set(CUDNN_DOWNLOAD_LINK "https://anaconda.org/anaconda/cudnn/${CUDNN_VERSION}/download/win-64/cudnn-${CUDNN_FULL_VERSION}.tar.bz2")
set(SHA512_CUDNN "c0218407e7bc2b3c1497f1709dedee345bc619603ec0efa094e392888c0d513d645a1241501f9b406f688defa811578f36b49f456eb533535ecd526702156eea")
set(CUDNN_OS "windows")
elseif(VCPKG_TARGET_IS_LINUX)
set(CUDNN_DOWNLOAD_LINK "https://anaconda.org/anaconda/cudnn/${CUDNN_VERSION}/download/linux-64/cudnn-${CUDNN_FULL_VERSION}.tar.bz2")
set(SHA512_CUDNN "128ccdc0ec24a1133947d7a8eff6cd8edc224134fa5065a11a1a01a99dbaee7d799db1454e0a59e411cf6db244f8c2420c160488a5dd4830addc3578b2011e3d")
set(CUDNN_OS "linux")
endif()

vcpkg_download_distfile(ARCHIVE
URLS ${CUDNN_DOWNLOAD_LINK}
FILENAME "cudnn-${CUDNN_FULL_VERSION}-${CUDNN_OS}.tar.bz2"
SHA512 ${SHA512_CUDNN}
)

vcpkg_extract_source_archive_ex(
OUT_SOURCE_PATH SOURCE_PATH
ARCHIVE ${ARCHIVE}
NO_REMOVE_ONE_LEVEL
)

if(VCPKG_TARGET_IS_WINDOWS)
file(INSTALL "${SOURCE_PATH}/Library/include/cudnn.h" DESTINATION ${CURRENT_PACKAGES_DIR}/include)
file(INSTALL "${SOURCE_PATH}/Library/lib/x64/cudnn.lib" DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
file(INSTALL "${SOURCE_PATH}/Library/bin/cudnn64_${CUDNN_VERSION_MAJOR}.dll" DESTINATION ${CURRENT_PACKAGES_DIR}/bin)
elseif(VCPKG_TARGET_IS_LINUX)
file(INSTALL "${SOURCE_PATH}/include/cudnn.h" DESTINATION ${CURRENT_PACKAGES_DIR}/include)
file(INSTALL "${SOURCE_PATH}/lib/libcudnn.so.${CUDNN_VERSION}" DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
file(INSTALL "${SOURCE_PATH}/lib/libcudnn.so.${CUDNN_VERSION_MAJOR}" DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
file(INSTALL "${SOURCE_PATH}/lib/libcudnn.so" DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
endif()

file(INSTALL "${SOURCE_PATH}/info/LICENSE.txt" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright)
file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/FindCUDNN.cmake" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT})
file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT})

endif() # NOT CUDNN_FOUND
message(FATAL_ERROR "Could not find cuDNN. Before continuing, please download and install a cuDNN version "
"\nthat matches your CUDA version from:"
"\n https://developer.nvidia.com/cuDNN\n")
endif()
40 changes: 6 additions & 34 deletions ports/nccl/portfile.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,55 +1,27 @@
vcpkg_fail_port_install(ON_TARGET "Windows" "OSX" ON_ARCH "x86" "arm")

# note: this port must be kept in sync with CUDA port: every time one is upgraded, the other must be too
# Minimum version to find -- should match the CUDA port's minimum version's corresponding cuDNN version
set(NCCL_VERSION "2.4.6.1")
set(NCCL_FULL_VERSION "${NCCL_VERSION}-cuda10.1_0")
string(REPLACE "." ";" VERSION_LIST ${NCCL_VERSION})
list(GET VERSION_LIST 0 NCCL_VERSION_MAJOR)
list(GET VERSION_LIST 1 NCCL_VERSION_MINOR)
list(GET VERSION_LIST 2 NCCL_VERSION_PATCH)

set(NCCL_DOWNLOAD_LINK "https://anaconda.org/nvidia/nccl/${NCCL_VERSION}/download/linux-64/nccl-${NCCL_FULL_VERSION}.tar.bz2")

# Try to find NCCL if it exists; only download if it doesn't exist
# Try to find NCCL if it exists
set(NCCL_PREV_MODULE_PATH ${CMAKE_MODULE_PATH})
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR})
find_package(NCCL ${NCCL_VERSION})
set(CMAKE_MODULE_PATH ${NCCL_PREV_MODULE_PATH})

# Download or return
# Download only if the CUDA version is exactly 10.1 (which matches the vcpkg CUDA port required version), else fail
if(NCCL_FOUND)
message(STATUS "Using NCCL located on system.")
set(VCPKG_POLICY_EMPTY_PACKAGE enabled)
else()
message(STATUS "NCCL not found on system. Downloading...")
vcpkg_check_linkage(ONLY_DYNAMIC_LIBRARY ONLY_DYNAMIC_CRT)

set(SHA512_NCCL "0fe69ad559f70aab97c78906296e2b909b4a9c042a228a2770252b3d03016c7c39acce3c0e0bd0ba651abd63471743dcffdfec307c486989c6e5745634aabde1")
set(NCCL_OS "linux")

vcpkg_download_distfile(ARCHIVE
URLS ${NCCL_DOWNLOAD_LINK}
FILENAME "nccl-${NCCL_FULL_VERSION}-${NCCL_OS}.tar.bz2"
SHA512 ${SHA512_NCCL}
)

vcpkg_extract_source_archive_ex(
OUT_SOURCE_PATH SOURCE_PATH
ARCHIVE ${ARCHIVE}
NO_REMOVE_ONE_LEVEL
)

file(INSTALL "${SOURCE_PATH}/include/nccl.h" DESTINATION ${CURRENT_PACKAGES_DIR}/include)
file(INSTALL "${SOURCE_PATH}/include/nccl_net.h" DESTINATION ${CURRENT_PACKAGES_DIR}/include)

file(INSTALL "${SOURCE_PATH}/lib/libnccl.so" DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
file(INSTALL "${SOURCE_PATH}/lib/libnccl.so.${NCCL_VERSION_MAJOR}" DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
file(INSTALL "${SOURCE_PATH}/lib/libnccl.so.${NCCL_VERSION_MAJOR}.${NCCL_VERSION_MINOR}.${NCCL_VERSION_PATCH}" DESTINATION ${CURRENT_PACKAGES_DIR}/lib)

file(INSTALL "${SOURCE_PATH}/info/licenses/LICENSE.txt" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright)
file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/FindNCCL.cmake" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT})
file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT})

message(FATAL_ERROR "Could not find NCCL. Before continuing, please download and install a NCCL version "
"\nthat matches your CUDA version from:"
"\n https://developer.nvidia.com/nccl\n")
endif()

file(INSTALL "${CURRENT_PORT_DIR}/usage" DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT})