diff --git a/build.sh b/build.sh index 5136ebb3d3dd..6a43674c642b 100755 --- a/build.sh +++ b/build.sh @@ -17,7 +17,7 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n -l --allgpuarch --disable_nvtx --show_depr_warn --ptds -h --build_metrics --incl_cache_stats" +VALIDARGS="clean libcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n -l --allgpuarch --disable_nvtx --opensource_nvcomp --show_depr_warn --ptds -h --build_metrics --incl_cache_stats" HELP="$0 [clean] [libcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"\\\"] clean - remove all existing build artifacts and configuration (start over) @@ -35,6 +35,7 @@ HELP="$0 [clean] [libcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [li -n - no install step --allgpuarch - build for all supported GPU architectures --disable_nvtx - disable inserting NVTX profiling ranges + --opensource_nvcomp - disable use of proprietary nvcomp extensions --show_depr_warn - show cmake deprecation warnings --ptds - enable per-thread default stream --build_metrics - generate build metrics report for libcudf @@ -67,6 +68,7 @@ BUILD_DISABLE_DEPRECATION_WARNING=ON BUILD_PER_THREAD_DEFAULT_STREAM=OFF BUILD_REPORT_METRICS=OFF BUILD_REPORT_INCL_CACHE_STATS=OFF +USE_PROPRIETARY_NVCOMP=ON # Set defaults for vars that may not have been defined externally # FIXME: if INSTALL_PREFIX is not set, check PREFIX, then check @@ -144,6 +146,7 @@ function buildLibCudfJniInDocker { -DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES} \ -DCMAKE_INSTALL_PREFIX=/usr/local/rapids \ -DUSE_NVTX=ON \ + -DCUDF_USE_PROPRIETARY_NVCOMP=ON \ -DCUDF_USE_ARROW_STATIC=ON \ -DCUDF_ENABLE_ARROW_S3=OFF \ -DBUILD_TESTS=OFF \ @@ -209,6 +212,9 @@ fi if hasArg --disable_nvtx; then BUILD_NVTX="OFF" fi +if hasArg --opensource_nvcomp; then + USE_PROPRIETARY_NVCOMP="OFF" +fi if hasArg --show_depr_warn; then BUILD_DISABLE_DEPRECATION_WARNING=OFF fi @@ -271,6 +277,7 @@ if buildAll || hasArg libcudf; then -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES} \ -DUSE_NVTX=${BUILD_NVTX} \ + -DCUDF_USE_PROPRIETARY_NVCOMP=${USE_PROPRIETARY_NVCOMP} \ -DBUILD_TESTS=${BUILD_TESTS} \ -DBUILD_BENCHMARKS=${BUILD_BENCHMARKS} \ -DDISABLE_DEPRECATION_WARNING=${BUILD_DISABLE_DEPRECATION_WARNING} \ diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml index 20bf2afa957d..b6c531ffde72 100644 --- a/conda/recipes/libcudf/meta.yaml +++ b/conda/recipes/libcudf/meta.yaml @@ -285,6 +285,8 @@ outputs: license_family: Apache license_file: LICENSE summary: libcudf library + prelink_message: + - nvcomp.txt - name: libcudf_kafka version: {{ version }} script: install_libcudf_kafka.sh diff --git a/conda/recipes/libcudf/nvcomp.txt b/conda/recipes/libcudf/nvcomp.txt new file mode 100644 index 000000000000..9a0047e71fac --- /dev/null +++ b/conda/recipes/libcudf/nvcomp.txt @@ -0,0 +1,3 @@ +By downloading and using the libcudf conda package, you accept the terms +and conditions of the NVIDIA NVCOMP Software License Agreement: + https://developer.download.nvidia.com/compute/nvcomp/2.3/LICENSE.txt diff --git a/conda/recipes/libcudf/post-link.sh b/conda/recipes/libcudf/post-link.sh new file mode 100644 index 000000000000..64e0b1ad305c --- /dev/null +++ b/conda/recipes/libcudf/post-link.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Copyright (c) 2022, NVIDIA CORPORATION. +# Only add the license notice to libcudf and not our examples / tests +if [[ "$PKG_NAME" == "libcudf" ]]; then + cat ./nvlink.txt >> $PREFIX/.messages.txt +fi diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index e2fd8ce56eec..d18955122d23 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -42,6 +42,7 @@ option(BUILD_TESTS "Configure CMake to build tests" ON) option(BUILD_BENCHMARKS "Configure CMake to build (google & nvbench) benchmarks" OFF) option(BUILD_SHARED_LIBS "Build cuDF shared libraries" ON) option(JITIFY_USE_CACHE "Use a file cache for JIT compiled kernels" ON) +option(CUDF_USE_PROPRIETARY_NVCOMP "Download and use NVCOMP with proprietary extensions" ON) option(CUDF_USE_ARROW_STATIC "Build and statically link Arrow libraries" OFF) option(CUDF_ENABLE_ARROW_ORC "Build the Arrow ORC adapter" OFF) option(CUDF_ENABLE_ARROW_PYTHON "Find (or build) Arrow with Python support" OFF) diff --git a/cpp/cmake/thirdparty/get_nvcomp.cmake b/cpp/cmake/thirdparty/get_nvcomp.cmake index 03213da72787..41bbf44abc88 100644 --- a/cpp/cmake/thirdparty/get_nvcomp.cmake +++ b/cpp/cmake/thirdparty/get_nvcomp.cmake @@ -13,26 +13,14 @@ # ============================================================================= # This function finds nvcomp and sets any additional necessary environment variables. -function(find_and_configure_nvcomp VERSION_MIN VERSION_MAX) - # Search for latest version of nvComp - rapids_find_package(nvcomp ${VERSION_MAX} QUIET) - # If latest isn't found, fall back to building oldest support from source - rapids_cpm_find( - nvcomp ${VERSION_MIN} - GLOBAL_TARGETS nvcomp::nvcomp - CPM_ARGS GITHUB_REPOSITORY NVIDIA/nvcomp - GIT_TAG v${VERSION_MIN} - OPTIONS "BUILD_STATIC ON" "BUILD_TESTS OFF" "BUILD_BENCHMARKS OFF" "BUILD_EXAMPLES OFF" - ) - - if(nvcomp_BINARY_DIR) - include("${rapids-cmake-dir}/export/find_package_root.cmake") - rapids_export_find_package_root(BUILD nvcomp "${nvcomp_BINARY_DIR}" cudf-exports) - endif() +function(find_and_configure_nvcomp) - if(NOT TARGET nvcomp::nvcomp) - add_library(nvcomp::nvcomp ALIAS nvcomp) - endif() + include(${rapids-cmake-dir}/cpm/nvcomp.cmake) + rapids_cpm_nvcomp( + BUILD_EXPORT_SET cudf-exports + INSTALL_EXPORT_SET cudf-exports + USE_PROPRIETARY_BINARY ${CUDF_USE_PROPRIETARY_NVCOMP} + ) # Per-thread default stream if(TARGET nvcomp AND CUDF_USE_PER_THREAD_DEFAULT_STREAM) @@ -40,6 +28,4 @@ function(find_and_configure_nvcomp VERSION_MIN VERSION_MAX) endif() endfunction() -set(CUDF_MIN_VERSION_nvCOMP 2.2.0) -set(CUDF_MAX_VERSION_nvCOMP 2.3.0) -find_and_configure_nvcomp(${CUDF_MIN_VERSION_nvCOMP} ${CUDF_MAX_VERSION_nvCOMP}) +find_and_configure_nvcomp() diff --git a/java/pom.xml b/java/pom.xml index be6a63b5cba7..074968d496bc 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -508,6 +508,9 @@ libcudfjni.so libcufilejni.so + libnvcomp.so + libnvcomp_gdeflate.so + libnvcomp_bitcomp.so diff --git a/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java b/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java index 9663fbcafb49..27322cca4364 100755 --- a/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java +++ b/java/src/main/java/ai/rapids/cudf/NativeDepsLoader.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -54,6 +54,12 @@ public class NativeDepsLoader { * subsequent stages are loaded. */ private static final String[][] loadOrder = new String[][]{ + new String[]{ + "nvcomp_bitcomp", "nvcomp_gdeflate" + }, + new String[]{ + "nvcomp" + }, new String[]{ "cudf" }, diff --git a/java/src/main/native/CMakeLists.txt b/java/src/main/native/CMakeLists.txt index 5d349b0c3831..5108dd414ef2 100755 --- a/java/src/main/native/CMakeLists.txt +++ b/java/src/main/native/CMakeLists.txt @@ -104,18 +104,20 @@ endif() # Set a default build type if none was specified rapids_cmake_build_type("Release") -# ################################################################################################## -# * nvcomp------------------------------------------------------------------------------------------ - -set(nvcomp_DIR "${CUDF_CPP_BUILD_DIR}/_deps/nvcomp-build") -rapids_find_package(nvcomp REQUIRED) - # ################################################################################################## # * CUDF ------------------------------------------------------------------------------------------ set(cudf_ROOT "${CUDF_CPP_BUILD_DIR}") rapids_find_package(cudf REQUIRED) +# ################################################################################################## +# * nvcomp------------------------------------------------------------------------------------------ + +if(NOT DEFINED nvcomp_DIR) + set(nvcomp_DIR "${CUDF_CPP_BUILD_DIR}/_deps/nvcomp-build") +endif() +rapids_find_package(nvcomp REQUIRED) + # ################################################################################################## # * find JNI ------------------------------------------------------------------------------------- find_package(JNI REQUIRED) @@ -254,3 +256,16 @@ else() # Tell CMake what CUDA language runtime to use set_target_properties(cudfjni PROPERTIES CUDA_RUNTIME_LIBRARY Shared) endif() + +# ################################################################################################## +# * install shared libraries ---------------------------------------------------------------------- +if(TARGET nvcomp::nvcomp) + add_custom_command( + TARGET cudfjni + PRE_LINK + COMMAND + ${CMAKE_COMMAND} -E copy $ $ + $ "${PROJECT_BINARY_DIR}" + COMMENT "Copying nvcomp libraries to ${PROJECT_BINARY_DIR}" + ) +endif()