diff --git a/scripts/setup-adapters.sh b/scripts/setup-adapters.sh index 4203ec58aae7..aa7108e348d8 100755 --- a/scripts/setup-adapters.sh +++ b/scripts/setup-adapters.sh @@ -61,32 +61,6 @@ function install_gcs-sdk-cpp { # Install gcs dependencies # https://github.com/googleapis/google-cloud-cpp/blob/main/doc/packaging.md#required-libraries - # abseil-cpp - github_checkout abseil/abseil-cpp 20240116.2 --depth 1 - cmake_install \ - -DABSL_BUILD_TESTING=OFF \ - -DCMAKE_CXX_STANDARD=17 \ - -DABSL_PROPAGATE_CXX_STD=ON \ - -DABSL_ENABLE_INSTALL=ON - - # protobuf - github_checkout protocolbuffers/protobuf v21.8 --depth 1 - cmake_install \ - -Dprotobuf_BUILD_TESTS=OFF \ - -Dprotobuf_ABSL_PROVIDER=package - - # grpc - github_checkout grpc/grpc v1.48.1 --depth 1 - cmake_install \ - -DgRPC_BUILD_TESTS=OFF \ - -DgRPC_ABSL_PROVIDER=package \ - -DgRPC_ZLIB_PROVIDER=package \ - -DgRPC_CARES_PROVIDER=package \ - -DgRPC_RE2_PROVIDER=package \ - -DgRPC_SSL_PROVIDER=package \ - -DgRPC_PROTOBUF_PROVIDER=package \ - -DgRPC_INSTALL=ON - # crc32 github_checkout google/crc32c 1.1.2 --depth 1 cmake_install \ @@ -180,13 +154,13 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then if [[ "$LINUX_DISTRIBUTION" == "ubuntu" || "$LINUX_DISTRIBUTION" == "debian" ]]; then apt install -y --no-install-recommends libxml2-dev libgsasl7-dev uuid-dev # Dependencies of GCS, probably a workaround until the docker image is rebuilt - apt install -y --no-install-recommends libc-ares-dev libcurl4-openssl-dev + apt install -y --no-install-recommends libcurl4-openssl-dev # Dependencies of Azure Storage Blob cpp apt install -y openssl else # Assume Fedora/CentOS dnf -y install libxml2-devel libgsasl-devel libuuid-devel krb5-devel # Dependencies of GCS, probably a workaround until the docker image is rebuilt - dnf -y install npm curl-devel c-ares-devel + dnf -y install npm curl-devel # Dependencies of Azure Storage Blob Cpp dnf -y install perl-IPC-Cmd dnf -y install openssl diff --git a/scripts/setup-centos9.sh b/scripts/setup-centos9.sh index 1cf43630c4f1..930203154043 100755 --- a/scripts/setup-centos9.sh +++ b/scripts/setup-centos9.sh @@ -35,6 +35,7 @@ export CXXFLAGS=$(get_cxx_flags) # Used by boost. export CFLAGS=${CXXFLAGS//"-std=c++17"/} # Used by LZO. CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" BUILD_DUCKDB="${BUILD_DUCKDB:-true}" +BUILD_ARROW_FLIGHT="${BUILD_ARROW_FLIGHT:-false}" USE_CLANG="${USE_CLANG:-false}" export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"} DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download} @@ -75,7 +76,7 @@ function install_velox_deps_from_dnf { dnf_install libevent-devel \ openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \ libdwarf-devel elfutils-libelf-devel curl-devel libicu-devel bison flex \ - libsodium-devel zlib-devel + libsodium-devel zlib-devel c-ares-devel # install sphinx for doc gen pip install sphinx sphinx-tabs breathe sphinx_rtd_theme @@ -142,7 +143,8 @@ function install_protobuf { ./configure CXXFLAGS="-fPIC" --prefix=${INSTALL_PREFIX} make "-j${NPROC}" make install - ldconfig + ldconfig -v 2>/dev/null | grep "${INSTALL_PREFIX}/lib" || \ + echo "${INSTALL_PREFIX}/lib" > /etc/ld.so.conf.d/local-libraries.conf && ldconfig ) } @@ -190,7 +192,38 @@ function install_stemmer { ) } +function install_abseil { + # abseil-cpp + github_checkout abseil/abseil-cpp 20240116.2 --depth 1 + cmake_install \ + -DABSL_BUILD_TESTING=OFF \ + -DCMAKE_CXX_STANDARD=17 \ + -DABSL_PROPAGATE_CXX_STD=ON \ + -DABSL_ENABLE_INSTALL=ON +} + +function install_grpc { + # grpc + github_checkout grpc/grpc v1.48.1 --depth 1 + cmake_install \ + -DgRPC_BUILD_TESTS=OFF \ + -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_ZLIB_PROVIDER=package \ + -DgRPC_CARES_PROVIDER=package \ + -DgRPC_RE2_PROVIDER=package \ + -DgRPC_SSL_PROVIDER=package \ + -DgRPC_PROTOBUF_PROVIDER=package \ + -DgRPC_INSTALL=ON +} + function install_arrow { + if $BUILD_ARROW_FLIGHT; then + run_and_time install_grpc + ARROW_FLIGHT=ON + else + ARROW_FLIGHT=OFF + fi + wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow cmake_install_dir arrow/cpp \ -DARROW_PARQUET=OFF \ @@ -204,6 +237,8 @@ function install_arrow { -DARROW_RUNTIME_SIMD_LEVEL=NONE \ -DARROW_WITH_UTF8PROC=OFF \ -DARROW_TESTING=ON \ + -DARROW_FLIGHT=${ARROW_FLIGHT} \ + -DARROW_BUILD_BENCHMARKS=${ARROW_FLIGHT} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_BUILD_TYPE=Release \ -DARROW_BUILD_STATIC=ON \ @@ -241,6 +276,7 @@ function install_velox_deps { run_and_time install_fbthrift run_and_time install_duckdb run_and_time install_stemmer + run_and_time install_abseil run_and_time install_arrow } diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index 3d502547a7f6..6fa91263b6cb 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -36,6 +36,7 @@ COMPILER_FLAGS=$(get_cxx_flags) export COMPILER_FLAGS NPROC=$(getconf _NPROCESSORS_ONLN) BUILD_DUCKDB="${BUILD_DUCKDB:-true}" +BUILD_ARROW_FLIGHT="${BUILD_ARROW_FLIGHT:-false}" export CMAKE_BUILD_TYPE=Release SUDO="${SUDO:-"sudo --preserve-env"}" USE_CLANG="${USE_CLANG:-false}" @@ -162,8 +163,10 @@ function install_protobuf { cd ${DEPENDENCY_DIR}/protobuf ./configure CXXFLAGS="-fPIC" --prefix=${INSTALL_PREFIX} make "-j${NPROC}" - make install - ldconfig + ${SUDO} make install + ${SUDO} ldconfig -v 2>/dev/null | grep "${INSTALL_PREFIX}/lib" || \ + echo "${INSTALL_PREFIX}/lib" | ${SUDO} tee /etc/ld.so.conf.d/local-libraries.conf > /dev/null \ + && ${SUDO} ldconfig ) } @@ -229,7 +232,38 @@ function install_stemmer { ) } +function install_abseil { + # abseil-cpp + github_checkout abseil/abseil-cpp 20240116.2 --depth 1 + cmake_install \ + -DABSL_BUILD_TESTING=OFF \ + -DCMAKE_CXX_STANDARD=17 \ + -DABSL_PROPAGATE_CXX_STD=ON \ + -DABSL_ENABLE_INSTALL=ON +} + +function install_grpc { + # grpc + github_checkout grpc/grpc v1.48.1 --depth 1 + cmake_install \ + -DgRPC_BUILD_TESTS=OFF \ + -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_ZLIB_PROVIDER=package \ + -DgRPC_CARES_PROVIDER=package \ + -DgRPC_RE2_PROVIDER=package \ + -DgRPC_SSL_PROVIDER=package \ + -DgRPC_PROTOBUF_PROVIDER=package \ + -DgRPC_INSTALL=ON +} + function install_arrow { + if $BUILD_ARROW_FLIGHT; then + run_and_time install_grpc + ARROW_FLIGHT=ON + else + ARROW_FLIGHT=OFF + fi + wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow cmake_install_dir arrow/cpp \ -DARROW_PARQUET=OFF \ @@ -243,6 +277,8 @@ function install_arrow { -DARROW_RUNTIME_SIMD_LEVEL=NONE \ -DARROW_WITH_UTF8PROC=OFF \ -DARROW_TESTING=ON \ + -DARROW_FLIGHT=${ARROW_FLIGHT} \ + -DARROW_BUILD_BENCHMARKS=${ARROW_FLIGHT} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_BUILD_TYPE=Release \ -DARROW_BUILD_STATIC=ON \ @@ -281,6 +317,7 @@ function install_velox_deps { run_and_time install_conda run_and_time install_duckdb run_and_time install_stemmer + run_and_time install_abseil run_and_time install_arrow }