From 6f93c75646ebfd9ea55f40f90aba26efca5eb7b6 Mon Sep 17 00:00:00 2001 From: Deepak Majeti Date: Mon, 8 Apr 2024 12:13:59 -0400 Subject: [PATCH] [native] Fix setup scripts to align with Velox setup scripts Velox setup scripts have been modified to not install the build dependencies by default. Fix setup scripts to adopt that behavior. Fix setup scripts to encapsulate presto packages into a function. Fix setup-centos8.sh script to include Velox dependencies. --- presto-native-execution/Makefile | 3 +- .../scripts/centos-container.dockfile | 2 +- .../release-centos-dockerfile/Dockerfile | 1 - .../scripts/setup-centos.sh | 62 ++++++++++++++----- .../scripts/setup-macos.sh | 14 +++-- .../scripts/setup-ubuntu.sh | 16 ++++- 6 files changed, 73 insertions(+), 25 deletions(-) diff --git a/presto-native-execution/Makefile b/presto-native-execution/Makefile index 6e13f534923de..96e1278602433 100644 --- a/presto-native-execution/Makefile +++ b/presto-native-execution/Makefile @@ -113,7 +113,8 @@ centos-container: #: Build the linux container for CircleCi linux-container: rm -rf /tmp/docker && \ mkdir -p /tmp/docker && \ - cp scripts/setup-$(CONTAINER_NAME).sh scripts/$(CONTAINER_NAME)-container.dockfile velox/scripts/setup-helper-functions.sh /tmp/docker && \ + $(eval VELOX_SETUP_SCRIPT=$(shell [ $(CONTAINER_NAME) == centos ] && echo setup-centos8.sh; [ $(CONTAINER_NAME) != centos ] && echo setup-$(CONTAINER_NAME).sh;)) \ + cp scripts/setup-$(CONTAINER_NAME).sh scripts/$(CONTAINER_NAME)-container.dockfile velox/scripts/${VELOX_SETUP_SCRIPT} velox/scripts/setup-helper-functions.sh /tmp/docker && \ cd /tmp/docker && \ docker build --build-arg cpu_target=$(CPU_TARGET) --tag "prestocpp/prestocpp-$(CPU_TARGET)-$(CONTAINER_NAME):$(USER)-$(shell date +%Y%m%d)" -f $(CONTAINER_NAME)-container.dockfile . diff --git a/presto-native-execution/scripts/centos-container.dockfile b/presto-native-execution/scripts/centos-container.dockfile index 94ff1bf766ade..7fcb6e19dde3c 100644 --- a/presto-native-execution/scripts/centos-container.dockfile +++ b/presto-native-execution/scripts/centos-container.dockfile @@ -16,4 +16,4 @@ FROM ghcr.io/facebookincubator/velox-dev:centos8 COPY setup-centos.sh / COPY setup-helper-functions.sh / RUN chmod +x ./setup-centos.sh -RUN mkdir build && ( cd build && ../setup-centos.sh ) && rm -rf build +RUN mkdir build && ( cd build && ../setup-centos.sh install_presto_deps ) && rm -rf build diff --git a/presto-native-execution/scripts/release-centos-dockerfile/Dockerfile b/presto-native-execution/scripts/release-centos-dockerfile/Dockerfile index de461627c4f0d..ea018f6d2f860 100644 --- a/presto-native-execution/scripts/release-centos-dockerfile/Dockerfile +++ b/presto-native-execution/scripts/release-centos-dockerfile/Dockerfile @@ -70,7 +70,6 @@ RUN --mount=type=ssh \ WORKDIR ${DEPENDENCY_DIR} RUN --mount=type=ssh \ set -exu && \ - bash "${PRESTODB_HOME}/_repo/presto-native-execution/velox/scripts/setup-centos8.sh" && \ bash "${PRESTODB_HOME}/_repo/presto-native-execution/scripts/setup-centos.sh" && \ python3 -m pip install six && \ set +exu && \ diff --git a/presto-native-execution/scripts/setup-centos.sh b/presto-native-execution/scripts/setup-centos.sh index 569b901482ab2..ae0880be0648b 100755 --- a/presto-native-execution/scripts/setup-centos.sh +++ b/presto-native-execution/scripts/setup-centos.sh @@ -17,42 +17,72 @@ set -x export FB_OS_VERSION=v2024.04.01.00 export RE2_VERSION=2021-04-01 export nproc=$(getconf _NPROCESSORS_ONLN) - -dnf install -y maven java python3-devel clang-tools-extra jq perl-XML-XPath - -python3 -m pip install regex pyyaml chevron black - export CC=/opt/rh/gcc-toolset-9/root/bin/gcc export CXX=/opt/rh/gcc-toolset-9/root/bin/g++ CPU_TARGET="${CPU_TARGET:-avx}" SCRIPT_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") -if [ -f "${SCRIPT_DIR}/setup-helper-functions.sh" ] +if [ -f "${SCRIPT_DIR}/setup-centos8.sh" ] then - source "${SCRIPT_DIR}/setup-helper-functions.sh" + source "${SCRIPT_DIR}/setup-centos8.sh" else - source "${SCRIPT_DIR}/../velox/scripts/setup-helper-functions.sh" + source "${SCRIPT_DIR}/../velox/scripts/setup-centos8.sh" fi -export COMPILER_FLAGS=$(echo -n $(get_cxx_flags $CPU_TARGET)) +function install_presto_deps_from_package_managers { + dnf install -y maven java clang-tools-extra jq perl-XML-XPath + # This python version is installed by the Velox setup scripts + pip3.9 install regex pyyaml chevron black +} -( +function install_gperf { wget http://ftp.gnu.org/pub/gnu/gperf/gperf-3.1.tar.gz && tar xvfz gperf-3.1.tar.gz && cd gperf-3.1 && ./configure --prefix=/usr/local/gperf/3_1 && make "-j$(nproc)" && - make install && - ln -s /usr/local/gperf/3_1/bin/gperf /usr/local/bin/ -) - + make install + if [ -f /usr/local/bin/gperf ]; then + echo "Did not create '/usr/local/bin/gperf' symlink as file already exists." + else + ln -s /usr/local/gperf/3_1/bin/gperf /usr/local/bin/ + fi +} -( +function install_proxygen { git clone https://github.com/facebook/proxygen && cd proxygen && git checkout $FB_OS_VERSION && cmake_install -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=ON -) +} +function install_presto_deps { + run_and_time install_presto_deps_from_package_managers + run_and_time install_gperf + run_and_time install_proxygen +} + +if [[ $# -ne 0 ]]; then + # Activate gcc9; enable errors on unset variables afterwards. + source /opt/rh/gcc-toolset-9/enable || exit 1 + set -u + for cmd in "$@"; do + run_and_time "${cmd}" + done + echo "All specified dependencies installed!" +else + if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then + echo "Installing build dependencies" + run_and_time install_build_prerequisites + else + echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set" + fi + # Activate gcc9; enable errors on unset variables afterwards. + source /opt/rh/gcc-toolset-9/enable || exit 1 + set -u + install_velox_deps + install_presto_deps + echo "All dependencies for Prestissimo installed!" +fi dnf clean all diff --git a/presto-native-execution/scripts/setup-macos.sh b/presto-native-execution/scripts/setup-macos.sh index 76cba797ad7cf..b9e0793d10956 100755 --- a/presto-native-execution/scripts/setup-macos.sh +++ b/presto-native-execution/scripts/setup-macos.sh @@ -16,18 +16,15 @@ set -eufx -o pipefail # Run the velox setup script first. source "$(dirname "${BASH_SOURCE}")/../velox/scripts/setup-macos.sh" -MACOS_DEPS="${MACOS_DEPS} bison gperf" export FB_OS_VERSION=v2024.04.01.00 -export PATH=$(brew --prefix bison)/bin:$PATH - function install_proxygen { github_checkout facebook/proxygen "${FB_OS_VERSION}" cmake_install -DBUILD_TESTS=OFF } function install_presto_deps { - install_velox_deps + install_from_brew "gperf" run_and_time install_proxygen } @@ -35,6 +32,15 @@ if [[ $# -ne 0 ]]; then for cmd in "$@"; do run_and_time "${cmd}" done + echo "All specified dependencies installed!" else + if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then + echo "Installing build dependencies" + run_and_time install_build_prerequisites + else + echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set" + fi + install_velox_deps install_presto_deps + echo "All dependencies for Prestissimo installed!" fi diff --git a/presto-native-execution/scripts/setup-ubuntu.sh b/presto-native-execution/scripts/setup-ubuntu.sh index 9ac9f5cc50d80..4aea16c59ab2d 100755 --- a/presto-native-execution/scripts/setup-ubuntu.sh +++ b/presto-native-execution/scripts/setup-ubuntu.sh @@ -19,7 +19,10 @@ set -eufx -o pipefail # Run the velox setup script first. source "$(dirname "${BASH_SOURCE}")/../velox/scripts/setup-ubuntu.sh" export FB_OS_VERSION=v2024.04.01.00 -sudo apt install -y gperf + +function install_presto_deps_from_apt { + sudo apt install -y gperf +} function install_proxygen { github_checkout facebook/proxygen "${FB_OS_VERSION}" @@ -27,7 +30,7 @@ function install_proxygen { } function install_presto_deps { - install_velox_deps + run_and_time install_presto_deps_from_apt run_and_time install_proxygen } @@ -35,6 +38,15 @@ if [[ $# -ne 0 ]]; then for cmd in "$@"; do run_and_time "${cmd}" done + echo "All specified dependencies installed!" else + if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then + echo "Installing build dependencies" + run_and_time install_build_prerequisites + else + echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set" + fi + install_velox_deps install_presto_deps + echo "All dependencies for Prestissimo installed!" fi