diff --git a/presto-native-execution/Makefile b/presto-native-execution/Makefile index 6e13f534923de..96e1278602433 100644 --- a/presto-native-execution/Makefile +++ b/presto-native-execution/Makefile @@ -113,7 +113,8 @@ centos-container: #: Build the linux container for CircleCi linux-container: rm -rf /tmp/docker && \ mkdir -p /tmp/docker && \ - cp scripts/setup-$(CONTAINER_NAME).sh scripts/$(CONTAINER_NAME)-container.dockfile velox/scripts/setup-helper-functions.sh /tmp/docker && \ + $(eval VELOX_SETUP_SCRIPT=$(shell [ $(CONTAINER_NAME) == centos ] && echo setup-centos8.sh; [ $(CONTAINER_NAME) != centos ] && echo setup-$(CONTAINER_NAME).sh;)) \ + cp scripts/setup-$(CONTAINER_NAME).sh scripts/$(CONTAINER_NAME)-container.dockfile velox/scripts/${VELOX_SETUP_SCRIPT} velox/scripts/setup-helper-functions.sh /tmp/docker && \ cd /tmp/docker && \ docker build --build-arg cpu_target=$(CPU_TARGET) --tag "prestocpp/prestocpp-$(CPU_TARGET)-$(CONTAINER_NAME):$(USER)-$(shell date +%Y%m%d)" -f $(CONTAINER_NAME)-container.dockfile . diff --git a/presto-native-execution/scripts/centos-container.dockfile b/presto-native-execution/scripts/centos-container.dockfile index 94ff1bf766ade..7fcb6e19dde3c 100644 --- a/presto-native-execution/scripts/centos-container.dockfile +++ b/presto-native-execution/scripts/centos-container.dockfile @@ -16,4 +16,4 @@ FROM ghcr.io/facebookincubator/velox-dev:centos8 COPY setup-centos.sh / COPY setup-helper-functions.sh / RUN chmod +x ./setup-centos.sh -RUN mkdir build && ( cd build && ../setup-centos.sh ) && rm -rf build +RUN mkdir build && ( cd build && ../setup-centos.sh install_presto_deps ) && rm -rf build diff --git a/presto-native-execution/scripts/release-centos-dockerfile/Dockerfile b/presto-native-execution/scripts/release-centos-dockerfile/Dockerfile index de461627c4f0d..ea018f6d2f860 100644 --- a/presto-native-execution/scripts/release-centos-dockerfile/Dockerfile +++ b/presto-native-execution/scripts/release-centos-dockerfile/Dockerfile @@ -70,7 +70,6 @@ RUN --mount=type=ssh \ WORKDIR ${DEPENDENCY_DIR} RUN --mount=type=ssh \ set -exu && \ - bash "${PRESTODB_HOME}/_repo/presto-native-execution/velox/scripts/setup-centos8.sh" && \ bash "${PRESTODB_HOME}/_repo/presto-native-execution/scripts/setup-centos.sh" && \ python3 -m pip install six && \ set +exu && \ diff --git a/presto-native-execution/scripts/setup-centos.sh b/presto-native-execution/scripts/setup-centos.sh index 569b901482ab2..ae0880be0648b 100755 --- a/presto-native-execution/scripts/setup-centos.sh +++ b/presto-native-execution/scripts/setup-centos.sh @@ -17,42 +17,72 @@ set -x export FB_OS_VERSION=v2024.04.01.00 export RE2_VERSION=2021-04-01 export nproc=$(getconf _NPROCESSORS_ONLN) - -dnf install -y maven java python3-devel clang-tools-extra jq perl-XML-XPath - -python3 -m pip install regex pyyaml chevron black - export CC=/opt/rh/gcc-toolset-9/root/bin/gcc export CXX=/opt/rh/gcc-toolset-9/root/bin/g++ CPU_TARGET="${CPU_TARGET:-avx}" SCRIPT_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") -if [ -f "${SCRIPT_DIR}/setup-helper-functions.sh" ] +if [ -f "${SCRIPT_DIR}/setup-centos8.sh" ] then - source "${SCRIPT_DIR}/setup-helper-functions.sh" + source "${SCRIPT_DIR}/setup-centos8.sh" else - source "${SCRIPT_DIR}/../velox/scripts/setup-helper-functions.sh" + source "${SCRIPT_DIR}/../velox/scripts/setup-centos8.sh" fi -export COMPILER_FLAGS=$(echo -n $(get_cxx_flags $CPU_TARGET)) +function install_presto_deps_from_package_managers { + dnf install -y maven java clang-tools-extra jq perl-XML-XPath + # This python version is installed by the Velox setup scripts + pip3.9 install regex pyyaml chevron black +} -( +function install_gperf { wget http://ftp.gnu.org/pub/gnu/gperf/gperf-3.1.tar.gz && tar xvfz gperf-3.1.tar.gz && cd gperf-3.1 && ./configure --prefix=/usr/local/gperf/3_1 && make "-j$(nproc)" && - make install && - ln -s /usr/local/gperf/3_1/bin/gperf /usr/local/bin/ -) - + make install + if [ -f /usr/local/bin/gperf ]; then + echo "Did not create '/usr/local/bin/gperf' symlink as file already exists." + else + ln -s /usr/local/gperf/3_1/bin/gperf /usr/local/bin/ + fi +} -( +function install_proxygen { git clone https://github.com/facebook/proxygen && cd proxygen && git checkout $FB_OS_VERSION && cmake_install -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=ON -) +} +function install_presto_deps { + run_and_time install_presto_deps_from_package_managers + run_and_time install_gperf + run_and_time install_proxygen +} + +if [[ $# -ne 0 ]]; then + # Activate gcc9; enable errors on unset variables afterwards. + source /opt/rh/gcc-toolset-9/enable || exit 1 + set -u + for cmd in "$@"; do + run_and_time "${cmd}" + done + echo "All specified dependencies installed!" +else + if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then + echo "Installing build dependencies" + run_and_time install_build_prerequisites + else + echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set" + fi + # Activate gcc9; enable errors on unset variables afterwards. + source /opt/rh/gcc-toolset-9/enable || exit 1 + set -u + install_velox_deps + install_presto_deps + echo "All dependencies for Prestissimo installed!" +fi dnf clean all diff --git a/presto-native-execution/scripts/setup-macos.sh b/presto-native-execution/scripts/setup-macos.sh index 76cba797ad7cf..b9e0793d10956 100755 --- a/presto-native-execution/scripts/setup-macos.sh +++ b/presto-native-execution/scripts/setup-macos.sh @@ -16,18 +16,15 @@ set -eufx -o pipefail # Run the velox setup script first. source "$(dirname "${BASH_SOURCE}")/../velox/scripts/setup-macos.sh" -MACOS_DEPS="${MACOS_DEPS} bison gperf" export FB_OS_VERSION=v2024.04.01.00 -export PATH=$(brew --prefix bison)/bin:$PATH - function install_proxygen { github_checkout facebook/proxygen "${FB_OS_VERSION}" cmake_install -DBUILD_TESTS=OFF } function install_presto_deps { - install_velox_deps + install_from_brew "gperf" run_and_time install_proxygen } @@ -35,6 +32,15 @@ if [[ $# -ne 0 ]]; then for cmd in "$@"; do run_and_time "${cmd}" done + echo "All specified dependencies installed!" else + if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then + echo "Installing build dependencies" + run_and_time install_build_prerequisites + else + echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set" + fi + install_velox_deps install_presto_deps + echo "All dependencies for Prestissimo installed!" fi diff --git a/presto-native-execution/scripts/setup-ubuntu.sh b/presto-native-execution/scripts/setup-ubuntu.sh index 9ac9f5cc50d80..4aea16c59ab2d 100755 --- a/presto-native-execution/scripts/setup-ubuntu.sh +++ b/presto-native-execution/scripts/setup-ubuntu.sh @@ -19,7 +19,10 @@ set -eufx -o pipefail # Run the velox setup script first. source "$(dirname "${BASH_SOURCE}")/../velox/scripts/setup-ubuntu.sh" export FB_OS_VERSION=v2024.04.01.00 -sudo apt install -y gperf + +function install_presto_deps_from_apt { + sudo apt install -y gperf +} function install_proxygen { github_checkout facebook/proxygen "${FB_OS_VERSION}" @@ -27,7 +30,7 @@ function install_proxygen { } function install_presto_deps { - install_velox_deps + run_and_time install_presto_deps_from_apt run_and_time install_proxygen } @@ -35,6 +38,15 @@ if [[ $# -ne 0 ]]; then for cmd in "$@"; do run_and_time "${cmd}" done + echo "All specified dependencies installed!" else + if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then + echo "Installing build dependencies" + run_and_time install_build_prerequisites + else + echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set" + fi + install_velox_deps install_presto_deps + echo "All dependencies for Prestissimo installed!" fi