Skip to content

[native ] Fix setup scripts to align with Velox setup scripts #22448

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
majetideepak merged 1 commit into from
Apr 23, 2024
Merged

[native ] Fix setup scripts to align with Velox setup scripts #22448

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion presto-native-execution/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,8 @@ centos-container: #: Build the linux container for CircleCi
linux-container:
rm -rf /tmp/docker && \
mkdir -p /tmp/docker && \
cp scripts/setup-$(CONTAINER_NAME).sh scripts/$(CONTAINER_NAME)-container.dockfile velox/scripts/setup-helper-functions.sh /tmp/docker && \
$(eval VELOX_SETUP_SCRIPT=$(shell [ $(CONTAINER_NAME) == centos ] && echo setup-centos8.sh; [ $(CONTAINER_NAME) != centos ] && echo setup-$(CONTAINER_NAME).sh;)) \
cp scripts/setup-$(CONTAINER_NAME).sh scripts/$(CONTAINER_NAME)-container.dockfile velox/scripts/${VELOX_SETUP_SCRIPT} velox/scripts/setup-helper-functions.sh /tmp/docker && \
cd /tmp/docker && \
docker build --build-arg cpu_target=$(CPU_TARGET) --tag "prestocpp/prestocpp-$(CPU_TARGET)-$(CONTAINER_NAME):$(USER)-$(shell date +%Y%m%d)" -f $(CONTAINER_NAME)-container.dockfile .

Expand Down
2 changes: 1 addition & 1 deletion presto-native-execution/scripts/centos-container.dockfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,4 @@ FROM ghcr.io/facebookincubator/velox-dev:centos8
COPY setup-centos.sh /
COPY setup-helper-functions.sh /
RUN chmod +x ./setup-centos.sh
RUN mkdir build && ( cd build && ../setup-centos.sh ) && rm -rf build
RUN mkdir build && ( cd build && ../setup-centos.sh install_presto_deps ) && rm -rf build
1 change: 0 additions & 1 deletion presto-native-execution/scripts/release-centos-dockerfile/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@ RUN --mount=type=ssh \
WORKDIR ${DEPENDENCY_DIR}
RUN --mount=type=ssh \
set -exu && \
bash "${PRESTODB_HOME}/_repo/presto-native-execution/velox/scripts/setup-centos8.sh" && \
bash "${PRESTODB_HOME}/_repo/presto-native-execution/scripts/setup-centos.sh" && \
python3 -m pip install six && \
set +exu && \
Expand Down
62 changes: 46 additions & 16 deletions presto-native-execution/scripts/setup-centos.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,42 +17,72 @@ set -x
export FB_OS_VERSION=v2024.04.01.00
export RE2_VERSION=2021-04-01
export nproc=$(getconf _NPROCESSORS_ONLN)

dnf install -y maven java python3-devel clang-tools-extra jq perl-XML-XPath

python3 -m pip install regex pyyaml chevron black

export CC=/opt/rh/gcc-toolset-9/root/bin/gcc
export CXX=/opt/rh/gcc-toolset-9/root/bin/g++

CPU_TARGET="${CPU_TARGET:-avx}"
SCRIPT_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")
if [ -f "${SCRIPT_DIR}/setup-helper-functions.sh" ]
if [ -f "${SCRIPT_DIR}/setup-centos8.sh" ]
then
source "${SCRIPT_DIR}/setup-helper-functions.sh"
source "${SCRIPT_DIR}/setup-centos8.sh"
else
source "${SCRIPT_DIR}/../velox/scripts/setup-helper-functions.sh"
source "${SCRIPT_DIR}/../velox/scripts/setup-centos8.sh"
fi

export COMPILER_FLAGS=$(echo -n $(get_cxx_flags $CPU_TARGET))
function install_presto_deps_from_package_managers {
dnf install -y maven java clang-tools-extra jq perl-XML-XPath
# This python version is installed by the Velox setup scripts
pip3.9 install regex pyyaml chevron black
}

(
function install_gperf {
wget http://ftp.gnu.org/pub/gnu/gperf/gperf-3.1.tar.gz &&
tar xvfz gperf-3.1.tar.gz &&
cd gperf-3.1 &&
./configure --prefix=/usr/local/gperf/3_1 &&
make "-j$(nproc)" &&
make install &&
ln -s /usr/local/gperf/3_1/bin/gperf /usr/local/bin/
)

make install
if [ -f /usr/local/bin/gperf ]; then
echo "Did not create '/usr/local/bin/gperf' symlink as file already exists."
else
ln -s /usr/local/gperf/3_1/bin/gperf /usr/local/bin/
fi
}

(
function install_proxygen {
git clone https://github.com/facebook/proxygen &&
cd proxygen &&
git checkout $FB_OS_VERSION &&
cmake_install -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=ON
)
}

function install_presto_deps {
run_and_time install_presto_deps_from_package_managers
run_and_time install_gperf
run_and_time install_proxygen
}

if [[ $# -ne 0 ]]; then
# Activate gcc9; enable errors on unset variables afterwards.
source /opt/rh/gcc-toolset-9/enable || exit 1
set -u
for cmd in "$@"; do
run_and_time "${cmd}"
done
echo "All specified dependencies installed!"
else
if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then
echo "Installing build dependencies"
run_and_time install_build_prerequisites
else
echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set"
fi
# Activate gcc9; enable errors on unset variables afterwards.
source /opt/rh/gcc-toolset-9/enable || exit 1
set -u
install_velox_deps
install_presto_deps
echo "All dependencies for Prestissimo installed!"
fi

dnf clean all
14 changes: 10 additions & 4 deletions presto-native-execution/scripts/setup-macos.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,31 @@ set -eufx -o pipefail
# Run the velox setup script first.
source "$(dirname "${BASH_SOURCE}")/../velox/scripts/setup-macos.sh"

MACOS_DEPS="${MACOS_DEPS} bison gperf"
export FB_OS_VERSION=v2024.04.01.00

export PATH=$(brew --prefix bison)/bin:$PATH
Copy link
Copy Markdown
Contributor

@czentgr czentgr Apr 19, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

None of the prestissimo/velox deps require bison?
On Silicon Macs this will be a problem going forward but is likely set later by the users. But removing this we don't expect issues?

Copy link
Copy Markdown
Collaborator Author

@majetideepak majetideepak Apr 22, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a temporary assignment on the current shell. I am thinking there should not be any issues since we don't run this script on every build and things build fine.

Copy link
Copy Markdown
Collaborator Author

@majetideepak majetideepak Apr 23, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, bison is installed in the Velox setup script.


function install_proxygen {
github_checkout facebook/proxygen "${FB_OS_VERSION}"
cmake_install -DBUILD_TESTS=OFF
}

function install_presto_deps {
install_velox_deps
install_from_brew "gperf"
run_and_time install_proxygen
}

if [[ $# -ne 0 ]]; then
for cmd in "$@"; do
run_and_time "${cmd}"
done
echo "All specified dependencies installed!"
else
if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then
echo "Installing build dependencies"
run_and_time install_build_prerequisites
else
echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set"
fi
install_velox_deps
install_presto_deps
echo "All dependencies for Prestissimo installed!"
fi
16 changes: 14 additions & 2 deletions presto-native-execution/scripts/setup-ubuntu.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,34 @@ set -eufx -o pipefail
# Run the velox setup script first.
source "$(dirname "${BASH_SOURCE}")/../velox/scripts/setup-ubuntu.sh"
export FB_OS_VERSION=v2024.04.01.00
sudo apt install -y gperf

function install_presto_deps_from_apt {
sudo apt install -y gperf
}

function install_proxygen {
github_checkout facebook/proxygen "${FB_OS_VERSION}"
cmake_install -DBUILD_TESTS=OFF
}

function install_presto_deps {
install_velox_deps
run_and_time install_presto_deps_from_apt
run_and_time install_proxygen
}

if [[ $# -ne 0 ]]; then
for cmd in "$@"; do
run_and_time "${cmd}"
done
echo "All specified dependencies installed!"
else
if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then
echo "Installing build dependencies"
run_and_time install_build_prerequisites
else
echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set"
fi
install_velox_deps
install_presto_deps
echo "All dependencies for Prestissimo installed!"
fi