Skip to content

[DO NOT MERGE] Native image build #21377

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
aaneja wants to merge 5 commits into from
Closed

[DO NOT MERGE] Native image build #21377

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
96d9e9d
Add build/debug scripts of native worker docker image
ethanyzhang Oct 28, 2023
67de744
Add GenerateDomainFilters
aaneja Nov 9, 2023
3bbf9c0
TPCDS plan changes from generating domain filters
aaneja Nov 10, 2023
747f4ae
Fix minor checkstyle issue
aaneja Nov 10, 2023
f819cae
Remove redundant sort columns
aaneja Nov 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions Dockerfile-native
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
ARG PRESTO_VERSION

FROM prestocpp/prestocpp-avx-centos:root-20230613 as Builder

WORKDIR /app
COPY . .
RUN cd presto-native-execution && \
make velox-submodule && \
source /opt/rh/gcc-toolset-9/enable && \
source velox/scripts/setup-helper-functions.sh && \
(mkdir -p third_party && cd third_party && github_checkout aws/aws-sdk-cpp 1.9.96 --depth 1 --recurse-submodules --config advice.detachedHead=false && \
cmake_install -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS:BOOL=OFF -DMINIMIZE_SIZE:BOOL=ON -DENABLE_TESTING:BOOL=OFF -DBUILD_ONLY:STRING="s3;identity-management") && \
EXTRA_CMAKE_FLAGS=" -DVELOX_ENABLE_INT64_BUILD_PARTITION_BOUND=ON" PRESTO_ENABLE_PARQUET=ON PRESTO_ENABLE_S3=ON PRESTO_ENABLE_TESTING=OFF MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=8 NUM_THREADS=8 TREAT_WARNINGS_AS_ERRORS=0 make release \
|| tail -n 500 _build/release/CMakeFiles/CMakeError.log

RUN cd presto-native-execution && \
mkdir -p prestissimo && \
cp _build/release/presto_cpp/main/presto_server prestissimo && \
tar cvf prestissimo.tar entrypoint.sh velox.properties prestissimo

##

FROM prestocpp/prestocpp-avx-centos:root-20230613
ENV PRESTO_HOME="/opt/presto-server"

RUN dnf update -y && dnf install -y \
awscli \
gperf \
iproute \
lsof \
procps \
python3 \
sysstat \
tar \
vim \
wget \
which \
&& \
mkdir -p $PRESTO_HOME/etc/catalog && \
mkdir -p /var/lib/presto/data

WORKDIR /app
COPY --from=Builder /app/presto-native-execution/prestissimo.tar .
RUN tar xvf prestissimo.tar && \
mkdir -p /opt/presto-server/etc && \
mv prestissimo/presto_server /usr/local/bin/ && \
mv velox.properties /opt/presto-server/etc/ && \
mv entrypoint.sh /opt/ && \
touch /opt/presto-native-execution-${PRESTO_VERSION}

ENTRYPOINT ["/opt/entrypoint.sh"]

52 changes: 52 additions & 0 deletions Dockerfile-native-debug
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
ARG PRESTO_VERSION

FROM prestocpp/prestocpp-avx-centos:root-20230613 as Builder

WORKDIR /app
COPY . .
RUN cd presto-native-execution && \
make velox-submodule && \
source /opt/rh/gcc-toolset-9/enable && \
source velox/scripts/setup-helper-functions.sh && \
(mkdir -p third_party && cd third_party && github_checkout aws/aws-sdk-cpp 1.9.96 --depth 1 --recurse-submodules --config advice.detachedHead=false && \
cmake_install -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS:BOOL=OFF -DMINIMIZE_SIZE:BOOL=ON -DENABLE_TESTING:BOOL=OFF -DBUILD_ONLY:STRING="s3;identity-management") && \
EXTRA_CMAKE_FLAGS=" -DVELOX_ENABLE_INT64_BUILD_PARTITION_BOUND=ON" PRESTO_ENABLE_PARQUET=ON PRESTO_ENABLE_S3=ON PRESTO_ENABLE_TESTING=OFF MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=4 NUM_THREADS=2 TREAT_WARNINGS_AS_ERRORS=0 make debug \
|| tail -n 500 _build/debug/CMakeFiles/CMakeError.log

RUN cd presto-native-execution && \
mkdir -p prestissimo && \
cp _build/debug/presto_cpp/main/presto_server prestissimo && \
tar cvf prestissimo.tar entrypoint.sh velox.properties prestissimo

##

FROM prestocpp/prestocpp-avx-centos:root-20230613
ENV PRESTO_HOME="/opt/presto-server"

RUN dnf update -y && dnf install -y \
awscli \
gperf \
iproute \
lsof \
procps \
python3 \
sysstat \
tar \
vim \
wget \
which \
&& \
mkdir -p $PRESTO_HOME/etc/catalog && \
mkdir -p /var/lib/presto/data

WORKDIR /app
COPY --from=Builder /app/presto-native-execution/prestissimo.tar .
RUN tar xvf prestissimo.tar && \
mkdir -p /opt/presto-server/etc && \
mv prestissimo/presto_server /usr/local/bin/ && \
mv velox.properties /opt/presto-server/etc/ && \
mv entrypoint.sh /opt/ && \
touch /opt/presto-native-execution-${PRESTO_VERSION}

ENTRYPOINT ["/opt/entrypoint.sh"]

19 changes: 19 additions & 0 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ pipeline {
sh(script: "git show -s --format=%cd --date=format:'%Y%m%d%H%M%S'", returnStdout: true).trim() + "-" +
env.PRESTO_COMMIT_SHA.substring(0, 7)
env.DOCKER_IMAGE = env.AWS_ECR + "/presto:${PRESTO_BUILD_VERSION}"
env.DOCKER_NATIVE_IMAGE = env.AWS_ECR + "/presto-native:${PRESTO_BUILD_VERSION}"
}
sh 'printenv | sort'

Expand Down Expand Up @@ -168,6 +169,22 @@ pipeline {
}
}

stage('Docker Native Build') {
steps {
echo "Building ${DOCKER_NATIVE_IMAGE}"
withCredentials([[
$class: 'AmazonWebServicesCredentialsBinding',
credentialsId: "${AWS_CREDENTIAL_ID}",
accessKeyVariable: 'AWS_ACCESS_KEY_ID',
secretKeyVariable: 'AWS_SECRET_ACCESS_KEY']]) {
sh '''#!/bin/bash -ex
docker buildx build -f Dockerfile-native --load --platform "linux/amd64" -t "${DOCKER_NATIVE_IMAGE}-amd64" \
--build-arg "PRESTO_VERSION=${PRESTO_VERSION}" .
'''
}
}
}

stage('Publish Docker') {
when {
anyOf {
Expand All @@ -194,6 +211,7 @@ pipeline {
docker manifest annotate "${DOCKER_IMAGE}" "${DOCKER_IMAGE}-amd64" --os linux --arch amd64
docker manifest annotate "${DOCKER_IMAGE}" "${DOCKER_IMAGE}-arm64" --os linux --arch arm64
docker manifest push "${DOCKER_IMAGE}"
docker push "${DOCKER_NATIVE_IMAGE}-amd64"
'''
}
}
Expand All @@ -202,3 +220,4 @@ pipeline {
}
}
}

38 changes: 19 additions & 19 deletions jenkins/agent-dind.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,23 +1,23 @@
apiVersion: v1
kind: Pod
metadata:
namespace: oss-agent
labels:
containers: dind
namespace: oss-agent
labels:
containers: dind
spec:
nodeSelector:
eks.amazonaws.com/nodegroup: eks-oss-presto-dynamic-managed-ng
serviceAccountName: oss-agent
containers:
- name: dind
image: docker:20.10.16-dind-alpine3.15
securityContext:
privileged: true
tty: true
resources:
requests:
memory: "4Gi"
cpu: "2000m"
limits:
memory: "4Gi"
cpu: "2000m"
nodeSelector:
eks.amazonaws.com/nodegroup: eks-oss-presto-dynamic-managed-ng
serviceAccountName: oss-agent
containers:
- name: dind
image: docker:20.10.16-dind-alpine3.15
securityContext:
privileged: true
tty: true
resources:
requests:
memory: "24Gi"
cpu: "7000m"
limits:
memory: "24Gi"
cpu: "7000m"
45 changes: 23 additions & 22 deletions jenkins/agent-maven.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,26 +1,27 @@
apiVersion: v1
kind: Pod
metadata:
namespace: oss-agent
labels:
containers: maven
namespace: oss-agent
labels:
containers: maven
spec:
nodeSelector:
eks.amazonaws.com/nodegroup: eks-oss-presto-dynamic-managed-ng
serviceAccountName: oss-agent
containers:
- name: maven
image: maven:3.8.6-openjdk-8-slim
env:
- name: MAVEN_OPTS
value: "-Xmx8000m -Xms8000m"
resources:
requests:
memory: "10Gi"
cpu: "4000m"
limits:
memory: "10Gi"
cpu: "4000m"
tty: true
command:
- cat
nodeSelector:
eks.amazonaws.com/nodegroup: eks-oss-presto-dynamic-managed-ng
serviceAccountName: oss-agent
containers:
- name: maven
image: maven:3.8.6-openjdk-8-slim
env:
- name: MAVEN_OPTS
value: "-Xmx8000m -Xms8000m"
resources:
requests:
memory: "10Gi"
cpu: "4000m"
limits:
memory: "10Gi"
cpu: "4000m"
tty: true
command:
- cat

4 changes: 3 additions & 1 deletion ...chto-benchmarks/src/test/java/com/facebook/presto/sql/planner/TestTpcdsCostBasedPlan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.util.stream.IntStream;
import java.util.stream.Stream;

import static com.facebook.presto.SystemSessionProperties.GENERATE_DOMAIN_FILTERS;
import static com.facebook.presto.SystemSessionProperties.HANDLE_COMPLEX_EQUI_JOINS;
import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE;
import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY;
Expand Down Expand Up @@ -56,7 +57,8 @@ public TestTpcdsCostBasedPlan()
.setSystemProperty(JOIN_REORDERING_STRATEGY, JoinReorderingStrategy.AUTOMATIC.name())
.setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name())
.setSystemProperty(OPTIMIZE_JOINS_WITH_EMPTY_SOURCES, "false")
.setSystemProperty(HANDLE_COMPLEX_EQUI_JOINS, "true");
.setSystemProperty(HANDLE_COMPLEX_EQUI_JOINS, "true")
.setSystemProperty(GENERATE_DOMAIN_FILTERS, "true");

LocalQueryRunner queryRunner = LocalQueryRunner.queryRunnerWithFakeNodeCountForStats(sessionBuilder.build(), 8);
queryRunner.createCatalog(
Expand Down
4 changes: 3 additions & 1 deletion ...nchto-benchmarks/src/test/java/com/facebook/presto/sql/planner/TestTpchCostBasedPlan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.stream.IntStream;
import java.util.stream.Stream;

import static com.facebook.presto.SystemSessionProperties.GENERATE_DOMAIN_FILTERS;
import static com.facebook.presto.SystemSessionProperties.HANDLE_COMPLEX_EQUI_JOINS;
import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE;
import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY;
Expand Down Expand Up @@ -56,7 +57,8 @@ public TestTpchCostBasedPlan()
.setSystemProperty("task_concurrency", "1") // these tests don't handle exchanges from local parallel
.setSystemProperty(JOIN_REORDERING_STRATEGY, JoinReorderingStrategy.AUTOMATIC.name())
.setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name())
.setSystemProperty(HANDLE_COMPLEX_EQUI_JOINS, "true");
.setSystemProperty(HANDLE_COMPLEX_EQUI_JOINS, "true")
.setSystemProperty(GENERATE_DOMAIN_FILTERS, "true");

LocalQueryRunner queryRunner = LocalQueryRunner.queryRunnerWithFakeNodeCountForStats(sessionBuilder.build(), 8);
queryRunner.createCatalog(
Expand Down
19 changes: 9 additions & 10 deletions presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q13.plan.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,22 +5,21 @@ final aggregation over ()
join (INNER, REPLICATED):
join (INNER, REPLICATED):
join (INNER, REPLICATED):
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, [ss_addr_sk])
join (INNER, REPLICATED):
scan store_sales
local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan date_dim
join (INNER, REPLICATED):
join (INNER, REPLICATED):
scan store_sales
local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan customer_address
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, [ca_address_sk])
scan customer_address
remote exchange (REPLICATE, BROADCAST, [])
scan customer_demographics
local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan household_demographics
local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan customer_demographics
scan date_dim
local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan store
19 changes: 9 additions & 10 deletions presto-benchto-benchmarks/src/test/resources/sql/presto/tpcds/q48.plan.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,19 @@ final aggregation over ()
remote exchange (GATHER, SINGLE, [])
partial aggregation over ()
join (INNER, REPLICATED):
join (INNER, PARTITIONED):
remote exchange (REPARTITION, HASH, [ss_addr_sk])
join (INNER, REPLICATED):
join (INNER, REPLICATED):
join (INNER, REPLICATED):
join (INNER, REPLICATED):
scan store_sales
local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan customer_demographics
scan store_sales
local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan date_dim
scan customer_demographics
local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan customer_address
local exchange (GATHER, SINGLE, [])
remote exchange (REPARTITION, HASH, [ca_address_sk])
scan customer_address
remote exchange (REPLICATE, BROADCAST, [])
scan date_dim
local exchange (GATHER, SINGLE, [])
remote exchange (REPLICATE, BROADCAST, [])
scan store
Loading