Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
The diff you're trying to view is too large. We only load the first 3000 changed files.
18 changes: 18 additions & 0 deletions .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
FROM ubuntu:24.04

# Install necessary tools
RUN apt-get update && apt-get install -y \
# Intellij IDEA dev container prerequisites
curl \
git \
unzip \
# Java 8 and 17 jdk
openjdk-8-jdk \
openjdk-17-jdk \
# for documentation
python3 \
python3-pip \
python3.12-venv \
fonts-freefont-otf \
xindy

6 changes: 6 additions & 0 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"name": "PrestoDB Dev Container",
"build": {
"dockerfile": "Dockerfile"
}
}
8 changes: 4 additions & 4 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ env:
concurrency:
group: "${{github.workflow}}-${{github.ref}}"

# Cancel in-progress jobs for efficiency. Exclude the `release-0.293-clp-connector` branch so
# that each commit to release-0.293-clp-connector is checked completely.
cancel-in-progress: "${{github.ref != 'refs/heads/release-0.293-clp-connector'}}"
# Cancel in-progress jobs for efficiency. Exclude the `release-0.297-edge-10-clp-connector` branch so
# that each commit to release-0.297-edge-10-clp-connector is checked completely.
cancel-in-progress: "${{github.ref != 'refs/heads/release-0.297-edge-10-clp-connector'}}"

jobs:
test:
Expand All @@ -44,7 +44,7 @@ jobs:
- uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: 8.0.442
java-version: 17.0.13
cache: 'maven'
- name: Maven Install
run: |
Expand Down
42 changes: 32 additions & 10 deletions .github/workflows/maven-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,16 @@ env:
concurrency:
group: "${{github.workflow}}-${{github.ref}}"

# Cancel in-progress jobs for efficiency. Exclude the `release-0.293-clp-connector` branch so
# that each commit to release-0.293-clp-connector is checked completely.
cancel-in-progress: "${{github.ref != 'refs/heads/release-0.293-clp-connector'}}"
# Cancel in-progress jobs for efficiency. Exclude the `release-0.297-edge-10-clp-connector` branch so
# that each commit to release-0.297-edge-10-clp-connector is checked completely.
cancel-in-progress: "${{github.ref != 'refs/heads/release-0.297-edge-10-clp-connector'}}"

jobs:
maven-checks:
strategy:
fail-fast: false
matrix:
java: [ 8.0.442, 17.0.13 ]
java: [ 17.0.13 ]
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
Expand All @@ -47,24 +47,46 @@ jobs:
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
./mvnw install -B -V -T 1C -DskipTests -Dmaven.javadoc.skip=true --no-transfer-progress -P ci -pl '!presto-test-coverage,!:presto-docs'
- name: "Upload presto-server"
if: matrix.java == '8.0.442'
uses: "actions/upload-artifact@v4"
with:
name: "presto-server"
path: "presto-server/target/presto-server-0.293.tar.gz"
path: "presto-server/target/presto-server-0.297-edge10.1-SNAPSHOT.tar.gz"
if-no-files-found: "error"
retention-days: 1
- name: "Upload presto-cli"
if: matrix.java == '8.0.442'
uses: "actions/upload-artifact@v4"
with:
name: "presto-cli"
path: "presto-cli/target/presto-cli-0.293-executable.jar"
path: "presto-cli/target/presto-cli-0.297-edge10.1-SNAPSHOT-executable.jar"
if-no-files-found: "error"
retention-days: 1
- name: "Clean Maven output"
run: "./mvnw clean -pl '!:presto-server,!:presto-cli,!presto-test-coverage'"

clp-connector-unit-tests:
name: "clp-connector-unit-tests"
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
with:
show-progress: false
- uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: 17.0.13
cache: 'maven'
- name: Download nodejs to maven cache
run: .github/bin/download_nodejs
- name: Install presto-clp dependencies
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
./mvnw install -B -V -T 1C -DskipTests -Dmaven.javadoc.skip=true --no-transfer-progress -am -pl 'presto-clp'
- name: Run presto-clp unit tests
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
./mvnw test -B --no-transfer-progress -pl 'presto-clp'

presto-coordinator-image:
name: "presto-coordinator-image"
needs: "maven-checks"
Expand Down Expand Up @@ -105,11 +127,11 @@ jobs:
with:
build-args: |-
JMX_PROMETHEUS_JAVA_AGENT_VERSION=0.20.0
PRESTO_VERSION=0.293
PRESTO_VERSION=0.297-edge10.1-SNAPSHOT
context: "./docker"
file: "./docker/Dockerfile"
push: >-
${{github.event_name != 'pull_request'
&& github.ref == 'refs/heads/release-0.293-clp-connector'}}
&& github.ref == 'refs/heads/release-0.297-edge-10-clp-connector'}}
tags: "${{steps.meta.outputs.tags}}"
labels: "${{steps.meta.outputs.labels}}"
2 changes: 1 addition & 1 deletion .github/workflows/pr-title-checks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:
# pull request triggered by this event.
# - Each job has `permissions` set to only those necessary.
types: ["edited", "opened", "reopened"]
branches: ["release-0.293-clp-connector"]
branches: ["release-0.297-edge-10-clp-connector"]

permissions: {}

Expand Down
8 changes: 3 additions & 5 deletions .github/workflows/prestissimo-worker-images-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
file: "./presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile"
push: >-
${{github.event_name != 'pull_request'
&& github.ref == 'refs/heads/release-0.293-clp-connector'}}
&& github.ref == 'refs/heads/release-0.297-edge-10-clp-connector'}}
tags: "${{steps.metadata-deps-image.outputs.tags}}"
labels: "${{steps.metadata-deps-image.outputs.labels}}"

Expand All @@ -56,15 +56,13 @@ jobs:
build-args: |-
BASE_IMAGE=ubuntu:22.04
DEPENDENCY_IMAGE=${{steps.metadata-deps-image.outputs.tags}}
EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF \
-DPRESTO_ENABLE_PARQUET=ON \
-DPRESTO_ENABLE_S3=ON
EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF -DPRESTO_ENABLE_PARQUET=ON -DPRESTO_ENABLE_S3=ON -DTREAT_WARNINGS_AS_ERRORS=0
NUM_THREADS=${{steps.get-cores.outputs.num_cores}}
OSNAME=ubuntu
context: "./presto-native-execution"
file: "./presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile"
push: >-
${{github.event_name != 'pull_request'
&& github.ref == 'refs/heads/release-0.293-clp-connector'}}
&& github.ref == 'refs/heads/release-0.297-edge-10-clp-connector'}}
tags: "${{steps.metadata-runtime-image.outputs.tags}}"
labels: "${{steps.metadata-runtime-image.outputs.labels}}"
6 changes: 3 additions & 3 deletions .github/workflows/prestocpp-format-and-header-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ on:
concurrency:
group: "${{github.workflow}}-${{github.ref}}"

# Cancel in-progress jobs for efficiency. Exclude the `release-0.293-clp-connector` branch so
# that each commit to release-0.293-clp-connector is checked completely.
cancel-in-progress: "${{github.ref != 'refs/heads/release-0.293-clp-connector'}}"
# Cancel in-progress jobs for efficiency. Exclude the `release-0.297-edge-10-clp-connector` branch so
# that each commit to release-0.297-edge-10-clp-connector is checked completely.
cancel-in-progress: "${{github.ref != 'refs/heads/release-0.297-edge-10-clp-connector'}}"

jobs:
prestocpp-format-and-header-check:
Expand Down
171 changes: 12 additions & 159 deletions .github/workflows/prestocpp-linux-build-and-unit-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,15 @@ on:
concurrency:
group: "${{github.workflow}}-${{github.ref}}"

# Cancel in-progress jobs for efficiency. Exclude the `release-0.293-clp-connector` branch so
# that each commit to release-0.293-clp-connector is checked completely.
cancel-in-progress: "${{github.ref != 'refs/heads/release-0.293-clp-connector'}}"
# Cancel in-progress jobs for efficiency. Exclude the `release-0.297-edge-10-clp-connector` branch so
# that each commit to release-0.297-edge-10-clp-connector is checked completely.
cancel-in-progress: "${{github.ref != 'refs/heads/release-0.297-edge-10-clp-connector'}}"

jobs:
prestocpp-linux-build-for-test:
runs-on: ubuntu-22.04
container:
image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e
image: prestodb/presto-native-dependency:0.297-202512180933-75d7d4ea
env:
CCACHE_DIR: "${{ github.workspace }}/ccache"
steps:
Expand Down Expand Up @@ -99,7 +99,7 @@ jobs:
needs: prestocpp-linux-build-for-test
runs-on: ubuntu-22.04
container:
image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e
image: prestodb/presto-native-dependency:0.297-202512180933-75d7d4ea
env:
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
Expand All @@ -112,6 +112,11 @@ jobs:
# it doesn't work
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}

- name: Update velox
run: |
cd presto-native-execution
make velox-submodule

- name: Download artifacts
uses: actions/download-artifact@v4
with:
Expand All @@ -130,7 +135,7 @@ jobs:
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: 8.0.442
java-version: 17.0.13
cache: 'maven'
- name: Download nodejs to maven cache
run: .github/bin/download_nodejs
Expand All @@ -145,17 +150,7 @@ jobs:
- name: Run presto-native e2e tests
run: |
export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoNative*.java'`
# Convert file paths to comma separated class names
export TESTCLASSES=
for test_file in $TESTFILES
do
tmp=${test_file##*/}
test_class=${tmp%%\.*}
export TESTCLASSES="${TESTCLASSES},$test_class"
done
export TESTCLASSES=${TESTCLASSES#,}
echo "TESTCLASSES = $TESTCLASSES"
export TESTCLASSES=TestPrestoNativeClpGeneralQueries
# TODO: neeed to enable remote function tests with
# "-Ppresto-native-execution-remote-functions" once
# > https://github.com/facebookincubator/velox/discussions/6163
Expand All @@ -170,145 +165,3 @@ jobs:
-Duser.timezone=America/Bahia_Banderas \
-T1C

prestocpp-linux-presto-native-tests:
needs: prestocpp-linux-build-for-test
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
storage-format: [ "PARQUET", "DWRF" ]
container:
image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e
env:
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
steps:
- uses: actions/checkout@v4

- name: Fix git permissions
# Usually actions/checkout does this but as we run in a container
# it doesn't work
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}

- name: Download artifacts
uses: actions/download-artifact@v4
with:
name: presto-native-build
path: presto-native-execution/_build/release

# Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38
- name: Restore execute permissions and library path
run: |
chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server
chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main
# Ensure transitive dependency libboost-iostreams is found.
ldconfig /usr/local/lib

- name: Install OpenJDK8
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '8.0.442'
cache: 'maven'
- name: Download nodejs to maven cache
run: .github/bin/download_nodejs

- name: Maven install
env:
# Use different Maven options to install.
MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
run: |
for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-tests' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)

- name: Run presto-native tests
run: |
export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
export TESTFILES=`find ./presto-native-tests/src/test -type f -name 'Test*.java'`
# Convert file paths to comma separated class names
export TESTCLASSES=
for test_file in $TESTFILES
do
tmp=${test_file##*/}
test_class=${tmp%%\.*}
export TESTCLASSES="${TESTCLASSES},$test_class"
done
export TESTCLASSES=${TESTCLASSES#,}
echo "TESTCLASSES = $TESTCLASSES"

mvn test \
${MAVEN_TEST} \
-pl 'presto-native-tests' \
-DstorageFormat=${{ matrix.storage-format }} \
-Dtest="${TESTCLASSES}" \
-DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
-DDATA_DIR=${RUNNER_TEMP} \
-Duser.timezone=America/Bahia_Banderas \
-T1C

prestocpp-linux-presto-sidecar-tests:
needs: prestocpp-linux-build-for-test
runs-on: ubuntu-22.04
container:
image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e
env:
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
steps:
- uses: actions/checkout@v4
- name: Fix git permissions
# Usually actions/checkout does this but as we run in a container
# it doesn't work
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}

- name: Download artifacts
uses: actions/download-artifact@v4
with:
name: presto-native-build
path: presto-native-execution/_build/release

# Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38
- name: Restore execute permissions and library path
run: |
chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server
chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main
# Ensure transitive dependency libboost-iostreams is found.
ldconfig /usr/local/lib
- name: Install OpenJDK8
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '8.0.442'
cache: 'maven'
- name: Download nodejs to maven cache
run: .github/bin/download_nodejs

- name: Maven install
env:
# Use different Maven options to install.
MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
run: |
for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
- name: Run presto-native sidecar tests
run: |
export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
export TESTFILES=`find ./presto-native-sidecar-plugin/src/test -type f -name 'Test*.java'`
# Convert file paths to comma separated class names
export TESTCLASSES=
for test_file in $TESTFILES
do
tmp=${test_file##*/}
test_class=${tmp%%\.*}
export TESTCLASSES="${TESTCLASSES},$test_class"
done
export TESTCLASSES=${TESTCLASSES#,}
echo "TESTCLASSES = $TESTCLASSES"
mvn test \
${MAVEN_TEST} \
-pl 'presto-native-sidecar-plugin' \
-Dtest="${TESTCLASSES}" \
-DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
-DDATA_DIR=${RUNNER_TEMP} \
-Duser.timezone=America/Bahia_Banderas \
-T1C
Loading