Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 49 additions & 5 deletions .github/workflows/arrow-flight-tests.yml
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
name: arrow flight tests

on:
pull_request:
paths-ignore:
- presto-docs/**
on: pull_request

permissions:
contents: read
Expand All @@ -17,8 +14,26 @@ env:
RETRY: .github/bin/retry

jobs:
changes:
runs-on: ubuntu-latest
# Required permissions
permissions:
pull-requests: read
# Set job outputs to values from filter step
outputs:
codechange: ${{ steps.filter.outputs.codechange }}
steps:
# For pull requests it's not necessary to checkout the code
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
id: filter
with:
filters: |
codechange:
- '!presto-docs/**'

arrowflight-java-tests:
runs-on: ubuntu-latest
needs: changes
strategy:
fail-fast: false
matrix:
Expand All @@ -34,40 +49,47 @@ jobs:
steps:
# Checkout the code only if there are changes in the relevant files
- uses: actions/checkout@v4
if: needs.changes.outputs.codechange == 'true'
with:
show-progress: false
persist-credentials: false

# Set up Java and dependencies for the build environment
- uses: actions/setup-java@v4
if: needs.changes.outputs.codechange == 'true'
with:
distribution: temurin
java-version: ${{ matrix.java }}
cache: maven

# Cleanup before build
- name: Clean up before build
if: needs.changes.outputs.codechange == 'true'
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
df -h
docker system prune -af || true

- name: Download nodejs to maven cache
if: needs.changes.outputs.codechange == 'true'
run: .github/bin/download_nodejs

# Install dependencies for the target module
- name: Maven Install
if: needs.changes.outputs.codechange == 'true'
run: |
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
./mvnw install ${MAVEN_FAST_INSTALL} -e -am -pl ${{ matrix.modules }}

# Run Maven tests for the target module, excluding native tests
- name: Maven Tests
if: needs.changes.outputs.codechange == 'true'
run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }} -Dtest="*,!TestArrowFlightNativeQueries*"

prestocpp-linux-build-for-test:
runs-on: ubuntu-22.04
needs: changes
container:
image: prestodb/presto-native-dependency:0.297-202602271419-160459b8
volumes:
Expand Down Expand Up @@ -101,20 +123,24 @@ jobs:
echo "New available disk space: " $(getAvailableSpace)

- uses: actions/checkout@v4
if: needs.changes.outputs.codechange == 'true'
with:
persist-credentials: false

- name: Fix git permissions
if: needs.changes.outputs.codechange == 'true'
# Usually actions/checkout does this but as we run in a container
# it doesn't work
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}

- name: Update velox
if: needs.changes.outputs.codechange == 'true'
run: |
cd presto-native-execution
make velox-submodule

- name: Install Arrow Flight
if: needs.changes.outputs.codechange == 'true'
run: |
mkdir -p ${DEPENDENCY_DIR}/adapter-deps/download
mkdir -p ${INSTALL_PREFIX}/adapter-deps/install
Expand All @@ -124,19 +150,23 @@ jobs:
PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh arrow_flight

- name: Install Github CLI for using apache/infrastructure-actions/stash
if: needs.changes.outputs.codechange == 'true'
run: |
curl -L https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.rpm > gh_2.63.2_linux_amd64.rpm
rpm -iv gh_2.63.2_linux_amd64.rpm

- uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
if: needs.changes.outputs.codechange == 'true'
with:
path: '${{ env.CCACHE_DIR }}'
key: ccache-prestocpp-linux-build-for-test

- name: Zero ccache statistics
if: needs.changes.outputs.codechange == 'true'
run: ccache -sz

- name: Build engine
if: needs.changes.outputs.codechange == 'true'
run: |
source /opt/rh/gcc-toolset-12/enable
cd presto-native-execution
Expand All @@ -155,32 +185,37 @@ jobs:
ninja -C _build/release -j 4

- name: Ccache after
if: needs.changes.outputs.codechange == 'true'
run: ccache -s

- uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
if: needs.changes.outputs.codechange == 'true'
with:
path: '${{ env.CCACHE_DIR }}'
key: ccache-prestocpp-linux-build-for-test

- name: Run Unit Tests for the Arrow Flight connector only
if: needs.changes.outputs.codechange == 'true'
run: |
cd presto-native-execution/_build/release
ctest -j 4 -VV --output-on-failure --tests-regex ^presto_flight.*

- name: Upload artifacts
if: needs.changes.outputs.codechange == 'true'
uses: actions/upload-artifact@v4
with:
name: arrow-flight-presto-native-build
path: presto-native-execution/_build/release/presto_cpp/main/presto_server

- name: Upload Arrow Flight install artifacts
if: needs.changes.outputs.codechange == 'true'
uses: actions/upload-artifact@v4
with:
name: arrow-flight-install
path: ${{ env.INSTALL_PREFIX }}/lib64/libarrow_flight*

arrowflight-native-e2e-tests:
needs: prestocpp-linux-build-for-test
needs: [changes, prestocpp-linux-build-for-test]
runs-on: ubuntu-22.04
container:
image: prestodb/presto-native-dependency:0.297-202602271419-160459b8
Expand Down Expand Up @@ -219,43 +254,51 @@ jobs:
echo "New available disk space: " $(getAvailableSpace)

- uses: actions/checkout@v4
if: needs.changes.outputs.codechange == 'true'
with:
persist-credentials: false

- name: Fix git permissions
if: needs.changes.outputs.codechange == 'true'
# Usually actions/checkout does this but as we run in a container
# it doesn't work
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}

- name: Download artifacts
if: needs.changes.outputs.codechange == 'true'
uses: actions/download-artifact@v4
with:
name: arrow-flight-presto-native-build
path: presto-native-execution/_build/release/presto_cpp/main

- name: Download Arrow Flight install artifacts
if: needs.changes.outputs.codechange == 'true'
uses: actions/download-artifact@v4
with:
name: arrow-flight-install
path: ${{ env.INSTALL_PREFIX }}/lib64

# Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38
- name: Restore execute permissions and library path
if: needs.changes.outputs.codechange == 'true'
run: |
chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server
# Ensure transitive dependency libboost-iostreams is found.
ldconfig /usr/local/lib

- name: Install OpenJDK8
if: needs.changes.outputs.codechange == 'true'
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: '17'
cache: maven
- name: Download nodejs to maven cache
if: needs.changes.outputs.codechange == 'true'
run: .github/bin/download_nodejs

- name: Maven install
if: needs.changes.outputs.codechange == 'true'
env:
# Use different Maven options to install.
MAVEN_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError
Expand All @@ -264,6 +307,7 @@ jobs:
./mvnw install ${MAVEN_FAST_INSTALL} -am -pl ${{ matrix.modules }}

- name: Run arrowflight native e2e tests
if: needs.changes.outputs.codechange == 'true'
run: |
export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
mvn test \
Expand Down
Loading