diff --git a/.github/workflows/arrow-flight-tests.yml b/.github/workflows/arrow-flight-tests.yml index 67290ace0e399..c8dec648623d5 100644 --- a/.github/workflows/arrow-flight-tests.yml +++ b/.github/workflows/arrow-flight-tests.yml @@ -1,9 +1,6 @@ name: arrow flight tests -on: - pull_request: - paths-ignore: - - presto-docs/** +on: pull_request permissions: contents: read @@ -17,8 +14,26 @@ env: RETRY: .github/bin/retry jobs: + changes: + runs-on: ubuntu-latest + # Required permissions + permissions: + pull-requests: read + # Set job outputs to values from filter step + outputs: + codechange: ${{ steps.filter.outputs.codechange }} + steps: + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: filter + with: + filters: | + codechange: + - '!presto-docs/**' + arrowflight-java-tests: runs-on: ubuntu-latest + needs: changes strategy: fail-fast: false matrix: @@ -34,12 +49,14 @@ jobs: steps: # Checkout the code only if there are changes in the relevant files - uses: actions/checkout@v4 + if: needs.changes.outputs.codechange == 'true' with: show-progress: false persist-credentials: false # Set up Java and dependencies for the build environment - uses: actions/setup-java@v4 + if: needs.changes.outputs.codechange == 'true' with: distribution: temurin java-version: ${{ matrix.java }} @@ -47,6 +64,7 @@ jobs: # Cleanup before build - name: Clean up before build + if: needs.changes.outputs.codechange == 'true' run: | sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc sudo rm -rf /opt/hostedtoolcache/CodeQL @@ -54,20 +72,24 @@ jobs: docker system prune -af || true - name: Download nodejs to maven cache + if: needs.changes.outputs.codechange == 'true' run: .github/bin/download_nodejs # Install dependencies for the target module - name: Maven Install + if: needs.changes.outputs.codechange == 'true' run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" ./mvnw install ${MAVEN_FAST_INSTALL} -e -am -pl ${{ matrix.modules }} # Run Maven tests for the target module, excluding native tests - name: Maven Tests + if: needs.changes.outputs.codechange == 'true' run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }} -Dtest="*,!TestArrowFlightNativeQueries*" prestocpp-linux-build-for-test: runs-on: ubuntu-22.04 + needs: changes container: image: prestodb/presto-native-dependency:0.297-202602271419-160459b8 volumes: @@ -101,20 +123,24 @@ jobs: echo "New available disk space: " $(getAvailableSpace) - uses: actions/checkout@v4 + if: needs.changes.outputs.codechange == 'true' with: persist-credentials: false - name: Fix git permissions + if: needs.changes.outputs.codechange == 'true' # Usually actions/checkout does this but as we run in a container # it doesn't work run: git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Update velox + if: needs.changes.outputs.codechange == 'true' run: | cd presto-native-execution make velox-submodule - name: Install Arrow Flight + if: needs.changes.outputs.codechange == 'true' run: | mkdir -p ${DEPENDENCY_DIR}/adapter-deps/download mkdir -p ${INSTALL_PREFIX}/adapter-deps/install @@ -124,19 +150,23 @@ jobs: PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh arrow_flight - name: Install Github CLI for using apache/infrastructure-actions/stash + if: needs.changes.outputs.codechange == 'true' run: | curl -L https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.rpm > gh_2.63.2_linux_amd64.rpm rpm -iv gh_2.63.2_linux_amd64.rpm - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 + if: needs.changes.outputs.codechange == 'true' with: path: '${{ env.CCACHE_DIR }}' key: ccache-prestocpp-linux-build-for-test - name: Zero ccache statistics + if: needs.changes.outputs.codechange == 'true' run: ccache -sz - name: Build engine + if: needs.changes.outputs.codechange == 'true' run: | source /opt/rh/gcc-toolset-12/enable cd presto-native-execution @@ -155,32 +185,37 @@ jobs: ninja -C _build/release -j 4 - name: Ccache after + if: needs.changes.outputs.codechange == 'true' run: ccache -s - uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 + if: needs.changes.outputs.codechange == 'true' with: path: '${{ env.CCACHE_DIR }}' key: ccache-prestocpp-linux-build-for-test - name: Run Unit Tests for the Arrow Flight connector only + if: needs.changes.outputs.codechange == 'true' run: | cd presto-native-execution/_build/release ctest -j 4 -VV --output-on-failure --tests-regex ^presto_flight.* - name: Upload artifacts + if: needs.changes.outputs.codechange == 'true' uses: actions/upload-artifact@v4 with: name: arrow-flight-presto-native-build path: presto-native-execution/_build/release/presto_cpp/main/presto_server - name: Upload Arrow Flight install artifacts + if: needs.changes.outputs.codechange == 'true' uses: actions/upload-artifact@v4 with: name: arrow-flight-install path: ${{ env.INSTALL_PREFIX }}/lib64/libarrow_flight* arrowflight-native-e2e-tests: - needs: prestocpp-linux-build-for-test + needs: [changes, prestocpp-linux-build-for-test] runs-on: ubuntu-22.04 container: image: prestodb/presto-native-dependency:0.297-202602271419-160459b8 @@ -219,21 +254,25 @@ jobs: echo "New available disk space: " $(getAvailableSpace) - uses: actions/checkout@v4 + if: needs.changes.outputs.codechange == 'true' with: persist-credentials: false - name: Fix git permissions + if: needs.changes.outputs.codechange == 'true' # Usually actions/checkout does this but as we run in a container # it doesn't work run: git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Download artifacts + if: needs.changes.outputs.codechange == 'true' uses: actions/download-artifact@v4 with: name: arrow-flight-presto-native-build path: presto-native-execution/_build/release/presto_cpp/main - name: Download Arrow Flight install artifacts + if: needs.changes.outputs.codechange == 'true' uses: actions/download-artifact@v4 with: name: arrow-flight-install @@ -241,21 +280,25 @@ jobs: # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 - name: Restore execute permissions and library path + if: needs.changes.outputs.codechange == 'true' run: | chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server # Ensure transitive dependency libboost-iostreams is found. ldconfig /usr/local/lib - name: Install OpenJDK8 + if: needs.changes.outputs.codechange == 'true' uses: actions/setup-java@v4 with: distribution: temurin java-version: '17' cache: maven - name: Download nodejs to maven cache + if: needs.changes.outputs.codechange == 'true' run: .github/bin/download_nodejs - name: Maven install + if: needs.changes.outputs.codechange == 'true' env: # Use different Maven options to install. MAVEN_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError @@ -264,6 +307,7 @@ jobs: ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl ${{ matrix.modules }} - name: Run arrowflight native e2e tests + if: needs.changes.outputs.codechange == 'true' run: | export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" mvn test \