feat(lance): Add configurable index and metadata cache sizes

.github/workflows/arrow-flight-tests.yml

@@ -1,9 +1,6 @@
 name: arrow flight tests
 
-on:
-  pull_request:
-    paths-ignore:
-      - presto-docs/**
+on: pull_request
 
 permissions:
   contents: read
@@ -17,12 +14,30 @@ env:
   RETRY: .github/bin/retry
 
 jobs:
+  changes:
+    runs-on: ubuntu-latest
+    # Required permissions
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step
+    outputs:
+      codechange: ${{ steps.filter.outputs.codechange }}
+    steps:
+      # For pull requests it's not necessary to checkout the code
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
+        id: filter
+        with:
+          filters: |
+            codechange:
+              - '!presto-docs/**'
+
   arrowflight-java-tests:
     runs-on: ubuntu-latest
+    needs: changes
     strategy:
       fail-fast: false
       matrix:
-        java: [17.0.15]
+        java: ['17']
         modules:
           - :presto-base-arrow-flight # Only run tests for the `presto-base-arrow-flight` module
 
@@ -34,33 +49,52 @@ jobs:
     steps:
       # Checkout the code only if there are changes in the relevant files
       - uses: actions/checkout@v4
+        if: needs.changes.outputs.codechange == 'true'
         with:
           show-progress: false
           persist-credentials: false
 
       # Set up Java and dependencies for the build environment
       - uses: actions/setup-java@v4
+        if: needs.changes.outputs.codechange == 'true'
         with:
           distribution: temurin
           java-version: ${{ matrix.java }}
           cache: maven
+
+      # Cleanup before build
+      - name: Clean up before build
+        if: needs.changes.outputs.codechange == 'true'
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          df -h
+          docker system prune -af || true
+
       - name: Download nodejs to maven cache
+        if: needs.changes.outputs.codechange == 'true'
         run: .github/bin/download_nodejs
 
       # Install dependencies for the target module
       - name: Maven Install
+        if: needs.changes.outputs.codechange == 'true'
         run: |
           export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
           ./mvnw install ${MAVEN_FAST_INSTALL} -e -am -pl ${{ matrix.modules }}
 
       # Run Maven tests for the target module, excluding native tests
       - name: Maven Tests
+        if: needs.changes.outputs.codechange == 'true'
         run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }} -Dtest="*,!TestArrowFlightNativeQueries*"
 
   prestocpp-linux-build-for-test:
     runs-on: ubuntu-22.04
+    needs: changes
     container:
-      image: prestodb/presto-native-dependency:0.292-20250204112033-cf8ba84
+      image: prestodb/presto-native-dependency:0.297-202602271419-160459b8
+      volumes:
+        - /usr:/host_usr
+        - /opt:/host_opt
     concurrency:
       group: ${{ github.workflow }}-prestocpp-linux-build-for-test-${{ github.event.pull_request.number }}
       cancel-in-progress: true
@@ -71,21 +105,42 @@ jobs:
     permissions:
       actions: write
     steps:
+      # We cannot use the github action to free disk space from the runner
+      # because we are in the container and not on the runner anymore.
+      - name: Free Disk Space
+        run: |
+          # Re-used from free-disk-space github action.
+          getAvailableSpace() { echo $(df -a $1 | awk 'NR > 1 {avail+=$4} END {print avail}'); }
+          # Show before
+          echo "Original available disk space: " $(getAvailableSpace)
+          # Remove DotNet.
+          rm -rf /host_usr/share/dotnet || true
+          # Remove android
+          rm -rf /host_usr/local/lib/android || true
+          # Remove CodeQL
+          rm -rf /host_opt/hostedtoolcache/CodeQL || true
+          # Show after
+          echo "New available disk space: " $(getAvailableSpace)
+
       - uses: actions/checkout@v4
+        if: needs.changes.outputs.codechange == 'true'
         with:
           persist-credentials: false
 
       - name: Fix git permissions
+        if: needs.changes.outputs.codechange == 'true'
         # Usually actions/checkout does this but as we run in a container
         # it doesn't work
         run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
 
       - name: Update velox
+        if: needs.changes.outputs.codechange == 'true'
         run: |
           cd presto-native-execution
           make velox-submodule
 
       - name: Install Arrow Flight
+        if: needs.changes.outputs.codechange == 'true'
         run: |
           mkdir -p ${DEPENDENCY_DIR}/adapter-deps/download
           mkdir -p ${INSTALL_PREFIX}/adapter-deps/install
@@ -95,19 +150,23 @@ jobs:
           PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh arrow_flight
 
       - name: Install Github CLI for using apache/infrastructure-actions/stash
+        if: needs.changes.outputs.codechange == 'true'
         run: |
           curl -L https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.rpm > gh_2.63.2_linux_amd64.rpm
           rpm -iv gh_2.63.2_linux_amd64.rpm
 
       - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
+        if: needs.changes.outputs.codechange == 'true'
         with:
           path: '${{ env.CCACHE_DIR }}'
           key: ccache-prestocpp-linux-build-for-test
 
       - name: Zero ccache statistics
+        if: needs.changes.outputs.codechange == 'true'
         run: ccache -sz
 
       - name: Build engine
+        if: needs.changes.outputs.codechange == 'true'
         run: |
           source /opt/rh/gcc-toolset-12/enable
           cd presto-native-execution
@@ -122,39 +181,47 @@ jobs:
             -DCMAKE_PREFIX_PATH=/usr/local \
             -DThrift_ROOT=/usr/local \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-            -DMAX_LINK_JOBS=4
+            -DMAX_LINK_JOBS=3
           ninja -C _build/release -j 4
 
       - name: Ccache after
+        if: needs.changes.outputs.codechange == 'true'
         run: ccache -s
 
       - uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
+        if: needs.changes.outputs.codechange == 'true'
         with:
           path: '${{ env.CCACHE_DIR }}'
           key: ccache-prestocpp-linux-build-for-test
 
       - name: Run Unit Tests for the Arrow Flight connector only
+        if: needs.changes.outputs.codechange == 'true'
         run: |
           cd presto-native-execution/_build/release
           ctest -j 4 -VV --output-on-failure --tests-regex ^presto_flight.*
 
       - name: Upload artifacts
+        if: needs.changes.outputs.codechange == 'true'
         uses: actions/upload-artifact@v4
         with:
           name: arrow-flight-presto-native-build
           path: presto-native-execution/_build/release/presto_cpp/main/presto_server
 
       - name: Upload Arrow Flight install artifacts
+        if: needs.changes.outputs.codechange == 'true'
         uses: actions/upload-artifact@v4
         with:
           name: arrow-flight-install
           path: ${{ env.INSTALL_PREFIX }}/lib64/libarrow_flight*
 
   arrowflight-native-e2e-tests:
-    needs: prestocpp-linux-build-for-test
+    needs: [changes, prestocpp-linux-build-for-test]
     runs-on: ubuntu-22.04
     container:
-      image: prestodb/presto-native-dependency:0.292-20250204112033-cf8ba84
+      image: prestodb/presto-native-dependency:0.297-202602271419-160459b8
+      volumes:
+        - /usr:/host_usr
+        - /opt:/host_opt
     env:
       INSTALL_PREFIX: "${{ github.workspace }}/adapter-deps/install"
     strategy:
@@ -169,44 +236,69 @@ jobs:
       cancel-in-progress: true
 
     steps:
+      # We cannot use the github action to free disk space from the runner
+      # because we are in the container and not on the runner anymore.
+      - name: Free Disk Space
+        run: |
+          # Re-used from free-disk-space github action.
+          getAvailableSpace() { echo $(df -a $1 | awk 'NR > 1 {avail+=$4} END {print avail}'); }
+          # Show before
+          echo "Original available disk space: " $(getAvailableSpace)
+          # Remove DotNet.
+          rm -rf /host_usr/share/dotnet || true
+          # Remove android
+          rm -rf /host_usr/local/lib/android || true
+          # Remove CodeQL
+          rm -rf /host_opt/hostedtoolcache/CodeQL || true
+          # Show after
+          echo "New available disk space: " $(getAvailableSpace)
+
       - uses: actions/checkout@v4
+        if: needs.changes.outputs.codechange == 'true'
         with:
           persist-credentials: false
 
       - name: Fix git permissions
+        if: needs.changes.outputs.codechange == 'true'
         # Usually actions/checkout does this but as we run in a container
         # it doesn't work
         run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
 
       - name: Download artifacts
+        if: needs.changes.outputs.codechange == 'true'
         uses: actions/download-artifact@v4
         with:
           name: arrow-flight-presto-native-build
           path: presto-native-execution/_build/release/presto_cpp/main
 
       - name: Download Arrow Flight install artifacts
+        if: needs.changes.outputs.codechange == 'true'
         uses: actions/download-artifact@v4
         with:
           name: arrow-flight-install
           path: ${{ env.INSTALL_PREFIX }}/lib64
 
       # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38
       - name: Restore execute permissions and library path
+        if: needs.changes.outputs.codechange == 'true'
         run: |
           chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server
           # Ensure transitive dependency libboost-iostreams is found.
           ldconfig /usr/local/lib
 
       - name: Install OpenJDK8
+        if: needs.changes.outputs.codechange == 'true'
         uses: actions/setup-java@v4
         with:
           distribution: temurin
-          java-version: 17.0.15
+          java-version: '17'
           cache: maven
       - name: Download nodejs to maven cache
+        if: needs.changes.outputs.codechange == 'true'
         run: .github/bin/download_nodejs
 
       - name: Maven install
+        if: needs.changes.outputs.codechange == 'true'
         env:
           # Use different Maven options to install.
           MAVEN_OPTS: -Xmx2G -XX:+ExitOnOutOfMemoryError
@@ -215,6 +307,7 @@ jobs:
           ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl ${{ matrix.modules }}
 
       - name: Run arrowflight native e2e tests
+        if: needs.changes.outputs.codechange == 'true'
         run: |
           export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
           mvn test \

.github/workflows/docs.yml

@@ -40,7 +40,7 @@ jobs:
       - uses: actions/setup-java@v4
         with:
           distribution: temurin
-          java-version: 17.0.15
+          java-version: '17'
           cache: maven
       - name: Maven Install
         run: |

.github/workflows/hive-tests.yml

@@ -35,7 +35,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [17.0.15]
+        java: ['17']
     runs-on: ubuntu-latest
     needs: changes
     timeout-minutes: 60
@@ -92,7 +92,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [17.0.15]
+        java: ['17']
     runs-on: ubuntu-latest
     needs: changes
     timeout-minutes: 20

.github/workflows/jdbc-connector-tests.yml

@@ -35,7 +35,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [17.0.15]
+        java: ['17']
     runs-on: ubuntu-latest
     needs: changes
     timeout-minutes: 60

.github/workflows/kudu.yml

@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [17.0.15]
+        java: ['17']
     runs-on: ubuntu-latest
     needs: changes
     timeout-minutes: 60

.github/workflows/maven-checks.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [17.0.15]
+        java: ['17']
     runs-on: ubuntu-latest
     timeout-minutes: 45
     concurrency: