Add CI workflow to test against Iceberg unreleased versions

.github/workflows/nightly-iceberg.yml

@@ -0,0 +1,120 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Runs smoke tests against Apache Iceberg nightly builds to detect compatibility issues early.
+
+name: Iceberg Nightly Smoke Tests
+
+on:
+  schedule:
+    # Run daily at 3:00 AM UTC
+    - cron: '0 3 * * *'
+  workflow_dispatch:
+    inputs:
+      iceberg_version:
+        description: 'Iceberg version to test (e.g., 1.11.0-SNAPSHOT). Leave empty to auto-detect latest snapshot.'
+        required: false
+        type: string
+      branch:
+        description: 'Branch to build'
+        required: false
+        default: 'main'
+        type: string
+
+# Setting explicit permissions for the action
+permissions:
+  actions: read
+  contents: read
+  issues: read
+
+env:
+  GRADLE_TOS_ACCEPTED: ${{ vars.GRADLE_TOS_ACCEPTED }}
+  DEVELOCITY_SERVER: ${{ vars.DEVELOCITY_SERVER }}
+  DEVELOCITY_PROJECT_ID: ${{ vars.DEVELOCITY_PROJECT_ID }}
+
+jobs:
+  smoke-tests:
+    name: Smoke Tests (Iceberg Nightly)
+    runs-on: ubuntu-latest
+    if: github.repository == 'apache/polaris'
+    steps:
+      - name: Detect latest Iceberg snapshot version
+        id: detect
+        run: |
+          if [ -n "${{ inputs.iceberg_version }}" ]; then
+            echo "Using provided Iceberg version: ${{ inputs.iceberg_version }}"
+            echo "iceberg_version=${{ inputs.iceberg_version }}" >> $GITHUB_OUTPUT
+          else
+            echo "Detecting latest Iceberg snapshot version from Apache Maven repository..."
+            # Fetch the maven-metadata.xml to find the latest snapshot version
+            METADATA_URL="https://repository.apache.org/content/repositories/snapshots/org/apache/iceberg/iceberg-api/maven-metadata.xml"
+            LATEST_VERSION=$(curl -s "$METADATA_URL" | grep -oP '(?<=<latest>)[^<]+' | head -1)
+            if [ -z "$LATEST_VERSION" ]; then
+              echo "Failed to detect Iceberg snapshot version"
+              exit 1
+            fi
+            echo "Detected Iceberg snapshot version: $LATEST_VERSION"
+            echo "iceberg_version=$LATEST_VERSION" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Checkout
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
+        with:
+          ref: ${{ inputs.branch || 'main' }}
+
+      - name: Set up JDK 21
+        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
+        with:
+          java-version: '21'
+          distribution: 'temurin'
+
+      - name: Setup test environment
+        uses: ./.github/actions/setup-test-env
+
+      - name: Prepare Gradle build cache
+        uses: ./.github/actions/ci-incr-build-cache-prepare
+
+      - name: Override Iceberg version in version catalog
+        run: |
+          ICEBERG_VERSION="${{ steps.detect.outputs.iceberg_version }}"
+          echo "Overriding Iceberg version to: $ICEBERG_VERSION"
+          sed -i "s/^iceberg = .*/iceberg = \"$ICEBERG_VERSION\"/" gradle/libs.versions.toml
+          echo "Updated gradle/libs.versions.toml:"
+          grep "^iceberg = " gradle/libs.versions.toml
+
+      - name: Run smoke tests with Iceberg nightly
+        env:
+          DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
+        run: |
+          echo "Running smoke tests with Iceberg version: ${{ steps.detect.outputs.iceberg_version }}"
+          ./gradlew \
+            :polaris-core:test \
+            :polaris-api-iceberg-service:test \
+            :polaris-runtime-service:test \
+            -PuseApacheSnapshots=true \
+            --continue
+
+      - name: Archive test results
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        if: failure()
+        with:
+          name: iceberg-nightly-test-results
+          path: |
+            **/build/test-results/**
+            **/build/reports/tests/**

settings.gradle.kts

@@ -125,6 +125,15 @@ dependencyResolutionManagement {
       url = uri("https://jitpack.io")
       content { includeModule("com.github.RoaringBitmap.RoaringBitmap", "roaringbitmap") }
     }
+    val useApacheSnapshots =
+      providers.gradleProperty("useApacheSnapshots").orNull?.toBoolean() == true
+    if (useApacheSnapshots) {
+      maven {
+        name = "ApacheSnapshots"
+        url = uri("https://repository.apache.org/content/repositories/snapshots/")
+        mavenContent { snapshotsOnly() }
+      }
+    }
     gradlePluginPortal()
   }
 }