From a031ad2e1e35f86529c73b51bf1c01fd40b82c70 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Sat, 31 Jan 2026 21:02:02 +0100 Subject: [PATCH 1/3] Add CI workflow to test against Iceberg unreleased versions --- .github/workflows/nightly-iceberg.yml | 132 ++++++++++++++++++++++++++ settings.gradle.kts | 9 ++ 2 files changed, 141 insertions(+) create mode 100644 .github/workflows/nightly-iceberg.yml diff --git a/.github/workflows/nightly-iceberg.yml b/.github/workflows/nightly-iceberg.yml new file mode 100644 index 0000000000..c3d4c34341 --- /dev/null +++ b/.github/workflows/nightly-iceberg.yml @@ -0,0 +1,132 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Runs smoke tests against Apache Iceberg nightly builds to detect compatibility issues early. + +name: Iceberg Nightly Smoke Tests + +on: + schedule: + # Run daily at 3:00 AM UTC + - cron: '0 3 * * *' + workflow_dispatch: + inputs: + iceberg_version: + description: 'Iceberg version to test (e.g., 1.11.0-SNAPSHOT). Leave empty to auto-detect latest snapshot.' + required: false + type: string + branch: + description: 'Branch to build' + required: false + default: 'main' + type: string + +# Setting explicit permissions for the action +permissions: + actions: read + contents: read + issues: read + +env: + GRADLE_TOS_ACCEPTED: ${{ vars.GRADLE_TOS_ACCEPTED }} + DEVELOCITY_SERVER: ${{ vars.DEVELOCITY_SERVER }} + DEVELOCITY_PROJECT_ID: ${{ vars.DEVELOCITY_PROJECT_ID }} + +jobs: + detect-iceberg-version: + name: Detect Iceberg Snapshot Version + runs-on: ubuntu-latest + if: github.repository == 'apache/polaris' + outputs: + iceberg_version: ${{ steps.detect.outputs.iceberg_version }} + steps: + - name: Detect latest Iceberg snapshot version + id: detect + run: | + if [ -n "${{ inputs.iceberg_version }}" ]; then + echo "Using provided Iceberg version: ${{ inputs.iceberg_version }}" + echo "iceberg_version=${{ inputs.iceberg_version }}" >> $GITHUB_OUTPUT + else + echo "Detecting latest Iceberg snapshot version from Apache Maven repository..." + # Fetch the maven-metadata.xml to find the latest snapshot version + METADATA_URL="https://repository.apache.org/content/repositories/snapshots/org/apache/iceberg/iceberg-api/maven-metadata.xml" + LATEST_VERSION=$(curl -s "$METADATA_URL" | grep -oP '(?<=)[^<]+' | head -1) + if [ -z "$LATEST_VERSION" ]; then + # Fallback: get the last version from the versions list + LATEST_VERSION=$(curl -s "$METADATA_URL" | grep -oP '(?<=)[^<]+' | grep SNAPSHOT | tail -1) + fi + if [ -z "$LATEST_VERSION" ]; then + echo "Failed to detect Iceberg snapshot version" + exit 1 + fi + echo "Detected Iceberg snapshot version: $LATEST_VERSION" + echo "iceberg_version=$LATEST_VERSION" >> $GITHUB_OUTPUT + fi + + smoke-tests: + name: Smoke Tests (Iceberg ${{ needs.detect-iceberg-version.outputs.iceberg_version }}) + runs-on: ubuntu-latest + needs: detect-iceberg-version + if: github.repository == 'apache/polaris' + steps: + - name: Checkout + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 + with: + ref: ${{ inputs.branch || 'main' }} + + - name: Set up JDK 21 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: + java-version: '21' + distribution: 'temurin' + + - name: Setup test environment + uses: ./.github/actions/setup-test-env + + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + + - name: Override Iceberg version in version catalog + run: | + ICEBERG_VERSION="${{ needs.detect-iceberg-version.outputs.iceberg_version }}" + echo "Overriding Iceberg version to: $ICEBERG_VERSION" + sed -i "s/^iceberg = .*/iceberg = \"$ICEBERG_VERSION\"/" gradle/libs.versions.toml + echo "Updated gradle/libs.versions.toml:" + grep "^iceberg = " gradle/libs.versions.toml + + - name: Run smoke tests with Iceberg nightly + env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + run: | + echo "Running smoke tests with Iceberg version: ${{ needs.detect-iceberg-version.outputs.iceberg_version }}" + ./gradlew \ + :polaris-core:test \ + :polaris-api-iceberg-service:test \ + :polaris-runtime-service:test \ + -PuseApacheSnapshots=true \ + --continue + + - name: Archive test results + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + if: failure() + with: + name: iceberg-nightly-test-results + path: | + **/build/test-results/** + **/build/reports/tests/** diff --git a/settings.gradle.kts b/settings.gradle.kts index 290416f981..bec990713a 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -125,6 +125,15 @@ dependencyResolutionManagement { url = uri("https://jitpack.io") content { includeModule("com.github.RoaringBitmap.RoaringBitmap", "roaringbitmap") } } + val useApacheSnapshots = + providers.gradleProperty("useApacheSnapshots").orNull?.toBoolean() == true + if (useApacheSnapshots) { + maven { + name = "ApacheSnapshots" + url = uri("https://repository.apache.org/content/repositories/snapshots/") + mavenContent { snapshotsOnly() } + } + } gradlePluginPortal() } } From 73ed955d2a96178ed12312a6b50e429aa26f76a9 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Mon, 2 Feb 2026 13:30:43 +0100 Subject: [PATCH 2/3] review: remove unnecessary code --- .github/workflows/nightly-iceberg.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/nightly-iceberg.yml b/.github/workflows/nightly-iceberg.yml index c3d4c34341..750b60d518 100644 --- a/.github/workflows/nightly-iceberg.yml +++ b/.github/workflows/nightly-iceberg.yml @@ -67,10 +67,6 @@ jobs: # Fetch the maven-metadata.xml to find the latest snapshot version METADATA_URL="https://repository.apache.org/content/repositories/snapshots/org/apache/iceberg/iceberg-api/maven-metadata.xml" LATEST_VERSION=$(curl -s "$METADATA_URL" | grep -oP '(?<=)[^<]+' | head -1) - if [ -z "$LATEST_VERSION" ]; then - # Fallback: get the last version from the versions list - LATEST_VERSION=$(curl -s "$METADATA_URL" | grep -oP '(?<=)[^<]+' | grep SNAPSHOT | tail -1) - fi if [ -z "$LATEST_VERSION" ]; then echo "Failed to detect Iceberg snapshot version" exit 1 From d42f7afdeb4e593835f13fc1d83a1a794d6c2db8 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Mon, 2 Feb 2026 13:30:57 +0100 Subject: [PATCH 3/3] review: merge two jobs into one --- .github/workflows/nightly-iceberg.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/nightly-iceberg.yml b/.github/workflows/nightly-iceberg.yml index 750b60d518..e4481f5172 100644 --- a/.github/workflows/nightly-iceberg.yml +++ b/.github/workflows/nightly-iceberg.yml @@ -49,12 +49,10 @@ env: DEVELOCITY_PROJECT_ID: ${{ vars.DEVELOCITY_PROJECT_ID }} jobs: - detect-iceberg-version: - name: Detect Iceberg Snapshot Version + smoke-tests: + name: Smoke Tests (Iceberg Nightly) runs-on: ubuntu-latest if: github.repository == 'apache/polaris' - outputs: - iceberg_version: ${{ steps.detect.outputs.iceberg_version }} steps: - name: Detect latest Iceberg snapshot version id: detect @@ -75,12 +73,6 @@ jobs: echo "iceberg_version=$LATEST_VERSION" >> $GITHUB_OUTPUT fi - smoke-tests: - name: Smoke Tests (Iceberg ${{ needs.detect-iceberg-version.outputs.iceberg_version }}) - runs-on: ubuntu-latest - needs: detect-iceberg-version - if: github.repository == 'apache/polaris' - steps: - name: Checkout uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 with: @@ -100,7 +92,7 @@ jobs: - name: Override Iceberg version in version catalog run: | - ICEBERG_VERSION="${{ needs.detect-iceberg-version.outputs.iceberg_version }}" + ICEBERG_VERSION="${{ steps.detect.outputs.iceberg_version }}" echo "Overriding Iceberg version to: $ICEBERG_VERSION" sed -i "s/^iceberg = .*/iceberg = \"$ICEBERG_VERSION\"/" gradle/libs.versions.toml echo "Updated gradle/libs.versions.toml:" @@ -110,7 +102,7 @@ jobs: env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} run: | - echo "Running smoke tests with Iceberg version: ${{ needs.detect-iceberg-version.outputs.iceberg_version }}" + echo "Running smoke tests with Iceberg version: ${{ steps.detect.outputs.iceberg_version }}" ./gradlew \ :polaris-core:test \ :polaris-api-iceberg-service:test \