diff --git a/.github/workflows/nightly-iceberg.yml b/.github/workflows/nightly-iceberg.yml new file mode 100644 index 0000000000..e4481f5172 --- /dev/null +++ b/.github/workflows/nightly-iceberg.yml @@ -0,0 +1,120 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Runs smoke tests against Apache Iceberg nightly builds to detect compatibility issues early. + +name: Iceberg Nightly Smoke Tests + +on: + schedule: + # Run daily at 3:00 AM UTC + - cron: '0 3 * * *' + workflow_dispatch: + inputs: + iceberg_version: + description: 'Iceberg version to test (e.g., 1.11.0-SNAPSHOT). Leave empty to auto-detect latest snapshot.' + required: false + type: string + branch: + description: 'Branch to build' + required: false + default: 'main' + type: string + +# Setting explicit permissions for the action +permissions: + actions: read + contents: read + issues: read + +env: + GRADLE_TOS_ACCEPTED: ${{ vars.GRADLE_TOS_ACCEPTED }} + DEVELOCITY_SERVER: ${{ vars.DEVELOCITY_SERVER }} + DEVELOCITY_PROJECT_ID: ${{ vars.DEVELOCITY_PROJECT_ID }} + +jobs: + smoke-tests: + name: Smoke Tests (Iceberg Nightly) + runs-on: ubuntu-latest + if: github.repository == 'apache/polaris' + steps: + - name: Detect latest Iceberg snapshot version + id: detect + run: | + if [ -n "${{ inputs.iceberg_version }}" ]; then + echo "Using provided Iceberg version: ${{ inputs.iceberg_version }}" + echo "iceberg_version=${{ inputs.iceberg_version }}" >> $GITHUB_OUTPUT + else + echo "Detecting latest Iceberg snapshot version from Apache Maven repository..." + # Fetch the maven-metadata.xml to find the latest snapshot version + METADATA_URL="https://repository.apache.org/content/repositories/snapshots/org/apache/iceberg/iceberg-api/maven-metadata.xml" + LATEST_VERSION=$(curl -s "$METADATA_URL" | grep -oP '(?<=)[^<]+' | head -1) + if [ -z "$LATEST_VERSION" ]; then + echo "Failed to detect Iceberg snapshot version" + exit 1 + fi + echo "Detected Iceberg snapshot version: $LATEST_VERSION" + echo "iceberg_version=$LATEST_VERSION" >> $GITHUB_OUTPUT + fi + + - name: Checkout + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6 + with: + ref: ${{ inputs.branch || 'main' }} + + - name: Set up JDK 21 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: + java-version: '21' + distribution: 'temurin' + + - name: Setup test environment + uses: ./.github/actions/setup-test-env + + - name: Prepare Gradle build cache + uses: ./.github/actions/ci-incr-build-cache-prepare + + - name: Override Iceberg version in version catalog + run: | + ICEBERG_VERSION="${{ steps.detect.outputs.iceberg_version }}" + echo "Overriding Iceberg version to: $ICEBERG_VERSION" + sed -i "s/^iceberg = .*/iceberg = \"$ICEBERG_VERSION\"/" gradle/libs.versions.toml + echo "Updated gradle/libs.versions.toml:" + grep "^iceberg = " gradle/libs.versions.toml + + - name: Run smoke tests with Iceberg nightly + env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + run: | + echo "Running smoke tests with Iceberg version: ${{ steps.detect.outputs.iceberg_version }}" + ./gradlew \ + :polaris-core:test \ + :polaris-api-iceberg-service:test \ + :polaris-runtime-service:test \ + -PuseApacheSnapshots=true \ + --continue + + - name: Archive test results + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + if: failure() + with: + name: iceberg-nightly-test-results + path: | + **/build/test-results/** + **/build/reports/tests/** diff --git a/settings.gradle.kts b/settings.gradle.kts index 290416f981..bec990713a 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -125,6 +125,15 @@ dependencyResolutionManagement { url = uri("https://jitpack.io") content { includeModule("com.github.RoaringBitmap.RoaringBitmap", "roaringbitmap") } } + val useApacheSnapshots = + providers.gradleProperty("useApacheSnapshots").orNull?.toBoolean() == true + if (useApacheSnapshots) { + maven { + name = "ApacheSnapshots" + url = uri("https://repository.apache.org/content/repositories/snapshots/") + mavenContent { snapshotsOnly() } + } + } gradlePluginPortal() } }