From f07da8d832918980611bd822a8003fd1f18e12eb Mon Sep 17 00:00:00 2001 From: Shanthoosh Pazhanjur Venkataraman Date: Thu, 18 Sep 2025 14:09:49 -0700 Subject: [PATCH 1/2] Remove the RC version number from the iceberg build suffix. --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 43c70d014..888e47b20 100644 --- a/build.gradle +++ b/build.gradle @@ -876,7 +876,7 @@ String getProjectVersion() { if (lastNum >= 900) { // Convert 1.2.0.900+ to 1.2.0.0-rcX format int rcNum = lastNum - 899 - return matcher.group(1) + "." + matcher.group(2) + "." + matcher.group(3) + ".0-rc" + rcNum + return matcher.group(1) + "." + matcher.group(2) + "." + matcher.group(3) + "." + rcNum } } return version From 8d17d531f3c15c4e9bcb2bcf7cc26ea1d842c460 Mon Sep 17 00:00:00 2001 From: Shanthoosh Pazhanjur Venkataraman Date: Thu, 18 Sep 2025 19:33:52 -0700 Subject: [PATCH 2/2] Rebase changes. --- .github/workflows/java-ci.yml | 52 ++++++++++---------- .github/workflows/spark-ci.yml | 86 ++++------------------------------ build.gradle | 27 +++++------ deploy.gradle | 25 +++++++++- shipkit.gradle | 62 ++++++++++++++++-------- version.properties | 2 +- version.txt | 2 +- versions.props | 1 + 8 files changed, 118 insertions(+), 139 deletions(-) diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index 5526d0ab4..305e2f022 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -21,19 +20,18 @@ name: "Java CI" on: push: branches: ['openhouse-1.2.0'] - tags-ignore: [v*] # release tags are autogenerated after a successful CI, no need to run CI against them + tags-ignore: [v*] pull_request: branches: ['**'] paths-ignore: - - '.github/ISSUE_TEMPLATE/iceberg_bug_report.yml' - '.github/workflows/python-ci.yml' - '.github/workflows/spark-ci.yml' - '.github/workflows/flink-ci.yml' - '.github/workflows/hive-ci.yml' - '.gitignore' - - '.asf.yml' - 'dev/**' - 'python/**' + - 'python_legacy/**' - 'docs/**' - 'open-api/**' - 'format/**' @@ -48,36 +46,38 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - build: + build-checks: runs-on: ubuntu-latest - if: "! contains(toJSON(github.event.commits.*.message), '[skip ci]')" - steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: zulu + java-version: 8 + - run: ./gradlew -DsparkVersions=3.1 build -x test -x javadoc -x integrationTest - - name: 1. Check out code - uses: actions/checkout@v2 # https://github.com/actions/checkout + release: + if: ${{ github.event_name == 'push' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 with: - fetch-depth: '0' # https://github.com/shipkit/shipkit-changelog#fetch-depth-on-ci - - - name: 2. Set up Java - uses: actions/setup-java@v1 + fetch-depth: '0' + - uses: actions/setup-java@v4 with: - java-version: 1.8 - - - name: 3. Perform build - run: ./gradlew build + distribution: zulu + java-version: 8 + - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - name: 4. Perform release - # Release job, only for pushes to the main development branch - if: github.event_name == 'push' + - name: Perform release + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/openhouse-1.2.0' && github.repository == 'linkedin/iceberg' - && !contains(toJSON(github.event.commits.*.message), '[skip release]') - - run: ./gradlew githubRelease publishToSonatype closeAndReleaseStagingRepository + && !contains(toJSON(github.event.commits.*.message), '[skip release]') }} + run: ./gradlew -DsparkVersions=3.1 githubRelease publishToSonatype closeAndReleaseStagingRepository -x test -x integrationTest env: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - SONATYPE_USER: ${{secrets.SONATYPE_USER}} - SONATYPE_PWD: ${{secrets.SONATYPE_PWD}} + SONATYPE_USER: ${{secrets.SONATYPE_USERNAME}} + SONATYPE_PWD: ${{secrets.SONATYPE_PASSWORD}} PGP_KEY: ${{secrets.PGP_KEY}} - PGP_PWD: ${{secrets.PGP_PWD}} \ No newline at end of file + PGP_PWD: ${{secrets.PGP_PWD}} diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml index 92adb797d..93df58fef 100644 --- a/.github/workflows/spark-ci.yml +++ b/.github/workflows/spark-ci.yml @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -21,18 +20,13 @@ name: "Spark CI" on: push: branches: - - 'master' - - '0.**' - tags: - - 'apache-iceberg-**' + - 'openhouse-1.2.0' pull_request: paths-ignore: - - '.github/ISSUE_TEMPLATE/iceberg_bug_report.yml' - '.github/workflows/python-ci.yml' - '.github/workflows/flink-ci.yml' - '.github/workflows/hive-ci.yml' - '.gitignore' - - '.asf.yml' - 'dev/**' - 'mr/**' - 'hive3/**' @@ -41,6 +35,7 @@ on: - 'flink/**' - 'pig/**' - 'python/**' + - 'python_legacy/**' - 'docs/**' - 'open-api/**' - 'format/**' @@ -55,38 +50,12 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - spark2-tests: - runs-on: ubuntu-22.04 - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: 8 - - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: ${{ runner.os }}-gradle- - - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - run: ./gradlew -DsparkVersions=2.4 -DhiveVersions= -DflinkVersions= :iceberg-spark:check :iceberg-spark:iceberg-spark-2.4:check :iceberg-spark:iceberg-spark-runtime-2.4:check -Pquick=true -x javadoc - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: test logs - path: | - **/build/testlogs - spark-3x-scala-2-12-tests: runs-on: ubuntu-22.04 + if: "github.repository == 'linkedin/iceberg'" strategy: matrix: - jvm: [8, 11] - spark: ['3.1', '3.2', '3.3'] + spark: ['3.1'] env: SPARK_LOCAL_IP: localhost steps: @@ -94,49 +63,12 @@ jobs: - uses: actions/setup-java@v4 with: distribution: zulu - java-version: ${{ matrix.jvm }} + java-version: 8 - uses: actions/cache@v4 with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: ${{ runner.os }}-gradle- + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }} + restore-keys: ${{ runner.os }}-gradle - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=2.12 -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_2.12:check -Pquick=true -x javadoc - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: test logs - path: | - **/build/testlogs - spark-3x-scala-2-13-tests: - runs-on: ubuntu-22.04 - strategy: - matrix: - jvm: [8, 11] - spark: ['3.2','3.3'] - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: zulu - java-version: ${{ matrix.jvm }} - - uses: actions/cache@v4 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: ${{ runner.os }}-gradle- - - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts - - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=2.13 -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.13:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_2.13:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_2.13:check -Pquick=true -x javadoc - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: test logs - path: | - **/build/testlogs + - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=2.12 -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_2.12:check -Pquick=true -x test -x integrationTest -x javadoc diff --git a/build.gradle b/build.gradle index 888e47b20..aba611650 100644 --- a/build.gradle +++ b/build.gradle @@ -28,7 +28,9 @@ buildscript { } dependencies { classpath 'gradle.plugin.com.github.jengelman.gradle.plugins:shadow:7.0.0' - classpath 'com.palantir.baseline:gradle-baseline-java:4.42.0' + classpath('com.palantir.baseline:gradle-baseline-java:4.42.0') { + exclude group: 'com.palantir.tritium' + } // com.palantir.baseline:gradle-baseline-java:4.42.0 (the last version supporting Java 8) pulls // in an old version of the errorprone, which doesn't work w/ Gradle 8, so bump errorpone as // well. @@ -131,6 +133,15 @@ subprojects { apply plugin: 'nebula.dependency-recommender' apply plugin: 'java-library' + configurations.all { + resolutionStrategy { + // Force specific versions of tritium to resolve version conflicts + force 'com.palantir.tritium:tritium-registry:0.17.0' + force 'com.palantir.tritium:tritium-core:0.17.0' + force 'com.palantir.tritium:tritium-metrics:0.17.0' + } + } + if (project.name in REVAPI_PROJECTS) { apply plugin: 'com.palantir.revapi' revapi { @@ -867,19 +878,7 @@ String getVersionFromFile() { String getProjectVersion() { if (versionFileExists()) { - String version = getVersionFromFile() - // If version is in 900+ series, treat it as a pre-release - Pattern pattern = Pattern.compile("^([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)?\$") - Matcher matcher = pattern.matcher(version) - if (matcher.matches()) { - int lastNum = Integer.valueOf(matcher.group(4)) - if (lastNum >= 900) { - // Convert 1.2.0.900+ to 1.2.0.0-rcX format - int rcNum = lastNum - 899 - return matcher.group(1) + "." + matcher.group(2) + "." + matcher.group(3) + "." + rcNum - } - } - return version + return getVersionFromFile() } try { diff --git a/deploy.gradle b/deploy.gradle index 595ec7adf..ef00413f9 100644 --- a/deploy.gradle +++ b/deploy.gradle @@ -95,7 +95,30 @@ subprojects { } issueManagement { system = 'GitHub' - url = 'https://github.com/apache/iceberg/issues' + url = 'https://github.com/linkedin/iceberg/issues' + } + developers { + [ + 'wmoustafa:Walaa Eldin Moustafa', + 'sumedhsakdeo:Sumedh Sakdeo', + 'maluchari:Malini Mahalakshmi Venkatachari', + 'rdsr:Ratandeep Ratti' + ].each { devData -> + developer { + def devInfo = devData.split(':') + id = devInfo[0] + name = devInfo[1] + url = 'https://github.com/' + devInfo[0] + roles = ["Core developer"] + } + } + } + scm { + url = 'https://github.com/linkedin/iceberg.git' + } + ciManagement { + url = 'https://github.com/linkedin/iceberg/actions' + system = 'GitHub Actions' } } } diff --git a/shipkit.gradle b/shipkit.gradle index bbef90d57..78f460420 100644 --- a/shipkit.gradle +++ b/shipkit.gradle @@ -25,23 +25,48 @@ tasks.named("generateChangelog") { previousRevision = project.ext.'shipkit-auto-version.previous-tag' githubToken = System.getenv("GITHUB_TOKEN") repository = "linkedin/iceberg" - - // Make generateChangelog depend on all necessary subproject tasks - subprojects.each { subproject -> - // Compilation tasks - dependsOn subproject.tasks.withType(JavaCompile) - - // Documentation tasks - dependsOn subproject.tasks.withType(Javadoc) - - // Publication tasks - dependsOn subproject.tasks.matching { it.name == 'generatePomFileForJavaLibraryPublication' } - - // Resource processing tasks - dependsOn subproject.tasks.withType(ProcessResources) + // Write changelog into the root build directory to avoid broad root-dir overlaps + outputFile = file("${buildDir}/CHANGELOG.md") +} + +// Ensure Gradle knows ordering relative to tasks that write inside the repository +// to avoid implicit dependency validation errors in Gradle 8+ +gradle.projectsEvaluated { + def genTaskProvider = tasks.named("generateChangelog") + genTaskProvider.configure { t -> + // Ensure we run after root task that writes to root build output + t.mustRunAfter(rootProject.tasks.matching { it.name == 'generateGitProperties' }) + + rootProject.subprojects.each { prj -> + // Run after resource processing tasks + t.mustRunAfter(prj.tasks.matching { it.name == 'processResources' }) + t.mustRunAfter(prj.tasks.matching { it.name == 'processTestResources' }) + + // Run after Java compilation tasks + t.mustRunAfter(prj.tasks.matching { it.name == 'compileJava' }) + t.mustRunAfter(prj.tasks.matching { it.name == 'compileTestJava' }) + + // Run after documentation and archive tasks that may touch repo files + t.mustRunAfter(prj.tasks.withType(org.gradle.api.tasks.javadoc.Javadoc)) + t.mustRunAfter(prj.tasks.withType(org.gradle.api.tasks.bundling.Jar)) + + // Run after any POM generation tasks created by maven-publish + t.mustRunAfter(prj.tasks.withType(org.gradle.api.publish.maven.tasks.GenerateMavenPom)) + + // Run after module metadata tasks, if present + t.mustRunAfter(prj.tasks.matching { it.name.startsWith('generateMetadataFileFor') }) + + // Run after signing tasks (from 'signing' plugin) + t.mustRunAfter(prj.tasks.matching { it.name.startsWith('sign') }) + + // Run after publish tasks to avoid implicit dependency warnings in release pipelines + t.mustRunAfter(prj.tasks.matching { it.name.startsWith('publish') }) + } } } + + apply plugin: "org.shipkit.shipkit-github-release" //https://github.com/shipkit/shipkit-changelog tasks.named("githubRelease") { def genTask = tasks.named("generateChangelog").get() @@ -59,12 +84,11 @@ nexusPublishing { sonatype { username = System.getenv("SONATYPE_USER") password = System.getenv("SONATYPE_PWD") - nexusUrl.set(uri("https://s01.oss.sonatype.org/service/local/")) - snapshotRepositoryUrl.set(uri("https://s01.oss.sonatype.org/content/repositories/snapshots/")) + nexusUrl.set(uri("https://ossrh-staging-api.central.sonatype.com/service/local/")) + // nexusUrl.set(uri("https://s01.oss.sonatype.org/service/local/")) + snapshotRepositoryUrl.set(uri("https://central.sonatype.com/repository/maven-snapshots/")) + // snapshotRepositoryUrl.set(uri("https://s01.oss.sonatype.org/content/repositories/snapshots/")) } } } - transitionCheckOptions { - maxRetries.set(100) - } } diff --git a/version.properties b/version.properties index 82e0b74d4..6895fe928 100644 --- a/version.properties +++ b/version.properties @@ -19,4 +19,4 @@ # Version of the produced binaries. # The version is inferred by shipkit-auto-version Gradle plugin (https://github.com/shipkit/shipkit-auto-version) # Using 900+ series to indicate pre-release versions -version=1.2.0.900 +version=1.2.0.901 diff --git a/version.txt b/version.txt index 7890aca85..9177da1cd 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.2.0.0-rc1 +1.2.0.* diff --git a/versions.props b/versions.props index 34c6a5b86..8c5772987 100644 --- a/versions.props +++ b/versions.props @@ -1,4 +1,5 @@ org.slf4j:* = 1.7.36 +com.palantir.tritium:* = 0.17.0 org.apache.avro:avro = 1.11.1 org.apache.calcite:* = 1.10.0 org.apache.hadoop:* = 2.7.3