Release automation overhaul: Sonatype Nexus, Shipkit and GH Actions

.github/workflows/java-ci.yml

@@ -20,55 +20,43 @@
 name: "Java CI"
 on:
   push:
-    paths-ignore:
-    - '.github/workflows/python-ci.yml'
-    - 'python/**'
-    - 'site/**'
+    branches: ['li-0.11.x']
+    tags-ignore: [v*] # release tags are autogenerated after a successful CI, no need to run CI against them
   pull_request:
-    paths-ignore:
-    - '.github/workflows/python-ci.yml'
-    - 'python/**'
-    - 'site/**'
+    branches: ['**']
 
 jobs:
-  run-tests:
+
+  build:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        jvm: [8, 11]
+    if: "! contains(toJSON(github.event.commits.*.message), '[skip ci]')"
 
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-java@v1
-      with:
-        java-version: ${{ matrix.jvm }}
-    - uses: actions/cache@v2
-      with:
-        path: ~/.gradle/caches
-        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
-        restore-keys: ${{ runner.os }}-gradle
-    - run: ./gradlew check -Pquick=true -x javadoc
-    - uses: actions/upload-artifact@v2
-      if: failure()
-      with:
-        name: test logs
-        path: |
-          **/build/testlogs
 
-  extra-checks:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-java@v1
-      with:
-        java-version: 8
-    - run: ./gradlew build -x test -x javadoc
+      - name: 1. Check out code
+        uses: actions/checkout@v2 # https://github.com/actions/checkout
+        with:
+          fetch-depth: '0' # https://github.com/shipkit/shipkit-changelog#fetch-depth-on-ci
 
-  build-javadoc:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-java@v1
-      with:
-        java-version: 8
-    - run: ./gradlew -Pquick=true javadoc
+      - name: 2. Set up Java
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+
+      - name: 3. Perform build
+        run: ./gradlew build
+
+      - name: 4. Perform release
+        # Release job, only for pushes to the main development branch
+        if: github.event_name == 'push'
+          && github.ref == 'refs/heads/li-0.11.x'
+          && github.repository == 'linkedin/iceberg'
+          && !contains(toJSON(github.event.commits.*.message), '[skip release]')
+
+        run: ./gradlew githubRelease publishToSonatype closeAndReleaseStagingRepository
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+          SONATYPE_USER: ${{secrets.SONATYPE_USER}}
+          SONATYPE_PWD: ${{secrets.SONATYPE_PWD}}
+          PGP_KEY: ${{secrets.PGP_KEY}}
+          PGP_PWD: ${{secrets.PGP_PWD}}

build.gradle

@@ -17,13 +17,11 @@
  * under the License.
  */
 
-import groovy.transform.Memoized
-
 buildscript {
   repositories {
+    maven { url "https://plugins.gradle.org/m2/" }
     jcenter()
     gradlePluginPortal()
-    maven { url "https://plugins.gradle.org/m2/" }
   }
   dependencies {
     classpath 'com.github.jengelman.gradle.plugins:shadow:5.0.0'
@@ -32,6 +30,9 @@ buildscript {
     classpath 'com.diffplug.spotless:spotless-plugin-gradle:3.14.0'
     classpath 'gradle.plugin.org.inferred:gradle-processors:2.1.0'
     classpath 'me.champeau.gradle:jmh-gradle-plugin:0.4.8'
+    classpath 'io.github.gradle-nexus:publish-plugin:1.1.0'
+    classpath 'org.shipkit:shipkit-auto-version:1.2.0'
+    classpath 'org.shipkit:shipkit-changelog:1.1.10'
   }
 }
 
@@ -60,8 +61,7 @@ dependencyRecommendations {
 }
 
 allprojects {
-  group = "org.apache.iceberg"
-  version = getProjectVersion()
+  group = "com.linkedin.iceberg"
   repositories {
     mavenCentral()
     mavenLocal()
@@ -1229,41 +1229,7 @@ project(':iceberg-runtime') {
   }
 }
 
-@Memoized
-boolean isVersionFileExists() {
-  return file('version.txt').exists()
-}
-
-@Memoized
-String getVersionFromFile() {
-  return file('version.txt').text.trim()
-}
-
-String getProjectVersion() {
-  if (isVersionFileExists()) {
-    return getVersionFromFile()
-  }
-
-  try {
-    return gitVersion()
-  } catch (NullPointerException e) {
-    throw new Exception("Neither version.txt nor git version exists")
-  }
-}
-
-String getJavadocVersion() {
-  if (isVersionFileExists()) {
-    return getVersionFromFile()
-  }
-
-  try {
-    // use the branch name in place of version in Javadoc
-    return versionDetails().branchName
-  } catch (NullPointerException e) {
-    throw new Exception("Neither version.txt nor git version exists")
-  }
-}
 
 apply from: 'baseline.gradle'
 apply from: 'deploy.gradle'
-apply from: 'tasks.gradle'
+apply from: 'shipkit.gradle'

deploy.gradle

@@ -60,7 +60,7 @@ subprojects {
 
   publishing {
     publications {
-      apache(MavenPublication) {
+      javaLibrary(MavenPublication) {
         if (tasks.matching({task -> task.name == 'shadowJar'}).isEmpty()) {
           from components.java
         } else {
@@ -77,25 +77,19 @@ subprojects {
           }
         }
 
-        groupId = 'org.apache.iceberg'
+        groupId = 'com.linkedin.iceberg'
+        artifactId = project.archivesBaseName
+
         pom {
-          name = 'Apache Iceberg'
+          name = artifactId
           description = 'A table format for huge analytic datasets'
-          url = 'https://iceberg.apache.org'
+          url = 'https://github.com/linkedin/iceberg'
           licenses {
             license {
               name = 'The Apache Software License, Version 2.0'
               url = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
             }
           }
-          mailingLists {
-            mailingList {
-              name = 'Dev Mailing List'
-              post = 'dev@iceberg.apache.org'
-              subscribe = 'dev-subscribe@iceberg.apache.org'
-              unsubscribe = 'dev-unsubscribe@iceberg.apache.org'
-            }
-          }
           issueManagement {
             system = 'GitHub'
             url = 'https://github.com/apache/iceberg/issues'
@@ -104,24 +98,17 @@ subprojects {
       }
     }
 
-    repositories {
-      maven {
-        credentials {
-          username project.hasProperty('mavenUser') ? "$mavenUser" : ""
-          password project.hasProperty('mavenPassword') ? "$mavenPassword" : ""
-        }
-        // upload to the releases repository using ./gradlew -Prelease publish
-        def snapshotsRepoUrl = 'https://repository.apache.org/content/repositories/snapshots'
-        def releasesRepoUrl = 'https://repository.apache.org/service/local/staging/deploy/maven2'
-        url = project.hasProperty('release') ? releasesRepoUrl : snapshotsRepoUrl
-      }
-    }
+    //useful for testing - running "publish" will create artifacts/pom in a local dir
+    repositories { maven { url = "$rootProject.buildDir/repo" } }
   }
 
-  if (project.hasProperty('release')) {
-    signing {
-      useGpgCmd()
-      sign publishing.publications.apache
+  //fleshes out problems with Maven pom generation when building
+  tasks.build.dependsOn("publishJavaLibraryPublicationToMavenLocal")
+
+  signing {
+    if (System.getenv("PGP_KEY")) {
+      useInMemoryPgpKeys(System.getenv("PGP_KEY"), System.getenv("PGP_PWD"))
+      sign publishing.publications.javaLibrary
     }
   }
 }

shipkit.gradle

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+//Plugin jars are added to the buildscript classpath in the root build.gradle file
+apply plugin: "org.shipkit.shipkit-auto-version" //https://github.com/shipkit/shipkit-auto-version
+
+apply plugin: "org.shipkit.shipkit-changelog" //https://github.com/shipkit/shipkit-changelog
+tasks.named("generateChangelog") {
+  previousRevision = 'li-0.11.x'
+  githubToken = System.getenv("GITHUB_TOKEN")
+  repository = "linkedin/iceberg"
+}
+
+apply plugin: "org.shipkit.shipkit-github-release" //https://github.com/shipkit/shipkit-changelog
+tasks.named("githubRelease") {
+  def genTask = tasks.named("generateChangelog").get()
+  dependsOn genTask
+  repository = genTask.repository
+  changelog = genTask.outputFile
+  githubToken = System.getenv("GITHUB_TOKEN")
+  newTagRevision = System.getenv("GITHUB_SHA")
+}
+
+apply plugin: "io.github.gradle-nexus.publish-plugin" //https://github.com/gradle-nexus/publish-plugin/
+nexusPublishing {
+  repositories {
+    if (System.getenv("SONATYPE_PWD")) {
+      sonatype {
+        username = System.getenv("SONATYPE_USER")
+        password = System.getenv("SONATYPE_PWD")
+        nexusUrl.set(uri("https://s01.oss.sonatype.org/service/local/"))
+        snapshotRepositoryUrl.set(uri("https://s01.oss.sonatype.org/content/repositories/snapshots/"))
+      }
+    }
+  }
+  transitionCheckOptions {
+    maxRetries.set(100)
+  }
+}

version.properties

@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Version of the produced binaries.
+# The version is inferred by shipkit-auto-version Gradle plugin (https://github.com/shipkit/shipkit-auto-version)
+version=0.11.1.*