[DO NOT MERGE] Sync upstream trunk 28 Feb 2023

Jenkinsfile

@@ -65,14 +65,18 @@ def job = {
     }
 
     stage("Check compilation compatibility with Scala 2.12") {
-        sh "./retry_zinc ./gradlew clean assemble spotlessScalaCheck checkstyleMain checkstyleTest spotbugsMain " +
-                 "--no-daemon --stacktrace -PxmlSpotBugsReport=true -PscalaVersion=2.12"
+        sh """
+            ./retry_zinc ./gradlew clean build -x test \
+                --no-daemon --stacktrace -PxmlSpotBugsReport=true -PscalaVersion=2.12
+           """
     }
 
 
     stage("Compile and validate") {
-        sh "./retry_zinc ./gradlew clean assemble publishToMavenLocal spotlessScalaCheck checkstyleMain checkstyleTest spotbugsMain " +
-                "--no-daemon --stacktrace -PxmlSpotBugsReport=true"
+        sh """
+            ./retry_zinc ./gradlew clean publishToMavenLocal build -x test \
+                --no-daemon --stacktrace -PxmlSpotBugsReport=true
+           """
     }
 
     if (config.publish) {

build.gradle

@@ -14,6 +14,7 @@
 // limitations under the License.
 
 import org.ajoberstar.grgit.Grgit
+import org.gradle.api.JavaVersion
 
 import java.nio.charset.StandardCharsets
 
@@ -30,26 +31,18 @@ buildscript {
 }
 
 plugins {
-  id 'com.diffplug.spotless' version '6.13.0'
   id 'com.github.ben-manes.versions' version '0.44.0'
   id 'idea'
   id 'java-library'
   id 'org.owasp.dependencycheck' version '8.0.2'
   id 'org.nosphere.apache.rat' version "0.8.0"
+  id "io.swagger.core.v3.swagger-gradle-plugin" version "2.2.0"
 
   id "com.github.spotbugs" version '5.0.13' apply false
   id 'org.gradle.test-retry' version '1.5.1' apply false
   id 'org.scoverage' version '7.0.1' apply false
   id 'com.github.johnrengelman.shadow' version '7.1.2' apply false
-  id "io.swagger.core.v3.swagger-gradle-plugin" version "2.2.0"
-}
-
-spotless {
-  scala {
-    target 'streams/**/*.scala'
-    scalafmt("$versions.scalafmt").configFile('checkstyle/.scalafmt.conf').scalaMajorVersion(versions.baseScala)
-    licenseHeaderFile 'checkstyle/java.header', 'package'
-  }
+  id 'com.diffplug.spotless' version '6.14.0' apply false // 6.14.1 and newer require Java 11 at compile time, so we can't upgrade until AK 4.0
 }
 
 task copyGitHooks(type: Copy) {
@@ -74,7 +67,7 @@ task installGitHooks(type: Exec) {
 
 ext {
   gradleVersion = versions.gradle
-  minJavaVersion = "8"
+  minJavaVersion = 8
   buildVersionFileName = "kafka-version.properties"
 
   defaultMaxHeapSize = "2g"
@@ -229,6 +222,7 @@ if (file('.git').exists()) {
         '**/build/**',
         'CONTRIBUTING.md',
         'PULL_REQUEST_TEMPLATE.md',
+        '.github/**',
         'gradlew',
         'gradlew.bat',
         'gradle/wrapper/gradle-wrapper.properties',
@@ -286,9 +280,6 @@ subprojects {
     fineTuneEclipseClasspathFile(eclipse, project)
   }
 
-  sourceCompatibility = minJavaVersion
-  targetCompatibility = minJavaVersion
-
   java {
     consistentResolution {
       // resolve the compileClasspath and then "inject" the result of resolution as strict constraints into the runtimeClasspath
@@ -306,12 +297,16 @@ subprojects {
     options.compilerArgs << "-Xlint:-try"
     options.compilerArgs << "-Werror"
     // --release is the recommended way to select the target release, but it's only supported in Java 9 so we also
-    // set --source and --target via `sourceCompatibility` and `targetCompatibility`. If/when Gradle supports `--release`
-    // natively (https://github.com/gradle/gradle/issues/2510), we should switch to that.
+    // set --source and --target via `sourceCompatibility` and `targetCompatibility` a couple of lines below
     if (JavaVersion.current().isJava9Compatible())
-      options.compilerArgs << "--release" << minJavaVersion
+      options.release = minJavaVersion
     dependsOn installGitHooks
   }
+
+  // We should only set this if Java version is < 9 (--release is recommended for >= 9), but the Scala plugin for IntelliJ sets
+  // `-target` incorrectly if this is unset
+  sourceCompatibility = minJavaVersion
+  targetCompatibility = minJavaVersion
 
   if (shouldPublish) {
 
@@ -447,7 +442,7 @@ subprojects {
       "**/KafkaConfigBackingStoreTest.*",
       "**/KafkaBasedLogTest.*", "**/StandaloneHerderTest.*",
       "**/WorkerSinkTaskTest.*", "**/WorkerSinkTaskThreadedTest.*",
-      "**/WorkerSourceTaskTest.*", "**/AbstractWorkerSourceTaskTest.*"
+      "**/WorkerSourceTaskTest.*"
     ])
   }
 
@@ -716,9 +711,12 @@ subprojects {
       ]
     }
 
-    // Scalac's `-release` requires Java 9 or higher
-    if (JavaVersion.current().isJava9Compatible())
-      scalaCompileOptions.additionalParameters += ["-release", minJavaVersion]
+    // Scalac 2.12 `-release` requires Java 9 or higher, but Scala 2.13 doesn't have that restriction
+    if (versions.baseScala == "2.13")
+      scalaCompileOptions.additionalParameters += ["-release:" + minJavaVersion] // Use `:` here to workaround Gradle bug (see https://github.com/gradle/gradle/issues/23962#issuecomment-1437348400)
+    else if (JavaVersion.current().isJava9Compatible())
+      scalaCompileOptions.additionalParameters += ["-release", String.valueOf(minJavaVersion)] // Don't use `:` here as it breaks compilation with Scala 2.12
+
 
     configure(scalaCompileOptions.forkOptions) {
       memoryMaximumSize = defaultMaxHeapSize
@@ -1040,6 +1038,7 @@ project(':core') {
   }
 
   compileJava.dependsOn 'processMessages'
+  srcJar.dependsOn 'processMessages'
 
   task genProtocolErrorDocs(type: JavaExec) {
     classpath = sourceSets.main.runtimeClasspath
@@ -1271,6 +1270,7 @@ project(':metadata') {
   }
 
   compileJava.dependsOn 'processMessages'
+  srcJar.dependsOn 'processMessages'
 
   sourceSets {
     main {
@@ -1345,6 +1345,7 @@ project(':group-coordinator') {
   }
 
   compileJava.dependsOn 'processMessages'
+  srcJar.dependsOn 'processMessages'
 }
 
 project(':examples') {
@@ -1483,6 +1484,7 @@ project(':clients') {
   }
 
   compileJava.dependsOn 'processMessages'
+  srcJar.dependsOn 'processMessages'
 
   compileTestJava.dependsOn 'processTestMessages'
 
@@ -1585,6 +1587,7 @@ project(':raft') {
   }
 
   compileJava.dependsOn 'processMessages'
+  srcJar.dependsOn 'processMessages'
 
   jar {
     dependsOn createVersionFile
@@ -1807,6 +1810,7 @@ project(':storage') {
   }
 
   compileJava.dependsOn 'processMessages'
+  srcJar.dependsOn 'processMessages'
 
   jar {
     dependsOn createVersionFile
@@ -2045,6 +2049,7 @@ project(':streams') {
   }
 
   compileJava.dependsOn 'processMessages'
+  srcJar.dependsOn 'processMessages'
 
   javadoc {
     include "**/org/apache/kafka/streams/**"
@@ -2167,7 +2172,17 @@ project(':streams:streams-scala') {
     dependsOn 'copyDependantLibs'
   }
 
-  test.dependsOn(':spotlessScalaCheck')
+  // spotless 6.14 requires Java 11 at runtime
+  if (JavaVersion.current().isJava11Compatible()) {
+    apply plugin: 'com.diffplug.spotless'
+    spotless {
+      scala {
+        target '**/*.scala'
+        scalafmt("$versions.scalafmt").configFile('../../checkstyle/.scalafmt.conf').scalaMajorVersion(versions.baseScala)
+        licenseHeaderFile '../../checkstyle/java.header', 'package'
+      }
+    }
+  }
 }
 
 project(':streams:test-utils') {

checkstyle/import-control-core.xml

@@ -75,10 +75,8 @@
   </subpackage>
 
   <subpackage name="server">
-    <subpackage name="builders">
-      <allow pkg="kafka" />
-      <allow pkg="org.apache.kafka" />
-    </subpackage>
+    <allow pkg="kafka" />
+    <allow pkg="org.apache.kafka" />
   </subpackage>
 
   <subpackage name="test">