KAFKA-14735: Improve KRaft metadata image change performance at high …

LICENSE-binary

@@ -313,7 +313,8 @@ argparse4j-0.7.0, see: licenses/argparse-MIT
 jopt-simple-5.0.4, see: licenses/jopt-simple-MIT
 slf4j-api-1.7.36, see: licenses/slf4j-MIT
 slf4j-reload4j-1.7.36, see: licenses/slf4j-MIT
-classgraph-4.8.138, see: license/classgraph-MIT
+classgraph-4.8.138, see: licenses/classgraph-MIT
+pcollections-4.0.1, see: licenses/pcollections-MIT
 
 ---------------------------------------
 BSD 2-Clause

build.gradle

@@ -1222,6 +1222,10 @@ project(':metadata') {
   javadoc {
     enabled = false
   }
+
+  checkstyle {
+    configProperties = checkstyleConfigProperties("import-control-metadata.xml")
+  }
 }
 
 project(':group-coordinator') {
@@ -1554,11 +1558,13 @@ project(':server-common') {
     implementation libs.slf4jApi
     implementation libs.metrics
     implementation libs.joptSimple
+    implementation libs.pcollections
 
     testImplementation project(':clients')
     testImplementation project(':clients').sourceSets.test.output
     testImplementation libs.junitJupiter
     testImplementation libs.mockitoCore
+    testImplementation libs.mockitoInline // supports mocking static methods, final classes, etc.
     testImplementation libs.hamcrest
 
     testRuntimeOnly libs.slf4jlog4j
@@ -1605,6 +1611,10 @@ project(':server-common') {
   clean.doFirst {
     delete "$buildDir/kafka/"
   }
+
+  checkstyle {
+    configProperties = checkstyleConfigProperties("import-control-server-common.xml")
+  }
 }
 
 project(':storage:api') {

checkstyle/import-control-jmh-benchmarks.xml

@@ -51,6 +51,7 @@
     <allow pkg="org.apache.kafka.storage"/>
     <allow pkg="org.apache.kafka.clients"/>
     <allow pkg="org.apache.kafka.coordinator.group"/>
+    <allow pkg="org.apache.kafka.image"/>
     <allow pkg="org.apache.kafka.metadata"/>
     <allow pkg="org.apache.kafka.timeline" />
 

checkstyle/import-control-metadata.xml

@@ -0,0 +1,176 @@
+<!DOCTYPE import-control PUBLIC
+        "-//Puppy Crawl//DTD Import Control 1.1//EN"
+        "http://www.puppycrawl.com/dtds/import_control_1_1.dtd">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<import-control pkg="org.apache.kafka">
+
+    <!-- THINK HARD ABOUT THE LAYERING OF THE PROJECT BEFORE CHANGING THIS FILE -->
+
+    <!-- common library dependencies -->
+    <allow pkg="java" />
+    <allow pkg="javax.management" />
+    <allow pkg="org.slf4j" />
+    <allow pkg="org.junit" />
+    <allow pkg="org.opentest4j" />
+    <allow pkg="org.hamcrest" />
+    <allow pkg="org.mockito" />
+    <allow pkg="org.easymock" />
+    <allow pkg="org.powermock" />
+    <allow pkg="java.security" />
+    <allow pkg="javax.net.ssl" />
+    <allow pkg="javax.security" />
+    <allow pkg="org.ietf.jgss" />
+    <allow pkg="net.jqwik.api" />
+
+    <!-- no one depends on the server -->
+    <disallow pkg="kafka" />
+
+    <!-- anyone can use public classes -->
+    <allow pkg="org.apache.kafka.common" exact-match="true" />
+    <allow pkg="org.apache.kafka.common.security" />
+    <allow pkg="org.apache.kafka.common.serialization" />
+    <allow pkg="org.apache.kafka.common.utils" />
+    <allow pkg="org.apache.kafka.common.errors" exact-match="true" />
+    <allow pkg="org.apache.kafka.common.memory" />
+
+    <!-- persistent collection factories/non-library-specific wrappers -->
+    <allow pkg="org.apache.kafka.server.immutable" exact-match="true" />
+
+    <subpackage name="common">
+        <subpackage name="metadata">
+            <allow pkg="com.fasterxml.jackson" />
+            <allow pkg="org.apache.kafka.common.protocol" />
+            <allow pkg="org.apache.kafka.common.protocol.types" />
+            <allow pkg="org.apache.kafka.common.message" />
+            <allow pkg="org.apache.kafka.common.metadata" />
+        </subpackage>
+    </subpackage>
+
+    <subpackage name="controller">
+        <allow pkg="com.yammer.metrics"/>
+        <allow pkg="org.apache.kafka.clients" />
+        <allow pkg="org.apache.kafka.clients.admin" />
+        <allow pkg="org.apache.kafka.common.acl" />
+        <allow pkg="org.apache.kafka.common.annotation" />
+        <allow pkg="org.apache.kafka.common.config" />
+        <allow pkg="org.apache.kafka.common.feature" />
+        <allow pkg="org.apache.kafka.common.internals" />
+        <allow pkg="org.apache.kafka.common.message" />
+        <allow pkg="org.apache.kafka.common.metadata" />
+        <allow pkg="org.apache.kafka.common.metrics" />
+        <allow pkg="org.apache.kafka.common.network" />
+        <allow pkg="org.apache.kafka.common.protocol" />
+        <allow pkg="org.apache.kafka.common.quota" />
+        <allow pkg="org.apache.kafka.common.requests" />
+        <allow pkg="org.apache.kafka.common.resource" />
+        <allow pkg="org.apache.kafka.controller" />
+        <allow pkg="org.apache.kafka.image" />
+        <allow pkg="org.apache.kafka.image.writer" />
+        <allow pkg="org.apache.kafka.metadata" />
+        <allow pkg="org.apache.kafka.metadata.authorizer" />
+        <allow pkg="org.apache.kafka.metadata.migration" />
+        <allow pkg="org.apache.kafka.metalog" />
+        <allow pkg="org.apache.kafka.queue" />
+        <allow pkg="org.apache.kafka.raft" />
+        <allow pkg="org.apache.kafka.server.authorizer" />
+        <allow pkg="org.apache.kafka.server.common" />
+        <allow pkg="org.apache.kafka.server.config" />
+        <allow pkg="org.apache.kafka.server.fault" />
+        <allow pkg="org.apache.kafka.server.metrics" />
+        <allow pkg="org.apache.kafka.server.policy"/>
+        <allow pkg="org.apache.kafka.server.util"/>
+        <allow pkg="org.apache.kafka.snapshot" />
+        <allow pkg="org.apache.kafka.test" />
+        <allow pkg="org.apache.kafka.timeline" />
+    </subpackage>
+
+    <subpackage name="image">
+        <allow pkg="org.apache.kafka.common.config" />
+        <allow pkg="org.apache.kafka.common.message" />
+        <allow pkg="org.apache.kafka.common.metadata" />
+        <allow pkg="org.apache.kafka.common.protocol" />
+        <allow pkg="org.apache.kafka.common.quota" />
+        <allow pkg="org.apache.kafka.common.record" />
+        <allow pkg="org.apache.kafka.common.requests" />
+        <allow pkg="org.apache.kafka.common.resource" />
+        <allow pkg="org.apache.kafka.image" />
+        <allow pkg="org.apache.kafka.image.writer" />
+        <allow pkg="org.apache.kafka.metadata" />
+        <allow pkg="org.apache.kafka.queue" />
+        <allow pkg="org.apache.kafka.clients.admin" />
+        <allow pkg="org.apache.kafka.raft" />
+        <allow pkg="org.apache.kafka.server.common" />
+        <allow pkg="org.apache.kafka.server.fault" />
+        <allow pkg="org.apache.kafka.server.util" />
+        <allow pkg="org.apache.kafka.snapshot" />
+        <allow pkg="org.apache.kafka.test" />
+    </subpackage>
+
+    <subpackage name="metadata">
+        <allow pkg="org.apache.kafka.clients" />
+        <allow pkg="org.apache.kafka.common.acl" />
+        <allow pkg="org.apache.kafka.common.annotation" />
+        <allow pkg="org.apache.kafka.common.config" />
+        <allow pkg="org.apache.kafka.common.message" />
+        <allow pkg="org.apache.kafka.common.metadata" />
+        <allow pkg="org.apache.kafka.common.protocol" />
+        <allow pkg="org.apache.kafka.common.record" />
+        <allow pkg="org.apache.kafka.common.resource" />
+        <allow pkg="org.apache.kafka.common.requests" />
+        <allow pkg="org.apache.kafka.image" />
+        <allow pkg="org.apache.kafka.metadata" />
+        <allow pkg="org.apache.kafka.metalog" />
+        <allow pkg="org.apache.kafka.queue" />
+        <allow pkg="org.apache.kafka.raft" />
+        <allow pkg="org.apache.kafka.server.authorizer" />
+        <allow pkg="org.apache.kafka.server.common" />
+        <allow pkg="org.apache.kafka.server.fault" />
+        <allow pkg="org.apache.kafka.server.config" />
+        <allow pkg="org.apache.kafka.server.util"/>
+        <allow pkg="org.apache.kafka.test" />
+        <subpackage name="authorizer">
+            <allow pkg="org.apache.kafka.common.acl" />
+            <allow pkg="org.apache.kafka.common.requests" />
+            <allow pkg="org.apache.kafka.common.resource" />
+            <allow pkg="org.apache.kafka.controller" />
+            <allow pkg="org.apache.kafka.metadata" />
+            <allow pkg="org.apache.kafka.common.internals" />
+        </subpackage>
+        <subpackage name="bootstrap">
+            <allow pkg="org.apache.kafka.snapshot" />
+        </subpackage>
+        <subpackage name="fault">
+            <allow pkg="org.apache.kafka.server.fault" />
+        </subpackage>
+    </subpackage>
+
+    <subpackage name="metalog">
+        <allow pkg="org.apache.kafka.common.metadata" />
+        <allow pkg="org.apache.kafka.common.protocol" />
+        <allow pkg="org.apache.kafka.common.record" />
+        <allow pkg="org.apache.kafka.metadata" />
+        <allow pkg="org.apache.kafka.metalog" />
+        <allow pkg="org.apache.kafka.raft" />
+        <allow pkg="org.apache.kafka.snapshot" />
+        <allow pkg="org.apache.kafka.queue" />
+        <allow pkg="org.apache.kafka.server.common" />
+        <allow pkg="org.apache.kafka.test" />
+    </subpackage>
+
+</import-control>

checkstyle/import-control-server-common.xml

@@ -0,0 +1,82 @@
+<!DOCTYPE import-control PUBLIC
+        "-//Puppy Crawl//DTD Import Control 1.1//EN"
+        "http://www.puppycrawl.com/dtds/import_control_1_1.dtd">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<import-control pkg="org.apache.kafka">
+
+    <!-- THINK HARD ABOUT THE LAYERING OF THE PROJECT BEFORE CHANGING THIS FILE -->
+
+    <!-- common library dependencies -->
+    <allow pkg="java" />
+    <allow pkg="javax.management" />
+    <allow pkg="org.slf4j" />
+    <allow pkg="org.junit" />
+    <allow pkg="org.opentest4j" />
+    <allow pkg="org.hamcrest" />
+    <allow pkg="org.mockito" />
+    <allow pkg="org.easymock" />
+    <allow pkg="org.powermock" />
+    <allow pkg="java.security" />
+    <allow pkg="javax.net.ssl" />
+    <allow pkg="javax.security" />
+    <allow pkg="org.ietf.jgss" />
+    <allow pkg="net.jqwik.api" />
+
+    <!-- no one depends on the server -->
+    <disallow pkg="kafka" />
+
+    <!-- anyone can use public classes -->
+    <allow pkg="org.apache.kafka.common" exact-match="true" />
+    <allow pkg="org.apache.kafka.common.security" />
+    <allow pkg="org.apache.kafka.common.serialization" />
+    <allow pkg="org.apache.kafka.common.utils" />
+    <allow pkg="org.apache.kafka.common.errors" exact-match="true" />
+    <allow pkg="org.apache.kafka.common.memory" />
+
+    <!-- persistent collection factories/non-library-specific wrappers -->
+    <allow pkg="org.apache.kafka.server.immutable" exact-match="true" />
+
+    <subpackage name="queue">
+        <allow pkg="org.apache.kafka.test" />
+    </subpackage>
+
+    <subpackage name="server">
+        <allow pkg="org.apache.kafka.common" />
+        <allow pkg="joptsimple" />
+
+        <subpackage name="common">
+            <allow pkg="org.apache.kafka.server.common" />
+        </subpackage>
+
+        <subpackage name="immutable">
+            <allow pkg="org.apache.kafka.server.util"/>
+            <!-- only the factory package can use persistent collection library-specific wrapper implementations -->
+            <!-- the library-specific wrapper implementation for PCollections -->
+            <allow pkg="org.apache.kafka.server.immutable.pcollections" />
+            <subpackage name="pcollections">
+                <allow pkg="org.pcollections" />
+            </subpackage>
+        </subpackage>
+
+        <subpackage name="metrics">
+            <allow pkg="com.yammer.metrics" />
+        </subpackage>
+    </subpackage>
+
+</import-control>