From 5d83328f053526d695e7b2ded3f2bb5e837a1c67 Mon Sep 17 00:00:00 2001 From: "Colin P. McCabe" Date: Wed, 19 Oct 2022 15:06:10 -0700 Subject: [PATCH 01/15] KAFKA-14327: Unify KRaft snapshot generation between broker and controller Unify KRaft metadata snapshot generation between the broker and the controller, so that it all flows through the MetadataImage#write code path. Previously, the broker generated MetadataImage objects inside BrokerMetadataListener, and published them to a single publisher, named BrokerMetadataPublisher. Now, with this PR, MetadataImage objects are loaded by the MetadataLoader, a generic metadata class shared between controller and broker. The image objects are now published to a list of publishers, rather than just to a single publisher. Also, publishers can also be dynamically added and removed. On the broker side, we use the dynamic addition and removal to only add BrokerMetadataPublisher when our metadata has "caught up" to the controller's. With this PR, snapshot generation for both the controller and the broker is handled by a metadata publisher named SnapshotGenerator. This replaces BrokerMetadataSnapshotter and the controller's SnapshotGenerator. This means that the code for generating snapshots from the controller data structures can be deleted and we can maintain only one code path here. The new SnapshotGenerator implements time-based snapshotting as specified by KIP-876. Using the aforementioned image publisher framework, this PR adds support for dynamic configuration to the controller. Currently, only socket server, authorizer, and metrics reporter reconfiguration is supported, but more will be added in the future. This PR also changes how we handle changing the metadata version in MetadataDelta and MetadataImage. Previously, we called handleMetadataVersionChange on all delta objects whenever we encountered a metadata change in the log. This approach was difficult to maintain, however, since changing metadata versions can add or remove fields from both deltas and images. In practice, we did not correctly implement version changes, which is why we had to disable KRaft downgrades in the 3.3 release. The new approach here is to perform an image save and load cycle. That way we only have to maintain one code path for upgrades and downgrades, in MetadataImage#write. Because doing a save and load cycle is somewhat out of scope for MetadataDelta itself, it simply throws a MetadataVersionChangedException and lets the MetadataLoader handle the change. On a related note, this PR removes offset and epoch tracking from MetadataDelta. This tracking is done by external classes (such as MetadataLoader) so it doesn't need to be done in MetadataDelta. It also introduces a builder for MetadataDelta so that we can add new constructor parameters more easily in the future. (The only new parameter here is MetadataVersion.) There is a new class, MetadataProvenance, which encompasses offset, epoch, and timestamp, which identifies the source (provenance) of all MetadataImages. This is tracked externally from MetadataDelta, however. --- checkstyle/import-control-core.xml | 1 + checkstyle/import-control.xml | 8 + .../scala/kafka/server/BrokerServer.scala | 73 +-- .../scala/kafka/server/ControllerServer.scala | 21 +- .../kafka/server/DynamicBrokerConfig.scala | 21 + .../main/scala/kafka/server/KafkaConfig.scala | 9 +- .../scala/kafka/server/KafkaRaftServer.scala | 64 ++- .../metadata/BrokerMetadataListener.scala | 407 -------------- .../metadata/BrokerMetadataPublisher.scala | 38 +- .../metadata/BrokerMetadataSnapshotter.scala | 135 ----- .../server/metadata/BrokerServerMetrics.scala | 57 +- .../ControllerMetadataPublisher.scala | 126 +++++ .../server/metadata/MetadataPublisher.scala | 38 -- .../server/metadata/MetadataSnapshotter.scala | 37 -- .../test/java/kafka/test/MockController.java | 5 - .../kafka/testkit/KafkaClusterTestKit.java | 88 +-- .../kafka/server/QuorumTestHarness.scala | 61 ++- .../metadata/BrokerServerMetricsTest.scala | 36 +- .../unit/kafka/server/MetadataCacheTest.scala | 27 +- .../ReplicaManagerConcurrencyTest.scala | 8 +- .../kafka/server/ReplicaManagerTest.scala | 5 +- .../metadata/BrokerMetadataListenerTest.scala | 413 -------------- .../BrokerMetadataPublisherTest.scala | 8 +- .../BrokerMetadataSnapshotterTest.scala | 142 ----- .../scala/unit/kafka/utils/TestUtils.scala | 2 +- .../kafka/controller/AclControlManager.java | 19 +- .../controller/ClientQuotaControlManager.java | 43 -- .../controller/ClusterControlManager.java | 57 -- .../ConfigurationControlManager.java | 36 -- .../apache/kafka/controller/Controller.java | 8 - .../controller/FeatureControlManager.java | 44 -- .../controller/ProducerIdControlManager.java | 24 +- .../kafka/controller/QuorumController.java | 211 +------- .../controller/ReplicationControlManager.java | 34 -- .../kafka/controller/SnapshotGenerator.java | 132 ----- .../org/apache/kafka/image/FeaturesDelta.java | 60 ++- .../org/apache/kafka/image/MetadataDelta.java | 120 +++-- .../org/apache/kafka/image/MetadataImage.java | 26 +- .../kafka/image/MetadataProvenance.java | 83 +++ .../kafka/image/MetadataVersionChange.java | 69 +++ .../MetadataVersionChangeException.java} | 25 +- .../kafka/image/loader/LogDeltaManifest.java | 106 ++++ .../kafka/image/loader/MetadataLoader.java | 504 ++++++++++++++++++ .../image/loader/MetadataLoaderMetrics.java | 46 ++ .../loader/PreVersionChangeManifest.java | 79 +++ .../kafka/image/loader/SnapshotManifest.java | 77 +++ .../image/publisher/MetadataPublisher.java | 87 +++ .../image/publisher/SnapshotEmitter.java | 118 ++++ .../image/publisher/SnapshotGenerator.java | 262 +++++++++ .../kafka/image/writer/ImageReWriter.java | 59 ++ .../image/writer/ImageWriterOptions.java | 6 +- .../controller/AclControlManagerTest.java | 24 +- .../ClientQuotaControlManagerTest.java | 59 +- .../controller/ClusterControlManagerTest.java | 27 +- .../ConfigurationControlManagerTest.java | 8 - .../controller/FeatureControlManagerTest.java | 38 +- .../ProducerIdControlManagerTest.java | 30 +- .../controller/QuorumControllerTest.java | 366 ++----------- .../ReplicationControlManagerTest.java | 11 - .../controller/SnapshotGeneratorTest.java | 112 ---- .../apache/kafka/image/FeaturesImageTest.java | 31 ++ .../kafka/image/ImageDowngradeTest.java | 18 +- .../apache/kafka/image/MetadataImageTest.java | 40 +- .../image/publisher/SnapshotEmitterTest.java | 198 +++++++ .../publisher/SnapshotGeneratorTest.java | 201 +++++++ .../kafka/image/writer/ImageReWriterTest.java | 63 +++ .../image/writer/ImageWriterOptionsTest.java | 52 ++ .../kafka/metadata/RecordTestUtils.java | 48 -- .../apache/kafka/metalog/LocalLogManager.java | 19 +- .../kafka/metalog/LocalLogManagerTestEnv.java | 7 + .../org/apache/kafka/queue/EventQueue.java | 5 + .../apache/kafka/queue/KafkaEventQueue.java | 16 + .../fault/ProcessExitingFaultHandler.java | 14 + .../kafka/queue/KafkaEventQueueTest.java | 29 +- 74 files changed, 2914 insertions(+), 2667 deletions(-) delete mode 100644 core/src/main/scala/kafka/server/metadata/BrokerMetadataListener.scala delete mode 100644 core/src/main/scala/kafka/server/metadata/BrokerMetadataSnapshotter.scala create mode 100644 core/src/main/scala/kafka/server/metadata/ControllerMetadataPublisher.scala delete mode 100644 core/src/main/scala/kafka/server/metadata/MetadataPublisher.scala delete mode 100644 core/src/main/scala/kafka/server/metadata/MetadataSnapshotter.scala delete mode 100644 core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataListenerTest.scala delete mode 100644 core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataSnapshotterTest.scala delete mode 100644 metadata/src/main/java/org/apache/kafka/controller/SnapshotGenerator.java create mode 100644 metadata/src/main/java/org/apache/kafka/image/MetadataProvenance.java create mode 100644 metadata/src/main/java/org/apache/kafka/image/MetadataVersionChange.java rename metadata/src/main/java/org/apache/kafka/{metadata/util/SnapshotReason.java => image/MetadataVersionChangeException.java} (58%) create mode 100644 metadata/src/main/java/org/apache/kafka/image/loader/LogDeltaManifest.java create mode 100644 metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java create mode 100644 metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoaderMetrics.java create mode 100644 metadata/src/main/java/org/apache/kafka/image/loader/PreVersionChangeManifest.java create mode 100644 metadata/src/main/java/org/apache/kafka/image/loader/SnapshotManifest.java create mode 100644 metadata/src/main/java/org/apache/kafka/image/publisher/MetadataPublisher.java create mode 100644 metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotEmitter.java create mode 100644 metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java create mode 100644 metadata/src/main/java/org/apache/kafka/image/writer/ImageReWriter.java delete mode 100644 metadata/src/test/java/org/apache/kafka/controller/SnapshotGeneratorTest.java create mode 100644 metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotEmitterTest.java create mode 100644 metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotGeneratorTest.java create mode 100644 metadata/src/test/java/org/apache/kafka/image/writer/ImageReWriterTest.java create mode 100644 metadata/src/test/java/org/apache/kafka/image/writer/ImageWriterOptionsTest.java diff --git a/checkstyle/import-control-core.xml b/checkstyle/import-control-core.xml index 4042cba402fdd..82802ff4c68ad 100644 --- a/checkstyle/import-control-core.xml +++ b/checkstyle/import-control-core.xml @@ -49,6 +49,7 @@ + diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 63137cf6c1703..32809c4b79a55 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -225,6 +225,7 @@ + @@ -252,10 +253,13 @@ + + + @@ -478,6 +482,10 @@ + + + + diff --git a/core/src/main/scala/kafka/server/BrokerServer.scala b/core/src/main/scala/kafka/server/BrokerServer.scala index eeddff4ba3230..59eccdf447a73 100644 --- a/core/src/main/scala/kafka/server/BrokerServer.scala +++ b/core/src/main/scala/kafka/server/BrokerServer.scala @@ -22,7 +22,6 @@ import java.util import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.locks.ReentrantLock import java.util.concurrent.{CompletableFuture, ExecutionException, TimeUnit, TimeoutException} - import kafka.cluster.Broker.ServerInfo import kafka.coordinator.group.GroupCoordinator import kafka.coordinator.transaction.{ProducerIdManager, TransactionCoordinator} @@ -30,9 +29,7 @@ import kafka.log.LogManager import kafka.network.{DataPlaneAcceptor, SocketServer} import kafka.raft.RaftManager import kafka.security.CredentialProvider -import kafka.server.KafkaRaftServer.ControllerRole -import kafka.server.metadata.BrokerServerMetrics -import kafka.server.metadata.{BrokerMetadataListener, BrokerMetadataPublisher, BrokerMetadataSnapshotter, ClientQuotaMetadataManager, KRaftMetadataCache, SnapshotWriterBuilder} +import kafka.server.metadata.{BrokerMetadataPublisher, ClientQuotaMetadataManager, KRaftMetadataCache} import kafka.utils.{CoreUtils, KafkaScheduler} import org.apache.kafka.common.feature.SupportedVersionRange import org.apache.kafka.common.message.ApiMessageType.ListenerType @@ -44,30 +41,20 @@ import org.apache.kafka.common.security.scram.internals.ScramMechanism import org.apache.kafka.common.security.token.delegation.internals.DelegationTokenCache import org.apache.kafka.common.utils.{AppInfoParser, LogContext, Time, Utils} import org.apache.kafka.common.{ClusterResource, Endpoint} +import org.apache.kafka.image.loader.MetadataLoader import org.apache.kafka.metadata.authorizer.ClusterMetadataAuthorizer import org.apache.kafka.metadata.{BrokerState, VersionRange} import org.apache.kafka.raft.RaftConfig.AddressSpec -import org.apache.kafka.raft.{RaftClient, RaftConfig} +import org.apache.kafka.raft.RaftConfig import org.apache.kafka.server.authorizer.Authorizer import org.apache.kafka.server.common.ApiMessageAndVersion import org.apache.kafka.server.fault.FaultHandler import org.apache.kafka.server.metrics.KafkaYammerMetrics -import org.apache.kafka.snapshot.SnapshotWriter import scala.collection.{Map, Seq} -import scala.compat.java8.OptionConverters._ import scala.jdk.CollectionConverters._ -class BrokerSnapshotWriterBuilder(raftClient: RaftClient[ApiMessageAndVersion]) - extends SnapshotWriterBuilder { - override def build(committedOffset: Long, - committedEpoch: Int, - lastContainedLogTime: Long): Option[SnapshotWriter[ApiMessageAndVersion]] = { - raftClient.createSnapshot(committedOffset, committedEpoch, lastContainedLogTime).asScala - } -} - /** * A Kafka broker that runs in KRaft (Kafka Raft) mode. */ @@ -77,15 +64,16 @@ class BrokerServer( val raftManager: RaftManager[ApiMessageAndVersion], val time: Time, val metrics: Metrics, - val brokerMetrics: BrokerServerMetrics, val threadNamePrefix: Option[String], val initialOfflineDirs: Seq[String], val controllerQuorumVotersFuture: CompletableFuture[util.Map[Integer, AddressSpec]], val fatalFaultHandler: FaultHandler, - val metadataLoadingFaultHandler: FaultHandler, - val metadataPublishingFaultHandler: FaultHandler + val metadataPublishingFaultHandler: FaultHandler, + val metadataLoader: MetadataLoader, ) extends KafkaBroker { + class Builder + override def brokerState: BrokerState = Option(lifecycleManager). flatMap(m => Some(m.state)).getOrElse(BrokerState.NOT_RUNNING) @@ -146,10 +134,6 @@ class BrokerServer( val clusterId: String = metaProps.clusterId - var metadataSnapshotter: Option[BrokerMetadataSnapshotter] = None - - var metadataListener: BrokerMetadataListener = _ - var metadataPublisher: BrokerMetadataPublisher = _ val brokerFeatures: BrokerFeatures = BrokerFeatures.createDefault() @@ -304,25 +288,6 @@ class BrokerServer( ConfigType.Topic -> new TopicConfigHandler(logManager, config, quotaManagers, None), ConfigType.Broker -> new BrokerConfigHandler(config, quotaManagers)) - if (!config.processRoles.contains(ControllerRole)) { - // If no controller is defined, we rely on the broker to generate snapshots. - metadataSnapshotter = Some(new BrokerMetadataSnapshotter( - config.nodeId, - time, - threadNamePrefix, - new BrokerSnapshotWriterBuilder(raftManager.client) - )) - } - - metadataListener = new BrokerMetadataListener( - config.nodeId, - time, - threadNamePrefix, - config.metadataSnapshotMaxNewRecordBytes, - metadataSnapshotter, - brokerMetrics, - metadataLoadingFaultHandler) - val networkListeners = new ListenerCollection() config.effectiveAdvertisedListeners.foreach { ep => networkListeners.add(new Listener(). @@ -347,16 +312,13 @@ class BrokerServer( config.brokerSessionTimeoutMs.toLong ) lifecycleManager.start( - () => metadataListener.highestMetadataOffset, + () => metadataLoader.lastAppliedOffset(), brokerLifecycleChannelManager, metaProps.clusterId, networkListeners, featuresRemapped ) - // Register a listener with the Raft layer to receive metadata event notifications - raftManager.register(metadataListener) - val endpoints = new util.ArrayList[Endpoint](networkListeners.size()) var interBrokerListener: Endpoint = null networkListeners.iterator().forEachRemaining(listener => { @@ -426,7 +388,12 @@ class BrokerServer( DataPlaneAcceptor.ThreadPrefix) // Block until we've caught up with the latest metadata from the controller quorum. - lifecycleManager.initialCatchUpFuture.get() + try { + lifecycleManager.initialCatchUpFuture.get() + } catch { + case t: Throwable => throw new RuntimeException("Received a fatal error while " + + "waiting for the broker to catch up with the current cluster metadata.", t) + } // Apply the metadata log changes that we've accumulated. metadataPublisher = new BrokerMetadataPublisher(config, @@ -446,10 +413,10 @@ class BrokerServer( // replicaManager, groupCoordinator, and txnCoordinator. The log manager may perform // a potentially lengthy recovery-from-unclean-shutdown operation here, if required. try { - metadataListener.startPublishing(metadataPublisher).get() + metadataLoader.installPublishers(List(metadataPublisher).asJava).get() } catch { case t: Throwable => throw new RuntimeException("Received a fatal error while " + - "waiting for the broker to catch up with the current cluster metadata.", t) + "publishing the initial metadata update.", t) } // Log static broker configurations. @@ -506,8 +473,7 @@ class BrokerServer( } } - if (metadataListener != null) - metadataListener.beginShutdown() + metadataLoader.beginShutdown() lifecycleManager.beginShutdown() @@ -523,10 +489,7 @@ class BrokerServer( if (controlPlaneRequestProcessor != null) CoreUtils.swallow(controlPlaneRequestProcessor.close(), this) CoreUtils.swallow(authorizer.foreach(_.close()), this) - if (metadataListener != null) { - CoreUtils.swallow(metadataListener.close(), this) - } - metadataSnapshotter.foreach(snapshotter => CoreUtils.swallow(snapshotter.close(), this)) + CoreUtils.swallow(metadataLoader.close(), this) /** * We must shutdown the scheduler early because otherwise, the scheduler could touch other diff --git a/core/src/main/scala/kafka/server/ControllerServer.scala b/core/src/main/scala/kafka/server/ControllerServer.scala index 6e4c09b7a3780..f221640495ece 100644 --- a/core/src/main/scala/kafka/server/ControllerServer.scala +++ b/core/src/main/scala/kafka/server/ControllerServer.scala @@ -29,6 +29,7 @@ import kafka.security.CredentialProvider import kafka.server.KafkaConfig.{AlterConfigPolicyClassNameProp, CreateTopicPolicyClassNameProp} import kafka.server.KafkaRaftServer.BrokerRole import kafka.server.QuotaFactory.QuotaManagers +import kafka.server.metadata.ControllerMetadataPublisher import kafka.utils.{CoreUtils, Logging} import org.apache.kafka.clients.ApiVersions import org.apache.kafka.common.message.ApiMessageType.ListenerType @@ -44,12 +45,14 @@ import org.apache.kafka.raft.RaftConfig.AddressSpec import org.apache.kafka.server.authorizer.Authorizer import org.apache.kafka.server.common.ApiMessageAndVersion import org.apache.kafka.common.config.ConfigException +import org.apache.kafka.image.loader.MetadataLoader import org.apache.kafka.metadata.authorizer.ClusterMetadataAuthorizer import org.apache.kafka.metadata.bootstrap.BootstrapMetadata import org.apache.kafka.server.fault.FaultHandler import org.apache.kafka.server.metrics.KafkaYammerMetrics import org.apache.kafka.server.policy.{AlterConfigPolicy, CreateTopicPolicy} +import scala.collection.{Map, immutable} import scala.jdk.CollectionConverters._ import scala.compat.java8.OptionConverters._ @@ -70,6 +73,7 @@ class ControllerServer( val bootstrapMetadata: BootstrapMetadata, val metadataFaultHandler: FaultHandler, val fatalFaultHandler: FaultHandler, + val metadataLoader: MetadataLoader, ) extends Logging with KafkaMetricsGroup { import kafka.server.Server._ @@ -91,6 +95,9 @@ class ControllerServer( var quotaManagers: QuotaManagers = _ var controllerApis: ControllerApis = _ var controllerApisHandlerPool: KafkaRequestHandlerPool = _ + var dynamicConfigHandlers: immutable.Map[String, ConfigHandler] = _ + var controllerMetadataPublisher: ControllerMetadataPublisher = _ + def kafkaYammerMetrics: KafkaYammerMetrics = KafkaYammerMetrics.INSTANCE private def maybeChangeStatus(from: ProcessStatus, to: ProcessStatus): Boolean = { lock.lock() @@ -200,7 +207,6 @@ class ControllerServer( setDefaultNumPartitions(config.numPartitions.intValue()). setSessionTimeoutNs(TimeUnit.NANOSECONDS.convert(config.brokerSessionTimeoutMs.longValue(), TimeUnit.MILLISECONDS)). - setSnapshotMaxNewRecordBytes(config.metadataSnapshotMaxNewRecordBytes). setLeaderImbalanceCheckIntervalNs(leaderImbalanceCheckIntervalNs). setMaxIdleIntervalNs(maxIdleIntervalNs). setMetrics(controllerMetrics). @@ -258,6 +264,14 @@ class ControllerServer( shutdown() throw e } + config.dynamicConfig.addReconfigurables(this) + dynamicConfigHandlers = immutable.Map[String, ConfigHandler]( + ConfigType.Broker -> new BrokerConfigHandler(config, quotaManagers)) + controllerMetadataPublisher = new ControllerMetadataPublisher( + config, + dynamicConfigHandlers, + fatalFaultHandler) + metadataLoader.installPublishers(List().asJava) } def shutdown(): Unit = { @@ -266,8 +280,10 @@ class ControllerServer( info("shutting down") if (socketServer != null) CoreUtils.swallow(socketServer.stopProcessingRequests(), this) - if (controller != null) + if (controller != null) { controller.beginShutdown() + } + metadataLoader.beginShutdown() if (socketServer != null) CoreUtils.swallow(socketServer.shutdown(), this) if (controllerApisHandlerPool != null) @@ -278,6 +294,7 @@ class ControllerServer( CoreUtils.swallow(quotaManagers.shutdown(), this) if (controller != null) controller.close() + CoreUtils.swallow(metadataLoader.close(), this) CoreUtils.swallow(authorizer.foreach(_.close()), this) createTopicPolicy.foreach(policy => CoreUtils.swallow(policy.close(), this)) alterConfigPolicy.foreach(policy => CoreUtils.swallow(policy.close(), this)) diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala index 38f7722d34378..30f7db1b1146a 100755 --- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala +++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala @@ -25,6 +25,7 @@ import kafka.cluster.EndPoint import kafka.log.{LogCleaner, LogConfig, LogManager, ProducerStateManagerConfig} import kafka.network.{DataPlaneAcceptor, SocketServer} import kafka.server.DynamicBrokerConfig._ +import kafka.server.KafkaRaftServer.BrokerRole import kafka.utils.{CoreUtils, Logging, PasswordEncoder} import kafka.utils.Implicits._ import kafka.zk.{AdminZkClient, KafkaZkClient} @@ -269,6 +270,26 @@ class DynamicBrokerConfig(private val kafkaConfig: KafkaConfig) extends Logging addBrokerReconfigurable(kafkaServer.logManager.producerStateManagerConfig) } + /** + * Add reconfigurables to be notified when a dynamic controller config is updated. + */ + def addReconfigurables(controller: ControllerServer): Unit = { + controller.authorizer match { + case Some(authz: Reconfigurable) => addReconfigurable(authz) + case _ => + } + if (!kafkaConfig.processRoles.contains(BrokerRole)) { + // In combined mode, these elements are shared between broker and controller, and therefore + // don't need to be updated here (since the broker will do it.) + addReconfigurable(controller.kafkaYammerMetrics) + addReconfigurable(new DynamicMetricsReporters( + kafkaConfig.brokerId, controller.config, controller.metrics, controller.clusterId)) + } + // TODO: add dynamic thread pool resizing here + // TODO: add dynamic listener reconfiguration here + addBrokerReconfigurable(controller.socketServer) + } + def addReconfigurable(reconfigurable: Reconfigurable): Unit = CoreUtils.inWriteLock(lock) { verifyReconfigurableConfigs(reconfigurable.reconfigurableConfigs.asScala) reconfigurables.add(reconfigurable) diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala index 6fcc7ba40b132..9c14816a002bd 100755 --- a/core/src/main/scala/kafka/server/KafkaConfig.scala +++ b/core/src/main/scala/kafka/server/KafkaConfig.scala @@ -81,6 +81,7 @@ object Defaults { val BrokerHeartbeatIntervalMs = 2000 val BrokerSessionTimeoutMs = 9000 val MetadataSnapshotMaxNewRecordBytes = 20 * 1024 * 1024 + val MetadataSnapshotMaxIntervalMs = TimeUnit.HOURS.toMillis(1) val MetadataMaxIdleIntervalMs = 500 /** KRaft mode configs */ @@ -400,6 +401,7 @@ object KafkaConfig { val NodeIdProp = "node.id" val MetadataLogDirProp = "metadata.log.dir" val MetadataSnapshotMaxNewRecordBytesProp = "metadata.log.max.record.bytes.between.snapshots" + val MetadataSnapshotMaxIntervalMsProp = "metadata.log.max.snapshot.interval.ms" val ControllerListenerNamesProp = "controller.listener.names" val SaslMechanismControllerProtocolProp = "sasl.mechanism.controller.protocol" val MetadataLogSegmentMinBytesProp = "metadata.log.segment.min.bytes" @@ -726,6 +728,7 @@ object KafkaConfig { val MetadataLogDirDoc = "This configuration determines where we put the metadata log for clusters in KRaft mode. " + "If it is not set, the metadata log is placed in the first log directory from log.dirs." val MetadataSnapshotMaxNewRecordBytesDoc = "This is the maximum number of bytes in the log between the latest snapshot and the high-watermark needed before generating a new snapshot." + val MetadataSnapshotMaxIntervalMsDoc = "This is the maximum number of milliseconds that can elapse between the previous metadata snapshot and the next one." val MetadataMaxIdleIntervalMsDoc = "This configuration controls how often the active " + "controller should write no-op records to the metadata partition. If the value is 0, no-op records " + s"are not appended to the metadata partition. The default value is ${Defaults.MetadataMaxIdleIntervalMs}"; @@ -1157,10 +1160,7 @@ object KafkaConfig { * KRaft mode configs. */ .define(MetadataSnapshotMaxNewRecordBytesProp, LONG, Defaults.MetadataSnapshotMaxNewRecordBytes, atLeast(1), HIGH, MetadataSnapshotMaxNewRecordBytesDoc) - - /* - * KRaft mode private configs. Note that these configs are defined as internal. We will make them public in the 3.0.0 release. - */ + .define(MetadataSnapshotMaxIntervalMsProp, LONG, Defaults.MetadataSnapshotMaxIntervalMs, atLeast(0), HIGH, MetadataSnapshotMaxIntervalMsDoc) .define(ProcessRolesProp, LIST, Collections.emptyList(), ValidList.in("broker", "controller"), HIGH, ProcessRolesDoc) .define(NodeIdProp, INT, Defaults.EmptyNodeId, null, HIGH, NodeIdDoc) .define(InitialBrokerRegistrationTimeoutMsProp, INT, Defaults.InitialBrokerRegistrationTimeoutMs, null, MEDIUM, InitialBrokerRegistrationTimeoutMsDoc) @@ -1697,6 +1697,7 @@ class KafkaConfig private(doLog: Boolean, val props: java.util.Map[_, _], dynami /************* Metadata Configuration ***********/ val metadataSnapshotMaxNewRecordBytes = getLong(KafkaConfig.MetadataSnapshotMaxNewRecordBytesProp) + val metadataSnapshotMaxIntervalMsProp = getLong(KafkaConfig.MetadataSnapshotMaxIntervalMsProp) val metadataMaxIdleIntervalNs: Option[Long] = { val value = TimeUnit.NANOSECONDS.convert(getInt(KafkaConfig.MetadataMaxIdleIntervalMsProp).toLong, TimeUnit.MILLISECONDS) if (value > 0) Some(value) else None diff --git a/core/src/main/scala/kafka/server/KafkaRaftServer.scala b/core/src/main/scala/kafka/server/KafkaRaftServer.scala index 84ea10da57fa1..3e9bc9530aa46 100644 --- a/core/src/main/scala/kafka/server/KafkaRaftServer.scala +++ b/core/src/main/scala/kafka/server/KafkaRaftServer.scala @@ -17,7 +17,7 @@ package kafka.server import java.io.File -import java.util.concurrent.CompletableFuture +import java.util.concurrent.{CompletableFuture, TimeUnit} import kafka.common.InconsistentNodeIdException import kafka.log.{LogConfig, UnifiedLog} import kafka.metrics.KafkaMetricsReporter @@ -30,6 +30,8 @@ import org.apache.kafka.common.internals.Topic import org.apache.kafka.common.utils.{AppInfoParser, Time} import org.apache.kafka.common.{KafkaException, Uuid} import org.apache.kafka.controller.QuorumControllerMetrics +import org.apache.kafka.image.loader.MetadataLoader +import org.apache.kafka.image.publisher.{SnapshotEmitter, SnapshotGenerator} import org.apache.kafka.metadata.bootstrap.{BootstrapDirectory, BootstrapMetadata} import org.apache.kafka.metadata.{KafkaConfigSchema, MetadataRecordSerde} import org.apache.kafka.raft.RaftConfig @@ -81,26 +83,69 @@ class KafkaRaftServer( controllerQuorumVotersFuture ) + private val snapshotEmitter: SnapshotEmitter = new SnapshotEmitter.Builder(). + setNodeId(config.nodeId). + setRaftClient(raftManager.client). + build() + + private val snapshotGenerator: SnapshotGenerator = new SnapshotGenerator.Builder(). + setTime(Time.SYSTEM). + setNodeId(config.nodeId). + setFaultHandler(new LoggingFaultHandler("snapshot generation", () => { })). + setEmitter(snapshotEmitter). + setMinBytesSinceLastSnapshot(config.metadataSnapshotMaxNewRecordBytes). + setMinTimeSinceLastSnapshotNs(TimeUnit.MILLISECONDS.toNanos(config.metadataSnapshotMaxIntervalMsProp)). + build() + + private val brokerMetrics: Option[BrokerServerMetrics] = if (config.processRoles.contains(BrokerRole)) { + Some(BrokerServerMetrics(metrics)) + } else { + None + } + + private val metadataLoader: MetadataLoader = { + val builder = new MetadataLoader.Builder(). + setTime(Time.SYSTEM). + setNodeId(config.nodeId) + if (config.processRoles.contains(ControllerRole)) { + // Controllers and combined nodes must exit immediately if there is a metadata loading error. + // We disable snapshot generation here so that we don't create a snapshot while exiting. + builder.setFaultHandler(new ProcessExitingFaultHandler(() => { + snapshotGenerator.disable("there was a metadata loading error") + })) + } else { + // Broker-only nodes disable snapshot creation and increment a metric if there is a metadata + // loading error. + builder.setFaultHandler(new LoggingFaultHandler("metadata loading", () => { + snapshotGenerator.disable("there was a metadata loading error") + brokerMetrics.get.metadataLoadErrorCount.getAndIncrement() + })) + } + if (config.processRoles.contains(BrokerRole)) { + // In broker-only mode or combined mode, we want to expose the broker metrics. + builder.setMetadataLoaderMetrics(brokerMetrics.get) + } + builder.build() + } + + raftManager.register(metadataLoader) + private val broker: Option[BrokerServer] = if (config.processRoles.contains(BrokerRole)) { - val brokerMetrics = BrokerServerMetrics(metrics) val fatalFaultHandler = new ProcessExitingFaultHandler() - val metadataLoadingFaultHandler = new LoggingFaultHandler("metadata loading", - () => brokerMetrics.metadataLoadErrorCount.getAndIncrement()) val metadataApplyingFaultHandler = new LoggingFaultHandler("metadata application", - () => brokerMetrics.metadataApplyErrorCount.getAndIncrement()) + () => brokerMetrics.get.metadataApplyErrorCount.getAndIncrement()) Some(new BrokerServer( config, metaProps, raftManager, time, metrics, - brokerMetrics, threadNamePrefix, offlineDirs, controllerQuorumVotersFuture, fatalFaultHandler, - metadataLoadingFaultHandler, - metadataApplyingFaultHandler + metadataApplyingFaultHandler, + metadataLoader, )) } else { None @@ -124,7 +169,8 @@ class KafkaRaftServer( raftManager.apiVersions, bootstrapMetadata, metadataFaultHandler, - fatalFaultHandler + fatalFaultHandler, + metadataLoader, )) } else { None diff --git a/core/src/main/scala/kafka/server/metadata/BrokerMetadataListener.scala b/core/src/main/scala/kafka/server/metadata/BrokerMetadataListener.scala deleted file mode 100644 index 179d934ddad8b..0000000000000 --- a/core/src/main/scala/kafka/server/metadata/BrokerMetadataListener.scala +++ /dev/null @@ -1,407 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package kafka.server.metadata - -import java.util -import java.util.concurrent.{CompletableFuture, TimeUnit} -import kafka.metrics.KafkaMetricsGroup -import org.apache.kafka.image.{MetadataDelta, MetadataImage} -import org.apache.kafka.common.utils.{LogContext, Time} -import org.apache.kafka.image.writer.{ImageWriterOptions, RecordListWriter} -import org.apache.kafka.metadata.util.SnapshotReason -import org.apache.kafka.queue.{EventQueue, KafkaEventQueue} -import org.apache.kafka.raft.{Batch, BatchReader, LeaderAndEpoch, RaftClient} -import org.apache.kafka.server.common.ApiMessageAndVersion -import org.apache.kafka.server.fault.FaultHandler -import org.apache.kafka.snapshot.SnapshotReader - -import java.util.concurrent.atomic.AtomicBoolean - - -object BrokerMetadataListener { - val MetadataBatchProcessingTimeUs = "MetadataBatchProcessingTimeUs" - val MetadataBatchSizes = "MetadataBatchSizes" -} - -class BrokerMetadataListener( - val brokerId: Int, - time: Time, - threadNamePrefix: Option[String], - val maxBytesBetweenSnapshots: Long, - val snapshotter: Option[MetadataSnapshotter], - brokerMetrics: BrokerServerMetrics, - _metadataLoadingFaultHandler: FaultHandler -) extends RaftClient.Listener[ApiMessageAndVersion] with KafkaMetricsGroup { - - private val metadataFaultOccurred = new AtomicBoolean(false) - private val metadataLoadingFaultHandler: FaultHandler = new FaultHandler() { - override def handleFault(failureMessage: String, cause: Throwable): RuntimeException = { - // If the broker has any kind of error handling metadata records or publishing a new image - // we will disable taking new snapshots in order to preserve the local metadata log. Once we - // encounter a metadata processing error, the broker will be in an undetermined state. - if (metadataFaultOccurred.compareAndSet(false, true)) { - error("Disabling metadata snapshots until this broker is restarted.") - } - _metadataLoadingFaultHandler.handleFault(failureMessage, cause) - } - } - - private val logContext = new LogContext(s"[BrokerMetadataListener id=$brokerId] ") - private val log = logContext.logger(classOf[BrokerMetadataListener]) - logIdent = logContext.logPrefix() - - /** - * A histogram tracking the time in microseconds it took to process batches of events. - */ - private val batchProcessingTimeHist = newHistogram(BrokerMetadataListener.MetadataBatchProcessingTimeUs) - - /** - * A histogram tracking the sizes of batches that we have processed. - */ - private val metadataBatchSizeHist = newHistogram(BrokerMetadataListener.MetadataBatchSizes) - - /** - * The highest metadata offset that we've seen. Written only from the event queue thread. - */ - @volatile var _highestOffset = -1L - - /** - * The highest metadata log time that we've seen. Written only from the event queue thread. - */ - private var _highestTimestamp = -1L - - /** - * The current broker metadata image. Accessed only from the event queue thread. - */ - private var _image = MetadataImage.EMPTY - - /** - * The current metadata delta. Accessed only from the event queue thread. - */ - private var _delta = new MetadataDelta(_image) - - /** - * The object to use to publish new metadata changes, or None if this listener has not - * been activated yet. Accessed only from the event queue thread. - */ - private var _publisher: Option[MetadataPublisher] = None - - /** - * The number of bytes of records that we have read since the last snapshot we took. - * This does not include records we read from a snapshot. - * Accessed only from the event queue thread. - */ - private var _bytesSinceLastSnapshot: Long = 0L - - /** - * The event queue which runs this listener. - */ - val eventQueue = new KafkaEventQueue(time, logContext, threadNamePrefix.getOrElse("")) - - /** - * Returns the highest metadata-offset. Thread-safe. - */ - def highestMetadataOffset: Long = _highestOffset - - /** - * Handle new metadata records. - */ - override def handleCommit(reader: BatchReader[ApiMessageAndVersion]): Unit = - eventQueue.append(new HandleCommitsEvent(reader)) - - class HandleCommitsEvent(reader: BatchReader[ApiMessageAndVersion]) - extends EventQueue.FailureLoggingEvent(log) { - override def run(): Unit = { - val results = try { - val loadResults = loadBatches(_delta, reader, None, None, None, None) - if (isDebugEnabled) { - debug(s"Loaded new commits: $loadResults") - } - loadResults - } catch { - case e: Throwable => - metadataLoadingFaultHandler.handleFault(s"Unable to load metadata commits " + - s"from the BatchReader starting at base offset ${reader.baseOffset()}", e) - return - } finally { - reader.close() - } - - _bytesSinceLastSnapshot = _bytesSinceLastSnapshot + results.numBytes - - val shouldTakeSnapshot: Set[SnapshotReason] = shouldSnapshot() - if (shouldTakeSnapshot.nonEmpty) { - maybeStartSnapshot(shouldTakeSnapshot) - } - - _publisher.foreach(publish) - } - } - - private def shouldSnapshot(): Set[SnapshotReason] = { - val metadataVersionHasChanged = metadataVersionChanged() - val maxBytesHaveExceeded = (_bytesSinceLastSnapshot >= maxBytesBetweenSnapshots) - - if (maxBytesHaveExceeded && metadataVersionHasChanged) { - Set(SnapshotReason.MetadataVersionChanged, SnapshotReason.MaxBytesExceeded) - } else if (maxBytesHaveExceeded) { - Set(SnapshotReason.MaxBytesExceeded) - } else if (metadataVersionHasChanged) { - Set(SnapshotReason.MetadataVersionChanged) - } else { - Set() - } - } - - private def metadataVersionChanged(): Boolean = { - // The _publisher is empty before starting publishing, and we won't compute feature delta - // until we starting publishing - _publisher.nonEmpty && Option(_delta.featuresDelta()).exists { featuresDelta => - featuresDelta.metadataVersionChange().isPresent - } - } - - private def maybeStartSnapshot(reason: Set[SnapshotReason]): Unit = { - snapshotter.foreach { snapshotter => - if (metadataFaultOccurred.get()) { - trace("Not starting metadata snapshot since we previously had an error") - } else if (snapshotter.maybeStartSnapshot(_highestTimestamp, _delta.apply(), reason)) { - _bytesSinceLastSnapshot = 0L - } - } - } - - /** - * Handle metadata snapshots - */ - override def handleSnapshot(reader: SnapshotReader[ApiMessageAndVersion]): Unit = - eventQueue.append(new HandleSnapshotEvent(reader)) - - class HandleSnapshotEvent(reader: SnapshotReader[ApiMessageAndVersion]) - extends EventQueue.FailureLoggingEvent(log) { - override def run(): Unit = { - val snapshotName = s"${reader.snapshotId().offset}-${reader.snapshotId().epoch}" - try { - info(s"Loading snapshot ${snapshotName}") - _delta = new MetadataDelta(_image) // Discard any previous deltas. - val loadResults = loadBatches(_delta, - reader, - Some(reader.lastContainedLogTimestamp), - Some(reader.lastContainedLogOffset), - Some(reader.lastContainedLogEpoch), - Some(snapshotName)) - try { - _delta.finishSnapshot() - } catch { - case e: Throwable => metadataLoadingFaultHandler.handleFault( - s"Error finishing snapshot ${snapshotName}", e) - } - info(s"Loaded snapshot ${snapshotName}: ${loadResults}") - } catch { - case t: Throwable => metadataLoadingFaultHandler.handleFault("Uncaught exception while " + - s"loading broker metadata from Metadata snapshot ${snapshotName}", t) - } finally { - reader.close() - } - _publisher.foreach(publish) - } - } - - case class BatchLoadResults(numBatches: Int, numRecords: Int, elapsedUs: Long, numBytes: Long) { - override def toString: String = { - s"$numBatches batch(es) with $numRecords record(s) in $numBytes bytes " + - s"ending at offset $highestMetadataOffset in $elapsedUs microseconds" - } - } - - /** - * Load and replay the batches to the metadata delta. - * - * When loading and replay a snapshot the appendTimestamp and snapshotId parameter should be provided. - * In a snapshot the append timestamp, offset and epoch reported by the batch is independent of the ones - * reported by the metadata log. - * - * @param delta metadata delta on which to replay the records - * @param iterator sequence of metadata record bacthes to replay - * @param lastAppendTimestamp optional append timestamp to use instead of the batches timestamp - * @param lastCommittedOffset optional offset to use instead of the batches offset - * @param lastCommittedEpoch optional epoch to use instead of the batches epoch - */ - private def loadBatches( - delta: MetadataDelta, - iterator: util.Iterator[Batch[ApiMessageAndVersion]], - lastAppendTimestamp: Option[Long], - lastCommittedOffset: Option[Long], - lastCommittedEpoch: Option[Int], - snapshotName: Option[String] - ): BatchLoadResults = { - val startTimeNs = time.nanoseconds() - var numBatches = 0 - var numRecords = 0 - var numBytes = 0L - - while (iterator.hasNext) { - val batch = iterator.next() - - val epoch = lastCommittedEpoch.getOrElse(batch.epoch()) - _highestTimestamp = lastAppendTimestamp.getOrElse(batch.appendTimestamp()) - - var index = 0 - batch.records().forEach { messageAndVersion => - if (isTraceEnabled) { - trace(s"Metadata batch ${batch.lastOffset}: processing [${index + 1}/${batch.records.size}]:" + - s" ${messageAndVersion.message}") - } - _highestOffset = lastCommittedOffset.getOrElse(batch.baseOffset() + index) - try { - delta.replay(highestMetadataOffset, epoch, messageAndVersion.message()) - } catch { - case e: Throwable => snapshotName match { - case None => metadataLoadingFaultHandler.handleFault( - s"Error replaying metadata log record at offset ${_highestOffset}", e) - case Some(name) => metadataLoadingFaultHandler.handleFault( - s"Error replaying record ${index} from snapshot ${name} at offset ${_highestOffset}", e) - } - } finally { - numRecords += 1 - index += 1 - } - } - numBytes = numBytes + batch.sizeInBytes() - metadataBatchSizeHist.update(batch.records().size()) - numBatches = numBatches + 1 - } - - val endTimeNs = time.nanoseconds() - val elapsedUs = TimeUnit.MICROSECONDS.convert(endTimeNs - startTimeNs, TimeUnit.NANOSECONDS) - batchProcessingTimeHist.update(elapsedUs) - BatchLoadResults(numBatches, numRecords, elapsedUs, numBytes) - } - - def startPublishing(publisher: MetadataPublisher): CompletableFuture[Void] = { - val event = new StartPublishingEvent(publisher) - eventQueue.append(event) - event.future - } - - class StartPublishingEvent(publisher: MetadataPublisher) - extends EventQueue.FailureLoggingEvent(log) { - val future = new CompletableFuture[Void]() - - override def run(): Unit = { - _publisher = Some(publisher) - log.info(s"Starting to publish metadata events at offset $highestMetadataOffset.") - try { - if (metadataVersionChanged()) { - maybeStartSnapshot(Set(SnapshotReason.MetadataVersionChanged)) - } - publish(publisher) - future.complete(null) - } catch { - case e: Throwable => - future.completeExceptionally(e) - throw e - } - } - } - - // This is used in tests to alter the publisher that is in use by the broker. - def alterPublisher(publisher: MetadataPublisher): CompletableFuture[Void] = { - val event = new AlterPublisherEvent(publisher) - eventQueue.append(event) - event.future - } - - class AlterPublisherEvent(publisher: MetadataPublisher) - extends EventQueue.FailureLoggingEvent(log) { - val future = new CompletableFuture[Void]() - - override def run(): Unit = { - _publisher = Some(publisher) - log.info(s"Set publisher to ${publisher}") - future.complete(null) - } - } - - private def publish(publisher: MetadataPublisher): Unit = { - val delta = _delta - try { - _image = _delta.apply() - } catch { - case t: Throwable => - // If we cannot apply the delta, this publish event will throw and we will not publish a new image. - // The broker will continue applying metadata records and attempt to publish again. - throw metadataLoadingFaultHandler.handleFault(s"Error applying metadata delta $delta", t) - } - - _delta = new MetadataDelta(_image) - if (isDebugEnabled) { - debug(s"Publishing new metadata delta $delta at offset ${_image.highestOffsetAndEpoch().offset}.") - } - - // This publish call is done with its own try-catch and fault handler - publisher.publish(delta, _image) - - // Update the metrics since the publisher handled the lastest image - brokerMetrics.lastAppliedRecordOffset.set(_highestOffset) - brokerMetrics.lastAppliedRecordTimestamp.set(_highestTimestamp) - } - - override def handleLeaderChange(leaderAndEpoch: LeaderAndEpoch): Unit = { - // Nothing to do. - } - - override def beginShutdown(): Unit = { - eventQueue.beginShutdown("beginShutdown", new ShutdownEvent()) - } - - class ShutdownEvent extends EventQueue.FailureLoggingEvent(log) { - override def run(): Unit = { - brokerMetrics.close() - removeMetric(BrokerMetadataListener.MetadataBatchProcessingTimeUs) - removeMetric(BrokerMetadataListener.MetadataBatchSizes) - } - } - - def close(): Unit = { - beginShutdown() - eventQueue.close() - } - - // VisibleForTesting - private[kafka] def getImageRecords(): CompletableFuture[util.List[ApiMessageAndVersion]] = { - val future = new CompletableFuture[util.List[ApiMessageAndVersion]]() - eventQueue.append(new GetImageRecordsEvent(future)) - future - } - - class GetImageRecordsEvent(future: CompletableFuture[util.List[ApiMessageAndVersion]]) - extends EventQueue.FailureLoggingEvent(log) { - override def run(): Unit = { - val writer = new RecordListWriter() - val options = new ImageWriterOptions.Builder(). - setMetadataVersion(_image.features().metadataVersion()). - build() - try { - _image.write(writer, options) - } finally { - writer.close() - } - future.complete(writer.records()) - } - } -} diff --git a/core/src/main/scala/kafka/server/metadata/BrokerMetadataPublisher.scala b/core/src/main/scala/kafka/server/metadata/BrokerMetadataPublisher.scala index 0192bb4afcfd7..70432fdc1cc14 100644 --- a/core/src/main/scala/kafka/server/metadata/BrokerMetadataPublisher.scala +++ b/core/src/main/scala/kafka/server/metadata/BrokerMetadataPublisher.scala @@ -18,7 +18,6 @@ package kafka.server.metadata import java.util.Properties -import java.util.concurrent.atomic.AtomicLong import kafka.coordinator.group.GroupCoordinator import kafka.coordinator.transaction.TransactionCoordinator import kafka.log.{LogManager, UnifiedLog} @@ -28,6 +27,8 @@ import kafka.utils.Logging import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.config.ConfigResource.Type.{BROKER, TOPIC} import org.apache.kafka.common.internals.Topic +import org.apache.kafka.image.loader.{LogDeltaManifest, SnapshotManifest} +import org.apache.kafka.image.publisher.MetadataPublisher import org.apache.kafka.image.{MetadataDelta, MetadataImage, TopicDelta, TopicsImage} import org.apache.kafka.metadata.authorizer.ClusterMetadataAuthorizer import org.apache.kafka.server.authorizer.Authorizer @@ -122,12 +123,25 @@ class BrokerMetadataPublisher( */ var _firstPublish = true - /** - * This is updated after all components (e.g. LogManager) has finished publishing the new metadata delta - */ - val publishedOffsetAtomic = new AtomicLong(-1) + override def name(): String = "BrokerMetadataPublisher" - override def publish(delta: MetadataDelta, newImage: MetadataImage): Unit = { + override def publishSnapshot( + delta: MetadataDelta, + newImage: MetadataImage, + manifest: SnapshotManifest + ): Unit = { + publish(delta, newImage) + } + + override def publishLogDelta( + delta: MetadataDelta, + newImage: MetadataImage, + manifest: LogDeltaManifest + ): Unit = { + publish(delta, newImage) + } + + def publish(delta: MetadataDelta, newImage: MetadataImage): Unit = { val highestOffsetAndEpoch = newImage.highestOffsetAndEpoch() val deltaName = if (_firstPublish) { @@ -156,8 +170,9 @@ class BrokerMetadataPublisher( } Option(delta.featuresDelta()).foreach { featuresDelta => - featuresDelta.metadataVersionChange().ifPresent{ metadataVersion => - info(s"Updating metadata.version to ${metadataVersion.featureLevel()} at offset $highestOffsetAndEpoch.") + featuresDelta.metadataVersionChange().ifPresent{ change => + info(s"Changing metadata.version from ${change.oldVersion()} to ${change.newVersion()} in " + + s"delta ending at offset ${highestOffsetAndEpoch}.") } } @@ -314,7 +329,6 @@ class BrokerMetadataPublisher( if (_firstPublish) { finishInitializingReplicaManager(newImage) } - publishedOffsetAtomic.set(newImage.highestOffsetAndEpoch().offset) } catch { case t: Throwable => metadataPublishingFaultHandler.handleFault("Uncaught exception while " + s"publishing broker metadata from ${deltaName}", t) @@ -323,8 +337,6 @@ class BrokerMetadataPublisher( } } - override def publishedOffset: Long = publishedOffsetAtomic.get() - def reloadUpdatedFilesWithoutConfigChange(props: Properties): Unit = { conf.dynamicConfig.reloadUpdatedFilesWithoutConfigChange(props) } @@ -431,4 +443,8 @@ class BrokerMetadataPublisher( "watermark checkpoint thread during startup", t) } } + + override def close(): Unit = { + // nothing to do + } } diff --git a/core/src/main/scala/kafka/server/metadata/BrokerMetadataSnapshotter.scala b/core/src/main/scala/kafka/server/metadata/BrokerMetadataSnapshotter.scala deleted file mode 100644 index 6c3d55be92bcb..0000000000000 --- a/core/src/main/scala/kafka/server/metadata/BrokerMetadataSnapshotter.scala +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package kafka.server.metadata - -import java.util.concurrent.RejectedExecutionException -import kafka.utils.Logging -import org.apache.kafka.image.MetadataImage -import org.apache.kafka.common.utils.{LogContext, Time} -import org.apache.kafka.image.writer.{ImageWriterOptions, RaftSnapshotWriter} -import org.apache.kafka.metadata.util.SnapshotReason -import org.apache.kafka.queue.{EventQueue, KafkaEventQueue} -import org.apache.kafka.server.common.ApiMessageAndVersion -import org.apache.kafka.snapshot.SnapshotWriter - -trait SnapshotWriterBuilder { - def build(committedOffset: Long, - committedEpoch: Int, - lastContainedLogTime: Long): Option[SnapshotWriter[ApiMessageAndVersion]] -} - -class BrokerMetadataSnapshotter( - brokerId: Int, - val time: Time, - threadNamePrefix: Option[String], - writerBuilder: SnapshotWriterBuilder -) extends Logging with MetadataSnapshotter { - /** - * The maximum number of records we will put in each batch. - * - * From the perspective of the Raft layer, the limit on batch size is specified in terms of - * bytes, not number of records. @See {@link KafkaRaftClient#MAX_BATCH_SIZE_BYTES} for details. - * However, it's more convenient to limit the batch size here in terms of number of records. - * So we chose a low number that will not cause problems. - */ - private val maxRecordsInBatch = 1024 - - private val logContext = new LogContext(s"[BrokerMetadataSnapshotter id=$brokerId] ") - logIdent = logContext.logPrefix() - - /** - * The offset of the snapshot in progress, or -1 if there isn't one. Accessed only under - * the object lock. - */ - private var _currentSnapshotOffset = -1L - - /** - * The event queue which runs this listener. - */ - val eventQueue = new KafkaEventQueue(time, logContext, threadNamePrefix.getOrElse("")) - - override def maybeStartSnapshot(lastContainedLogTime: Long, image: MetadataImage, snapshotReasons: Set[SnapshotReason]): Boolean = synchronized { - if (_currentSnapshotOffset != -1) { - info(s"Declining to create a new snapshot at ${image.highestOffsetAndEpoch()} because " + - s"there is already a snapshot in progress at offset ${_currentSnapshotOffset}") - false - } else { - val writer = writerBuilder.build( - image.highestOffsetAndEpoch().offset, - image.highestOffsetAndEpoch().epoch, - lastContainedLogTime - ) - if (writer.nonEmpty) { - _currentSnapshotOffset = image.highestOffsetAndEpoch().offset - - info(s"Creating a new snapshot at offset ${_currentSnapshotOffset} because, ${snapshotReasons.mkString(" and ")}") - eventQueue.append(new CreateSnapshotEvent(image, writer.get)) - true - } else { - info(s"Declining to create a new snapshot at ${image.highestOffsetAndEpoch()} because " + - s"there is already a snapshot at offset ${image.highestOffsetAndEpoch().offset}") - false - } - } - } - - class CreateSnapshotEvent( - image: MetadataImage, - snapshotWriter: SnapshotWriter[ApiMessageAndVersion] - ) extends EventQueue.Event { - - override def run(): Unit = { - val writer = new RaftSnapshotWriter(snapshotWriter, maxRecordsInBatch) - val options = new ImageWriterOptions.Builder(). - setMetadataVersion(image.features().metadataVersion()). - build() - try { - image.write(writer, options) - } finally { - try { - writer.close() - } finally { - BrokerMetadataSnapshotter.this.synchronized { - _currentSnapshotOffset = -1L - } - } - } - } - - override def handleException(e: Throwable): Unit = { - e match { - case _: RejectedExecutionException => - info("Not processing CreateSnapshotEvent because the event queue is closed.") - case _ => error("Unexpected error handling CreateSnapshotEvent", e) - } - } - } - - def beginShutdown(): Unit = { - eventQueue.beginShutdown("beginShutdown", new ShutdownEvent()) - } - - class ShutdownEvent() extends EventQueue.Event { - override def run(): Unit = { - } - } - - def close(): Unit = { - beginShutdown() - eventQueue.close() - } -} diff --git a/core/src/main/scala/kafka/server/metadata/BrokerServerMetrics.scala b/core/src/main/scala/kafka/server/metadata/BrokerServerMetrics.scala index 3e68ae85f9232..c52d6254a06be 100644 --- a/core/src/main/scala/kafka/server/metadata/BrokerServerMetrics.scala +++ b/core/src/main/scala/kafka/server/metadata/BrokerServerMetrics.scala @@ -17,17 +17,48 @@ package kafka.server.metadata -import java.util.concurrent.atomic.AtomicLong +import kafka.metrics.KafkaMetricsGroup + +import java.util.concurrent.atomic.{AtomicLong, AtomicReference} import org.apache.kafka.common.MetricName import org.apache.kafka.common.metrics.Gauge import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.metrics.MetricConfig +import org.apache.kafka.image.MetadataProvenance +import org.apache.kafka.image.loader.MetadataLoaderMetrics +import org.apache.kafka.server.metrics.KafkaYammerMetrics + +import java.util.concurrent.TimeUnit.{MICROSECONDS, NANOSECONDS} -final class BrokerServerMetrics private (metrics: Metrics) extends AutoCloseable { +final class BrokerServerMetrics private ( + metrics: Metrics +) extends MetadataLoaderMetrics with KafkaMetricsGroup { import BrokerServerMetrics._ - val lastAppliedRecordOffset: AtomicLong = new AtomicLong(0) - val lastAppliedRecordTimestamp: AtomicLong = new AtomicLong(0) + private val batchProcessingTimeHistName = explicitMetricName("kafka.server", + "BrokerMetadataListener", + "MetadataBatchProcessingTimeUs", + Map.empty) + + /** + * A histogram tracking the time in microseconds it took to process batches of events. + */ + private val batchProcessingTimeHist = + KafkaYammerMetrics.defaultRegistry().newHistogram(batchProcessingTimeHistName, true) + + private val batchSizeHistName = explicitMetricName("kafka.server", + "BrokerMetadataListener", + "MetadataBatchSizes", + Map.empty) + + /** + * A histogram tracking the sizes of batches that we have processed. + */ + private val batchSizeHist = + KafkaYammerMetrics.defaultRegistry().newHistogram(batchSizeHistName, true) + + val lastAppliedImageProvenance: AtomicReference[MetadataProvenance] = + new AtomicReference[MetadataProvenance](MetadataProvenance.EMPTY) val metadataLoadErrorCount: AtomicLong = new AtomicLong(0) val metadataApplyErrorCount: AtomicLong = new AtomicLong(0) @@ -62,15 +93,15 @@ final class BrokerServerMetrics private (metrics: Metrics) extends AutoCloseable ) addMetric(metrics, lastAppliedRecordOffsetName) { _ => - lastAppliedRecordOffset.get + lastAppliedImageProvenance.get.offset() } addMetric(metrics, lastAppliedRecordTimestampName) { _ => - lastAppliedRecordTimestamp.get + lastAppliedImageProvenance.get.lastContainedLogTimeMs() } addMetric(metrics, lastAppliedRecordLagMsName) { now => - now - lastAppliedRecordTimestamp.get + now - lastAppliedImageProvenance.get.lastContainedLogTimeMs() } addMetric(metrics, metadataLoadErrorCountName) { _ => @@ -82,6 +113,8 @@ final class BrokerServerMetrics private (metrics: Metrics) extends AutoCloseable } override def close(): Unit = { + KafkaYammerMetrics.defaultRegistry().removeMetric(batchProcessingTimeHistName) + KafkaYammerMetrics.defaultRegistry().removeMetric(batchSizeHistName) List( lastAppliedRecordOffsetName, lastAppliedRecordTimestampName, @@ -90,6 +123,16 @@ final class BrokerServerMetrics private (metrics: Metrics) extends AutoCloseable metadataApplyErrorCountName ).foreach(metrics.removeMetric) } + + override def updateBatchProcessingTime(elapsedNs: Long): Unit = + batchProcessingTimeHist.update(MICROSECONDS.convert(elapsedNs, NANOSECONDS)) + + override def updateBatchSize(size: Int): Unit = batchSizeHist.update(size) + + override def updateLastAppliedImageProvenance(provenance: MetadataProvenance): Unit = + lastAppliedImageProvenance.set(provenance) + + override def lastAppliedOffset(): Long = lastAppliedImageProvenance.get().offset() } diff --git a/core/src/main/scala/kafka/server/metadata/ControllerMetadataPublisher.scala b/core/src/main/scala/kafka/server/metadata/ControllerMetadataPublisher.scala new file mode 100644 index 0000000000000..17134ceb8ebc6 --- /dev/null +++ b/core/src/main/scala/kafka/server/metadata/ControllerMetadataPublisher.scala @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.server.metadata + +import kafka.server.ConfigAdminManager.toLoggableProps +import kafka.server.{ConfigEntityName, ConfigHandler, ConfigType, KafkaConfig} +import kafka.utils.Logging +import org.apache.kafka.common.config.ConfigResource.Type.BROKER +import org.apache.kafka.image.loader.{LogDeltaManifest, SnapshotManifest} +import org.apache.kafka.image.publisher.MetadataPublisher +import org.apache.kafka.image.{MetadataDelta, MetadataImage} +import org.apache.kafka.server.fault.FaultHandler + + +class ControllerMetadataPublisher( + conf: KafkaConfig, + dynamicConfigHandlers: Map[String, ConfigHandler], + fatalFaultHandler: FaultHandler, +) extends MetadataPublisher with Logging { + logIdent = s"[ControllerMetadataPublisher id=${conf.nodeId}] " + var firstPublish = true + + val nodeId: Int = conf.nodeId + + override def name(): String = "ControllerMetadataPublisher" + + /** + * Publish a new cluster metadata snapshot that we loaded. + * + * @param delta The delta between the previous state and the new one. + * @param newImage The complete new state. + * @param manifest The contents of what was published. + */ + override def publishSnapshot( + delta: MetadataDelta, + newImage: MetadataImage, + manifest: SnapshotManifest + ): Unit = { + publish(delta, newImage) + } + + /** + * Publish a change to the cluster metadata. + * + * @param delta The delta between the previous state and the new one. + * @param newImage The complete new state. + * @param manifest The contents of what was published. + */ + override def publishLogDelta( + delta: MetadataDelta, + newImage: MetadataImage, + manifest: LogDeltaManifest + ): Unit = { + publish(delta, newImage) + } + + def publish(delta: MetadataDelta, newImage: MetadataImage): Unit = { + val deltaName = if (firstPublish) { + s"initial MetadataDelta up to ${newImage.highestOffsetAndEpoch().offset}" + } else { + s"MetadataDelta up to ${newImage.highestOffsetAndEpoch().offset}" + } + firstPublish = false + + // Apply configuration deltas. + Option(delta.configsDelta()).foreach { configsDelta => + configsDelta.changes().keySet().forEach { resource => + val props = newImage.configs().configProperties(resource) + resource.`type`() match { + case BROKER => + if (resource.name().isEmpty) { + try { + // Apply changes to "cluster configs" (also known as default BROKER configs). + // These are stored in KRaft with an empty name field. + info("Updating cluster configuration : " + + toLoggableProps(resource, props).mkString(",")) + dynamicConfigHandlers(ConfigType.Broker). + processConfigChanges(ConfigEntityName.Default, props) + } catch { + case t: Throwable => fatalFaultHandler.handleFault("Error updating " + + s"cluster with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + + s"in ${deltaName}", t) + } + } else if (resource.name() == nodeId.toString) { + try { + // Apply changes to this broker's dynamic configuration. + info(s"Updating controller $nodeId with new configuration : " + + toLoggableProps(resource, props).mkString(",")) + dynamicConfigHandlers(ConfigType.Broker). + processConfigChanges(resource.name(), props) + // When applying a per broker config (not a cluster config), we also + // reload any associated file. For example, if the ssl.keystore is still + // set to /tmp/foo, we still want to reload /tmp/foo in case its contents + // have changed. This doesn't apply to topic configs or cluster configs. + conf.dynamicConfig.reloadUpdatedFilesWithoutConfigChange(props) + } catch { + case t: Throwable => fatalFaultHandler.handleFault("Error updating the " + + s"controller with a new configuration: ${toLoggableProps(resource, props).mkString(",")} " + + s"in ${deltaName}", t) + } + } + case _ => // nothing to do + } + } + } + } + + override def close(): Unit = { + // nothing to do + } +} diff --git a/core/src/main/scala/kafka/server/metadata/MetadataPublisher.scala b/core/src/main/scala/kafka/server/metadata/MetadataPublisher.scala deleted file mode 100644 index b63a2c056c05f..0000000000000 --- a/core/src/main/scala/kafka/server/metadata/MetadataPublisher.scala +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package kafka.server.metadata - -import org.apache.kafka.image.{MetadataDelta, MetadataImage} - -/** - * An object which publishes a new metadata image. - */ -trait MetadataPublisher { - /** - * Publish a new metadata image. - * - * @param delta The delta between the old image and the new one. - * @param newImage The new image, which is the result of applying the - * delta to the previous image. - */ - def publish(delta: MetadataDelta, newImage: MetadataImage): Unit - - /** - * The highest offset of metadata topic which has been published - */ - def publishedOffset: Long -} diff --git a/core/src/main/scala/kafka/server/metadata/MetadataSnapshotter.scala b/core/src/main/scala/kafka/server/metadata/MetadataSnapshotter.scala deleted file mode 100644 index b175bf78e4964..0000000000000 --- a/core/src/main/scala/kafka/server/metadata/MetadataSnapshotter.scala +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package kafka.server.metadata - -import org.apache.kafka.image.MetadataImage -import org.apache.kafka.metadata.util.SnapshotReason - - -/** - * Handles creating snapshots. - */ -trait MetadataSnapshotter { - /** - * If there is no other snapshot being written out, start writing out a snapshot. - * - * @param lastContainedLogTime The highest time contained in the snapshot. - * @param image The metadata image to write out. - * @param reason Set of reasons due to which a snapshot is being taken. - * - * @return True if we will write out a new snapshot; false otherwise. - */ - def maybeStartSnapshot(lastContainedLogTime: Long, image: MetadataImage, reason: Set[SnapshotReason]): Boolean -} diff --git a/core/src/test/java/kafka/test/MockController.java b/core/src/test/java/kafka/test/MockController.java index ff1154d2119cd..061e19213e397 100644 --- a/core/src/test/java/kafka/test/MockController.java +++ b/core/src/test/java/kafka/test/MockController.java @@ -436,11 +436,6 @@ synchronized public CompletableFuture> createP return CompletableFuture.completedFuture(results); } - @Override - public CompletableFuture beginWritingSnapshot() { - throw new UnsupportedOperationException(); - } - @Override public void beginShutdown() { this.active = false; diff --git a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java index 417c083457f49..33617c50986b6 100644 --- a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java +++ b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java @@ -24,7 +24,6 @@ import kafka.server.KafkaConfig$; import kafka.server.KafkaRaftServer; import kafka.server.MetaProperties; -import kafka.server.metadata.BrokerServerMetrics$; import kafka.tools.StorageTool; import kafka.utils.Logging; import org.apache.kafka.clients.CommonClientConfigs; @@ -37,6 +36,7 @@ import org.apache.kafka.common.utils.Utils; import org.apache.kafka.controller.Controller; import org.apache.kafka.controller.MockControllerMetrics; +import org.apache.kafka.image.loader.MetadataLoader; import org.apache.kafka.metadata.MetadataRecordSerde; import org.apache.kafka.metadata.bootstrap.BootstrapMetadata; import org.apache.kafka.raft.RaftConfig; @@ -191,21 +191,36 @@ public KafkaClusterTestKit build() throws Exception { KafkaRaftManager raftManager = new KafkaRaftManager<>( metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftServer.MetadataTopicId(), Time.SYSTEM, new Metrics(), Option.apply(threadNamePrefix), connectFutureManager.future); - ControllerServer controller = new ControllerServer( - nodes.controllerProperties(node.id()), - config, - raftManager, - Time.SYSTEM, - new Metrics(), - new MockControllerMetrics(), - Option.apply(threadNamePrefix), - connectFutureManager.future, - KafkaRaftServer.configSchema(), - raftManager.apiVersions(), - bootstrapMetadata, - metadataFaultHandler, - fatalFaultHandler - ); + MetadataLoader metadataLoader = null; + ControllerServer controller = null; + try { + metadataLoader = new MetadataLoader.Builder(). + setNodeId(node.id()). + setTime(Time.SYSTEM). + setThreadNamePrefix(threadNamePrefix). + setFaultHandler(fatalFaultHandler). + build(); + controller = new ControllerServer( + nodes.controllerProperties(node.id()), + config, + raftManager, + Time.SYSTEM, + new Metrics(), + new MockControllerMetrics(), + Option.apply(threadNamePrefix), + connectFutureManager.future, + KafkaRaftServer.configSchema(), + raftManager.apiVersions(), + bootstrapMetadata, + metadataFaultHandler, + fatalFaultHandler, + metadataLoader); + } catch (Throwable e) { + log.error("Error creating controller {}", node.id(), e); + metadataLoader.close(); + controller.shutdown(); + throw e; + } controllers.put(node.id(), controller); controller.socketServerFirstBoundPortFuture().whenComplete((port, e) -> { if (e != null) { @@ -255,20 +270,33 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS raftManagers.put(node.id(), raftManager); } Metrics metrics = new Metrics(); - BrokerServer broker = new BrokerServer( - config, - nodes.brokerProperties(node.id()), - raftManager, - Time.SYSTEM, - metrics, - BrokerServerMetrics$.MODULE$.apply(metrics), - Option.apply(threadNamePrefix), - JavaConverters.asScalaBuffer(Collections.emptyList()).toSeq(), - connectFutureManager.future, - fatalFaultHandler, - metadataFaultHandler, - metadataFaultHandler - ); + MetadataLoader metadataLoader = null; + BrokerServer broker = null; + try { + metadataLoader = new MetadataLoader.Builder(). + setNodeId(node.id()). + setTime(Time.SYSTEM). + setThreadNamePrefix(threadNamePrefix). + setFaultHandler(fatalFaultHandler). + build(); + broker = new BrokerServer( + config, + nodes.brokerProperties(node.id()), + raftManager, + Time.SYSTEM, + metrics, + Option.apply(threadNamePrefix), + JavaConverters.asScalaBuffer(Collections.emptyList()).toSeq(), + connectFutureManager.future, + fatalFaultHandler, + metadataFaultHandler, + metadataLoader); + } catch (Throwable e) { + log.error("Error creating broker {}", node.id(), e); + metadataLoader.close(); + broker.shutdown(); + throw e; + } brokers.put(node.id(), broker); } } catch (Exception e) { diff --git a/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala b/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala index c0a55948cfd2c..0351ebb2077e7 100755 --- a/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala +++ b/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala @@ -24,7 +24,6 @@ import java.util.{Collections, Properties} import java.util.concurrent.CompletableFuture import javax.security.auth.login.Configuration import kafka.raft.KafkaRaftManager -import kafka.server.metadata.BrokerServerMetrics import kafka.tools.StorageTool import kafka.utils.{CoreUtils, Logging, TestInfoUtils, TestUtils} import kafka.zk.{AdminZkClient, EmbeddedZookeeper, KafkaZkClient} @@ -34,6 +33,7 @@ import org.apache.kafka.common.security.JaasUtils import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.utils.{Exit, Time} import org.apache.kafka.controller.QuorumControllerMetrics +import org.apache.kafka.image.loader.MetadataLoader import org.apache.kafka.metadata.MetadataRecordSerde import org.apache.kafka.metadata.bootstrap.BootstrapMetadata import org.apache.kafka.raft.RaftConfig.{AddressSpec, InetAddressSpec} @@ -99,21 +99,35 @@ class KRaftQuorumImplementation( startup: Boolean, threadNamePrefix: Option[String], ): KafkaBroker = { - val metrics = new Metrics() - val broker = new BrokerServer(config = config, - metaProps = new MetaProperties(clusterId, config.nodeId), - raftManager = raftManager, - time = time, - metrics = metrics, - brokerMetrics = BrokerServerMetrics(metrics), - threadNamePrefix = Some("Broker%02d_".format(config.nodeId)), - initialOfflineDirs = Seq(), - controllerQuorumVotersFuture = controllerQuorumVotersFuture, - fatalFaultHandler = faultHandler, - metadataLoadingFaultHandler = faultHandler, - metadataPublishingFaultHandler = faultHandler) - if (startup) broker.startup() - broker + val threadNamePrefix = "Broker%02d_".format(config.nodeId) + var metadataLoader: MetadataLoader = null + var broker: BrokerServer = null + try { + metadataLoader = new MetadataLoader.Builder(). + setNodeId(config.nodeId). + setTime(time). + setThreadNamePrefix(threadNamePrefix). + build() + broker = new BrokerServer(config = config, + metaProps = new MetaProperties(clusterId, config.nodeId), + raftManager = raftManager, + time = time, + metrics = new Metrics(), + threadNamePrefix = Some(threadNamePrefix), + initialOfflineDirs = Seq(), + controllerQuorumVotersFuture = controllerQuorumVotersFuture, + fatalFaultHandler = faultHandler, + metadataPublishingFaultHandler = faultHandler, + metadataLoader = metadataLoader) + if (startup) broker.startup() + broker + } catch { + case e: Throwable => { + if (metadataLoader != null) CoreUtils.swallow(metadataLoader.close(), log) + if (broker != null) CoreUtils.swallow(broker.shutdown(), log) + throw e + } + } } override def shutdown(): Unit = { @@ -308,13 +322,20 @@ abstract class QuorumTestHarness extends Logging { metrics = controllerMetrics, threadNamePrefixOpt = Option(threadNamePrefix), controllerQuorumVotersFuture = controllerQuorumVotersFuture) + val time: Time = Time.SYSTEM + var metadataLoader: MetadataLoader = null var controllerServer: ControllerServer = null try { + metadataLoader = new MetadataLoader.Builder(). + setNodeId(config.nodeId). + setTime(time). + setThreadNamePrefix(threadNamePrefix). + build() controllerServer = new ControllerServer( metaProperties = metaProperties, config = config, raftManager = raftManager, - time = Time.SYSTEM, + time = time, metrics = controllerMetrics, controllerMetrics = new QuorumControllerMetrics(KafkaYammerMetrics.defaultRegistry(), Time.SYSTEM), threadNamePrefix = Option(threadNamePrefix), @@ -323,7 +344,8 @@ abstract class QuorumTestHarness extends Logging { raftApiVersions = raftManager.apiVersions, bootstrapMetadata = BootstrapMetadata.fromVersion(metadataVersion, "test harness"), metadataFaultHandler = faultHandler, - fatalFaultHandler = faultHandler + fatalFaultHandler = faultHandler, + metadataLoader = metadataLoader, ) controllerServer.socketServerFirstBoundPortFuture.whenComplete((port, e) => { if (e != null) { @@ -338,8 +360,9 @@ abstract class QuorumTestHarness extends Logging { raftManager.startup() } catch { case e: Throwable => - CoreUtils.swallow(raftManager.shutdown(), this) if (controllerServer != null) CoreUtils.swallow(controllerServer.shutdown(), this) + if (metadataLoader != null) CoreUtils.swallow(metadataLoader.close(), this) + CoreUtils.swallow(raftManager.shutdown(), this) throw e } new KRaftQuorumImplementation(raftManager, diff --git a/core/src/test/scala/kafka/server/metadata/BrokerServerMetricsTest.scala b/core/src/test/scala/kafka/server/metadata/BrokerServerMetricsTest.scala index ea2b439c166ae..6dd9f05fce022 100644 --- a/core/src/test/scala/kafka/server/metadata/BrokerServerMetricsTest.scala +++ b/core/src/test/scala/kafka/server/metadata/BrokerServerMetricsTest.scala @@ -22,12 +22,38 @@ import kafka.utils.TestUtils import org.apache.kafka.common.MetricName import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.utils.MockTime -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertTrue +import org.apache.kafka.image.MetadataProvenance +import org.apache.kafka.server.metrics.KafkaYammerMetrics +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} import org.junit.jupiter.api.Test + +import scala.collection.mutable import scala.jdk.CollectionConverters._ final class BrokerServerMetricsTest { + @Test + def testYammerMetrics(): Unit = { + val expectedYammerMetrics = Set( + "kafka.server:type=BrokerMetadataListener,name=MetadataBatchProcessingTimeUs" + ) + TestUtils.resource(BrokerServerMetrics(new Metrics())) { brokerMetrics => + val notFound = new mutable.HashSet[String] ++ expectedYammerMetrics + KafkaYammerMetrics.defaultRegistry.allMetrics.keySet().forEach(bean => { + expectedYammerMetrics.foreach(expected => { + if (bean.getMBeanName().equals(expected)) { + notFound -= expected + } + }) + }) + assertEquals(Set.empty, notFound, "Failed to find some Yammer metrics") + } + KafkaYammerMetrics.defaultRegistry.allMetrics.keySet().forEach(bean => { + expectedYammerMetrics.foreach(expected => { + assertFalse(bean.getMBeanName().equals(expected), s"${expected} did not get cleaned up.") + }) + }) + } + @Test def testMetricsExported(): Unit = { val metrics = new Metrics() @@ -59,11 +85,11 @@ final class BrokerServerMetricsTest { val metrics = new Metrics() TestUtils.resource(BrokerServerMetrics(metrics)) { brokerMetrics => val offsetMetric = metrics.metrics().get(brokerMetrics.lastAppliedRecordOffsetName) - assertEquals(0, offsetMetric.metricValue.asInstanceOf[Long]) + assertEquals(-1L, offsetMetric.metricValue.asInstanceOf[Long]) // Update metric value and check val expectedValue = 1000 - brokerMetrics.lastAppliedRecordOffset.set(expectedValue) + brokerMetrics.lastAppliedImageProvenance.set(new MetadataProvenance(expectedValue, 0, 0L)) assertEquals(expectedValue, offsetMetric.metricValue.asInstanceOf[Long]) } } @@ -82,7 +108,7 @@ final class BrokerServerMetricsTest { // Update metric value and check val timestamp = 500 - brokerMetrics.lastAppliedRecordTimestamp.set(timestamp) + brokerMetrics.lastAppliedImageProvenance.set(new MetadataProvenance(0L, 0, timestamp)) assertEquals(timestamp, timestampMetric.metricValue.asInstanceOf[Long]) assertEquals(time.milliseconds - timestamp, lagMetric.metricValue.asInstanceOf[Long]) } diff --git a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala index d92c76f71189d..838a3c0c4e550 100644 --- a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala +++ b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala @@ -21,7 +21,6 @@ import org.apache.kafka.common.{Node, TopicPartition, Uuid} import java.util import java.util.Arrays.asList import java.util.Collections - import kafka.api.LeaderAndIsr import kafka.server.metadata.{KRaftMetadataCache, ZkMetadataCache} import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadataBroker, UpdateMetadataEndpoint, UpdateMetadataPartitionState, UpdateMetadataTopicState} @@ -32,10 +31,8 @@ import org.apache.kafka.common.requests.UpdateMetadataRequest import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.metadata.{BrokerRegistrationChangeRecord, PartitionRecord, RegisterBrokerRecord, RemoveTopicRecord, TopicRecord} import org.apache.kafka.common.metadata.RegisterBrokerRecord.{BrokerEndpoint, BrokerEndpointCollection} -import org.apache.kafka.image.{ClusterImage, MetadataDelta, MetadataImage} +import org.apache.kafka.image.{ClusterImage, MetadataDelta, MetadataImage, MetadataProvenance} import org.apache.kafka.server.common.MetadataVersion -import org.apache.kafka.raft.{OffsetAndEpoch => RaftOffsetAndEpoch} - import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.MethodSource @@ -65,7 +62,7 @@ object MetadataCacheTest { // contains no brokers, but which contains the previous partitions. val image = c.currentImage() val partialImage = new MetadataImage( - new RaftOffsetAndEpoch(100, 10), + new MetadataProvenance(100L, 10, 1000L), image.features(), ClusterImage.EMPTY, image.topics(), @@ -73,7 +70,7 @@ object MetadataCacheTest { image.clientQuotas(), image.producerIds(), image.acls()) - val delta = new MetadataDelta(partialImage) + val delta = new MetadataDelta.Builder().setImage(partialImage).build() def toRecord(broker: UpdateMetadataBroker): RegisterBrokerRecord = { val endpoints = new BrokerEndpointCollection() @@ -100,7 +97,7 @@ object MetadataCacheTest { setFenced(fenced) } request.liveBrokers().iterator().asScala.foreach { brokerInfo => - delta.replay(100, 10, toRecord(brokerInfo)) + delta.replay(toRecord(brokerInfo)) } def toRecords(topic: UpdateMetadataTopicState): Seq[ApiMessage] = { @@ -125,9 +122,9 @@ object MetadataCacheTest { results } request.topicStates().forEach { topic => - toRecords(topic).foreach(delta.replay(100, 10, _)) + toRecords(topic).foreach(delta.replay) } - c.setImage(delta.apply()) + c.setImage(delta.apply(new MetadataProvenance(100L, 10, 1000L))) } case _ => throw new RuntimeException("Unsupported cache type") } @@ -646,12 +643,12 @@ class MetadataCacheTest { def testIsBrokerFenced(): Unit = { val metadataCache = MetadataCache.kRaftMetadataCache(0) - val delta = new MetadataDelta(MetadataImage.EMPTY) + val delta = new MetadataDelta.Builder().build() delta.replay(new RegisterBrokerRecord() .setBrokerId(0) .setFenced(false)) - metadataCache.setImage(delta.apply()) + metadataCache.setImage(delta.apply(MetadataProvenance.EMPTY)) assertFalse(metadataCache.isBrokerFenced(0)) @@ -659,7 +656,7 @@ class MetadataCacheTest { .setBrokerId(0) .setFenced(1.toByte)) - metadataCache.setImage(delta.apply()) + metadataCache.setImage(delta.apply(MetadataProvenance.EMPTY)) assertTrue(metadataCache.isBrokerFenced(0)) } @@ -668,12 +665,12 @@ class MetadataCacheTest { def testIsBrokerInControlledShutdown(): Unit = { val metadataCache = MetadataCache.kRaftMetadataCache(0) - val delta = new MetadataDelta(MetadataImage.EMPTY) + val delta = new MetadataDelta.Builder().build() delta.replay(new RegisterBrokerRecord() .setBrokerId(0) .setInControlledShutdown(false)) - metadataCache.setImage(delta.apply()) + metadataCache.setImage(delta.apply(MetadataProvenance.EMPTY)) assertFalse(metadataCache.isBrokerShuttingDown(0)) @@ -681,7 +678,7 @@ class MetadataCacheTest { .setBrokerId(0) .setInControlledShutdown(1.toByte)) - metadataCache.setImage(delta.apply()) + metadataCache.setImage(delta.apply(MetadataProvenance.EMPTY)) assertTrue(metadataCache.isBrokerShuttingDown(0)) } diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerConcurrencyTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerConcurrencyTest.scala index 651451afad430..559485ff4ddd6 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerConcurrencyTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerConcurrencyTest.scala @@ -349,7 +349,7 @@ class ReplicaManagerConcurrencyTest { override def doWork(): Unit = { channel.poll() match { case InitializeEvent => - val delta = new MetadataDelta(latestImage) + val delta = new MetadataDelta.Builder().setImage(latestImage).build() brokerIds.foreach { brokerId => delta.replay(new RegisterBrokerRecord() .setBrokerId(brokerId) @@ -357,14 +357,14 @@ class ReplicaManagerConcurrencyTest { ) } topic.initialize(delta) - latestImage = delta.apply() + latestImage = delta.apply(latestImage.provenance()) metadataCache.setImage(latestImage) replicaManager.applyDelta(delta.topicsDelta, latestImage) case AlterIsrEvent(future, topicPartition, leaderAndIsr) => - val delta = new MetadataDelta(latestImage) + val delta = new MetadataDelta.Builder().setImage(latestImage).build() val updatedLeaderAndIsr = topic.alterIsr(topicPartition, leaderAndIsr, delta) - latestImage = delta.apply() + latestImage = delta.apply(latestImage.provenance()) future.complete(updatedLeaderAndIsr) replicaManager.applyDelta(delta.topicsDelta, latestImage) diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index 19199ff71b925..f5437df2b51c8 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -53,10 +53,9 @@ import org.apache.kafka.common.requests._ import org.apache.kafka.common.security.auth.KafkaPrincipal import org.apache.kafka.common.utils.{LogContext, Time, Utils} import org.apache.kafka.common.{IsolationLevel, Node, TopicIdPartition, TopicPartition, Uuid} -import org.apache.kafka.image.{AclsImage, ClientQuotasImage, ClusterImageTest, ConfigurationsImage, FeaturesImage, MetadataImage, ProducerIdsImage, TopicsDelta, TopicsImage} +import org.apache.kafka.image.{AclsImage, ClientQuotasImage, ClusterImageTest, ConfigurationsImage, FeaturesImage, MetadataImage, MetadataProvenance, ProducerIdsImage, TopicsDelta, TopicsImage} import org.apache.kafka.metadata.LeaderConstants.NO_LEADER import org.apache.kafka.metadata.LeaderRecoveryState -import org.apache.kafka.raft.{OffsetAndEpoch => RaftOffsetAndEpoch} import org.apache.kafka.server.common.MetadataVersion.IBP_2_6_IV0 import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} @@ -4041,7 +4040,7 @@ class ReplicaManagerTest { private def imageFromTopics(topicsImage: TopicsImage): MetadataImage = { new MetadataImage( - new RaftOffsetAndEpoch(100, 10), + new MetadataProvenance(100L, 10, 1000L), FeaturesImage.EMPTY, ClusterImageTest.IMAGE1, topicsImage, diff --git a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataListenerTest.scala b/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataListenerTest.scala deleted file mode 100644 index a589c0572ddd8..0000000000000 --- a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataListenerTest.scala +++ /dev/null @@ -1,413 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.server.metadata - -import java.util -import java.util.concurrent.atomic.AtomicReference -import java.util.{Collections, Optional} -import org.apache.kafka.common.metadata.{FeatureLevelRecord, PartitionChangeRecord, PartitionRecord, RegisterBrokerRecord, TopicRecord} -import org.apache.kafka.common.metrics.Metrics -import org.apache.kafka.common.utils.Time -import org.apache.kafka.common.{Endpoint, Uuid} -import org.apache.kafka.image.{MetadataDelta, MetadataImage} -import org.apache.kafka.metadata.util.SnapshotReason -import org.apache.kafka.metadata.{BrokerRegistration, RecordTestUtils, VersionRange} -import org.apache.kafka.server.common.{ApiMessageAndVersion, MetadataVersion} -import org.apache.kafka.server.fault.{FaultHandler, MockFaultHandler} -import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue} -import org.junit.jupiter.api.{AfterEach, Test} - -import scala.jdk.CollectionConverters._ - -class BrokerMetadataListenerTest { - private val metadataLoadingFaultHandler = new MockFaultHandler("metadata loading") - - @AfterEach - def verifyNoFaults(): Unit = { - metadataLoadingFaultHandler.maybeRethrowFirstException() - } - - private def newBrokerMetadataListener( - metrics: BrokerServerMetrics = BrokerServerMetrics(new Metrics()), - snapshotter: Option[MetadataSnapshotter] = None, - maxBytesBetweenSnapshots: Long = 1000000L, - faultHandler: FaultHandler = metadataLoadingFaultHandler - ): BrokerMetadataListener = { - new BrokerMetadataListener( - brokerId = 0, - time = Time.SYSTEM, - threadNamePrefix = None, - maxBytesBetweenSnapshots = maxBytesBetweenSnapshots, - snapshotter = snapshotter, - brokerMetrics = metrics, - _metadataLoadingFaultHandler = faultHandler - ) - } - - @Test - def testCreateAndClose(): Unit = { - val listener = newBrokerMetadataListener() - listener.close() - } - - @Test - def testPublish(): Unit = { - val metrics = BrokerServerMetrics(new Metrics()) - val listener = newBrokerMetadataListener(metrics = metrics) - try { - val unfencedTimestamp = 300L - listener.handleCommit( - RecordTestUtils.mockBatchReader( - 100, - unfencedTimestamp, - util.Arrays.asList(new ApiMessageAndVersion(new RegisterBrokerRecord(). - setBrokerId(0). - setBrokerEpoch(100L). - setFenced(false). - setRack(null). - setIncarnationId(Uuid.fromString("GFBwlTcpQUuLYQ2ig05CSg")), 0.toShort)) - ) - ) - val imageRecords = listener.getImageRecords().get() - assertEquals(0, imageRecords.size()) - assertEquals(100L, listener.highestMetadataOffset) - assertEquals(0L, metrics.lastAppliedRecordOffset.get) - assertEquals(0L, metrics.lastAppliedRecordTimestamp.get) - assertEquals(0L, metrics.metadataLoadErrorCount.get) - assertEquals(0L, metrics.metadataApplyErrorCount.get) - - val fencedTimestamp = 500L - val fencedLastOffset = 200L - listener.handleCommit( - RecordTestUtils.mockBatchReader( - fencedLastOffset, - fencedTimestamp, - util.Arrays.asList(new ApiMessageAndVersion(new RegisterBrokerRecord(). - setBrokerId(1). - setBrokerEpoch(200L). - setFenced(true). - setRack(null). - setIncarnationId(Uuid.fromString("QkOQtNKVTYatADcaJ28xDg")), 0.toShort)) - ) - ) - listener.startPublishing(new MetadataPublisher { - override def publish(delta: MetadataDelta, newImage: MetadataImage): Unit = { - assertEquals(200L, newImage.highestOffsetAndEpoch().offset) - assertEquals(new BrokerRegistration(0, 100L, - Uuid.fromString("GFBwlTcpQUuLYQ2ig05CSg"), Collections.emptyList[Endpoint](), - Collections.emptyMap[String, VersionRange](), Optional.empty[String](), false, false), - delta.clusterDelta().broker(0)) - assertEquals(new BrokerRegistration(1, 200L, - Uuid.fromString("QkOQtNKVTYatADcaJ28xDg"), Collections.emptyList[Endpoint](), - Collections.emptyMap[String, VersionRange](), Optional.empty[String](), true, false), - delta.clusterDelta().broker(1)) - } - - override def publishedOffset: Long = -1 - }).get() - - assertEquals(fencedLastOffset, metrics.lastAppliedRecordOffset.get) - assertEquals(fencedTimestamp, metrics.lastAppliedRecordTimestamp.get) - assertEquals(0L, metrics.metadataLoadErrorCount.get) - assertEquals(0L, metrics.metadataApplyErrorCount.get) - } finally { - listener.close() - } - } - - class MockMetadataSnapshotter extends MetadataSnapshotter { - var image = MetadataImage.EMPTY - val failure = new AtomicReference[Throwable](null) - var activeSnapshotOffset = -1L - var prevCommittedOffset = -1L - var prevCommittedEpoch = -1 - var prevLastContainedLogTime = -1L - - override def maybeStartSnapshot(lastContainedLogTime: Long, newImage: MetadataImage, reason: Set[SnapshotReason]): Boolean = { - try { - if (activeSnapshotOffset == -1L) { - assertTrue(prevCommittedOffset <= newImage.highestOffsetAndEpoch().offset) - assertTrue(prevCommittedEpoch <= newImage.highestOffsetAndEpoch().epoch) - assertTrue(prevLastContainedLogTime <= lastContainedLogTime) - prevCommittedOffset = newImage.highestOffsetAndEpoch().offset - prevCommittedEpoch = newImage.highestOffsetAndEpoch().epoch - prevLastContainedLogTime = lastContainedLogTime - image = newImage - activeSnapshotOffset = newImage.highestOffsetAndEpoch().offset - true - } else { - false - } - } catch { - case t: Throwable => failure.compareAndSet(null, t) - } - } - } - - class MockMetadataPublisher extends MetadataPublisher { - var image = MetadataImage.EMPTY - - override def publish(delta: MetadataDelta, newImage: MetadataImage): Unit = { - image = newImage - } - - override def publishedOffset: Long = -1 - } - - private val FOO_ID = Uuid.fromString("jj1G9utnTuCegi_gpnRgYw") - private val BAR_ID = Uuid.fromString("SzN5j0LvSEaRIJHrxfMAlg") - - private def generateManyRecords(listener: BrokerMetadataListener, - endOffset: Long): Unit = { - (0 to 10000).foreach { _ => - listener.handleCommit( - RecordTestUtils.mockBatchReader( - endOffset, - 0, - util.Arrays.asList( - new ApiMessageAndVersion(new PartitionChangeRecord(). - setPartitionId(0). - setTopicId(FOO_ID). - setRemovingReplicas(Collections.singletonList(1)), 0.toShort), - new ApiMessageAndVersion(new PartitionChangeRecord(). - setPartitionId(0). - setTopicId(FOO_ID). - setRemovingReplicas(Collections.emptyList()), 0.toShort) - ) - ) - ) - } - listener.getImageRecords().get() - } - - private def generateBadRecords(listener: BrokerMetadataListener, - endOffset: Long): Unit = { - listener.handleCommit( - RecordTestUtils.mockBatchReader( - endOffset, - 0, - util.Arrays.asList( - new ApiMessageAndVersion(new PartitionChangeRecord(). - setPartitionId(0). - setTopicId(BAR_ID). - setRemovingReplicas(Collections.singletonList(1)), 0.toShort), - new ApiMessageAndVersion(new PartitionChangeRecord(). - setPartitionId(0). - setTopicId(BAR_ID). - setRemovingReplicas(Collections.emptyList()), 0.toShort) - ) - ) - ) - listener.getImageRecords().get() - } - - @Test - def testHandleCommitsWithNoSnapshotterDefined(): Unit = { - val listener = newBrokerMetadataListener(maxBytesBetweenSnapshots = 1000L) - try { - val brokerIds = 0 to 3 - - registerBrokers(listener, brokerIds, endOffset = 100L) - createTopicWithOnePartition(listener, replicas = brokerIds, endOffset = 200L) - listener.getImageRecords().get() - assertEquals(200L, listener.highestMetadataOffset) - - generateManyRecords(listener, endOffset = 1000L) - assertEquals(1000L, listener.highestMetadataOffset) - } finally { - listener.close() - } - } - - @Test - def testCreateSnapshot(): Unit = { - val snapshotter = new MockMetadataSnapshotter() - val listener = newBrokerMetadataListener(snapshotter = Some(snapshotter), - maxBytesBetweenSnapshots = 1000L) - try { - val brokerIds = 0 to 3 - - registerBrokers(listener, brokerIds, endOffset = 100L) - createTopicWithOnePartition(listener, replicas = brokerIds, endOffset = 200L) - listener.getImageRecords().get() - assertEquals(200L, listener.highestMetadataOffset) - - // Check that we generate at least one snapshot once we see enough records. - assertEquals(-1L, snapshotter.prevCommittedOffset) - generateManyRecords(listener, 1000L) - assertEquals(1000L, snapshotter.prevCommittedOffset) - assertEquals(1000L, snapshotter.activeSnapshotOffset) - snapshotter.activeSnapshotOffset = -1L - - // Test creating a new snapshot after publishing it. - val publisher = new MockMetadataPublisher() - listener.startPublishing(publisher).get() - generateManyRecords(listener, 2000L) - listener.getImageRecords().get() - assertEquals(2000L, snapshotter.activeSnapshotOffset) - assertEquals(2000L, snapshotter.prevCommittedOffset) - - // Test how we handle the snapshotter returning false. - generateManyRecords(listener, 3000L) - assertEquals(2000L, snapshotter.activeSnapshotOffset) - generateManyRecords(listener, 4000L) - assertEquals(2000L, snapshotter.activeSnapshotOffset) - snapshotter.activeSnapshotOffset = -1L - generateManyRecords(listener, 5000L) - assertEquals(5000L, snapshotter.activeSnapshotOffset) - assertEquals(null, snapshotter.failure.get()) - } finally { - listener.close() - } - } - - @Test - def testNotSnapshotAfterMetadataVersionChangeBeforePublishing(): Unit = { - val snapshotter = new MockMetadataSnapshotter() - val listener = newBrokerMetadataListener(snapshotter = Some(snapshotter), - maxBytesBetweenSnapshots = 1000L) - - updateFeature(listener, feature = MetadataVersion.FEATURE_NAME, MetadataVersion.latest.featureLevel(), 100L) - listener.getImageRecords().get() - assertEquals(-1L, snapshotter.activeSnapshotOffset, "We won't generate snapshot on metadata version change before starting publishing") - } - - @Test - def testSnapshotAfterMetadataVersionChangeWhenStarting(): Unit = { - val snapshotter = new MockMetadataSnapshotter() - val listener = newBrokerMetadataListener(snapshotter = Some(snapshotter), - maxBytesBetweenSnapshots = 1000L) - - val endOffset = 100L - updateFeature(listener, feature = MetadataVersion.FEATURE_NAME, MetadataVersion.latest.featureLevel(), endOffset) - listener.startPublishing(new MockMetadataPublisher()).get() - assertEquals(endOffset, snapshotter.activeSnapshotOffset, "We should try to generate snapshot when starting publishing") - } - - @Test - def testSnapshotAfterMetadataVersionChange(): Unit = { - val snapshotter = new MockMetadataSnapshotter() - val listener = newBrokerMetadataListener(snapshotter = Some(snapshotter), - maxBytesBetweenSnapshots = 1000L) - listener.startPublishing(new MockMetadataPublisher()).get() - - val endOffset = 100L - updateFeature(listener, feature = MetadataVersion.FEATURE_NAME, (MetadataVersion.latest().featureLevel() - 1).toShort, endOffset) - // Waiting for the metadata version update to get processed - listener.getImageRecords().get() - assertEquals(endOffset, snapshotter.activeSnapshotOffset, "We should generate snapshot on feature update") - } - - @Test - def testNoSnapshotAfterError(): Unit = { - val snapshotter = new MockMetadataSnapshotter() - val faultHandler = new MockFaultHandler("metadata loading") - - val listener = newBrokerMetadataListener( - snapshotter = Some(snapshotter), - maxBytesBetweenSnapshots = 1000L, - faultHandler = faultHandler) - try { - val brokerIds = 0 to 3 - - registerBrokers(listener, brokerIds, endOffset = 100L) - createTopicWithOnePartition(listener, replicas = brokerIds, endOffset = 200L) - listener.getImageRecords().get() - assertEquals(200L, listener.highestMetadataOffset) - assertEquals(-1L, snapshotter.prevCommittedOffset) - assertEquals(-1L, snapshotter.activeSnapshotOffset) - - // Append invalid records that will normally trigger a snapshot - generateBadRecords(listener, 1000L) - assertEquals(-1L, snapshotter.prevCommittedOffset) - assertEquals(-1L, snapshotter.activeSnapshotOffset) - - // Generate some records that will not throw an error, verify still no snapshots - generateManyRecords(listener, 2000L) - assertEquals(-1L, snapshotter.prevCommittedOffset) - assertEquals(-1L, snapshotter.activeSnapshotOffset) - } finally { - listener.close() - } - } - - private def registerBrokers( - listener: BrokerMetadataListener, - brokerIds: Iterable[Int], - endOffset: Long - ): Unit = { - brokerIds.foreach { brokerId => - listener.handleCommit( - RecordTestUtils.mockBatchReader( - endOffset, - 0, - util.Arrays.asList(new ApiMessageAndVersion(new RegisterBrokerRecord(). - setBrokerId(brokerId). - setBrokerEpoch(100L). - setFenced(false). - setRack(null). - setIncarnationId(Uuid.fromString("GFBwlTcpQUuLYQ2ig05CS" + brokerId)), 0.toShort)) - ) - ) - } - } - - private def createTopicWithOnePartition( - listener: BrokerMetadataListener, - replicas: Seq[Int], - endOffset: Long - ): Unit = { - listener.handleCommit( - RecordTestUtils.mockBatchReader( - endOffset, - 0, - util.Arrays.asList( - new ApiMessageAndVersion(new TopicRecord(). - setName("foo"). - setTopicId(FOO_ID), 0.toShort), - new ApiMessageAndVersion(new PartitionRecord(). - setPartitionId(0). - setTopicId(FOO_ID). - setIsr(replicas.map(Int.box).asJava). - setLeader(0). - setReplicas(replicas.map(Int.box).asJava), 0.toShort) - ) - ) - ) - } - - private def updateFeature( - listener: BrokerMetadataListener, - feature: String, - version: Short, - endOffset: Long - ): Unit = { - listener.handleCommit( - RecordTestUtils.mockBatchReader( - endOffset, - 0, - util.Arrays.asList( - new ApiMessageAndVersion(new FeatureLevelRecord(). - setName(feature). - setFeatureLevel(version), 0.toShort) - ) - ) - ) - } - -} diff --git a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala b/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala index 472d7ef550b44..7bb8b86e06fda 100644 --- a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala +++ b/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala @@ -18,7 +18,7 @@ package kafka.server.metadata import java.util.Collections.{singleton, singletonList, singletonMap} -import java.util.Properties +import java.util.{Collections, Properties} import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} import kafka.log.UnifiedLog import kafka.server.{BrokerServer, KafkaConfig} @@ -213,7 +213,8 @@ class BrokerMetadataPublisherTest { thenAnswer(new Answer[Unit]() { override def answer(invocation: InvocationOnMock): Unit = numTimesReloadCalled.addAndGet(1) }) - broker.metadataListener.alterPublisher(publisher).get() + broker.metadataLoader.removeAndClosePublisher(broker.metadataPublisher).get() + broker.metadataLoader.installPublishers(Collections.singletonList(publisher)).get() val admin = Admin.create(cluster.clientProperties()) try { assertEquals(0, numTimesReloadCalled.get()) @@ -256,7 +257,8 @@ class BrokerMetadataPublisherTest { } val publisher = Mockito.spy(broker.metadataPublisher) doThrow(new RuntimeException("injected failure")).when(publisher).updateCoordinator(any(), any(), any(), any(), any()) - broker.metadataListener.alterPublisher(publisher).get() + broker.metadataLoader.removeAndClosePublisher(broker.metadataPublisher).get() + broker.metadataLoader.installPublishers(Collections.singletonList(publisher)).get() val admin = Admin.create(cluster.clientProperties()) try { admin.createTopics(singletonList(new NewTopic("foo", 1, 1.toShort))).all().get() diff --git a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataSnapshotterTest.scala b/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataSnapshotterTest.scala deleted file mode 100644 index e93846c0cef35..0000000000000 --- a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataSnapshotterTest.scala +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.server.metadata - -import java.nio.ByteBuffer -import java.util.Optional -import java.util.concurrent.{CompletableFuture, CountDownLatch} -import org.apache.kafka.common.memory.MemoryPool -import org.apache.kafka.common.protocol.ByteBufferAccessor -import org.apache.kafka.common.record.{CompressionType, MemoryRecords} -import org.apache.kafka.common.utils.Time -import org.apache.kafka.image.{MetadataDelta, MetadataImage, MetadataImageTest} -import org.apache.kafka.metadata.MetadataRecordSerde -import org.apache.kafka.metadata.util.SnapshotReason -import org.apache.kafka.queue.EventQueue -import org.apache.kafka.raft.OffsetAndEpoch -import org.apache.kafka.server.common.ApiMessageAndVersion -import org.apache.kafka.snapshot.{MockRawSnapshotWriter, RecordsSnapshotWriter, SnapshotWriter} -import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} -import org.junit.jupiter.api.Test - -import java.util -import scala.compat.java8.OptionConverters._ - -class BrokerMetadataSnapshotterTest { - @Test - def testCreateAndClose(): Unit = { - val snapshotter = new BrokerMetadataSnapshotter(0, Time.SYSTEM, None, - (_, _, _) => throw new RuntimeException("unimplemented")) - snapshotter.close() - } - - class MockSnapshotWriterBuilder extends SnapshotWriterBuilder { - var image = new CompletableFuture[MetadataImage] - - override def build(committedOffset: Long, - committedEpoch: Int, - lastContainedLogTime: Long): Option[SnapshotWriter[ApiMessageAndVersion]] = { - val offsetAndEpoch = new OffsetAndEpoch(committedOffset, committedEpoch) - RecordsSnapshotWriter.createWithHeader( - () => { - Optional.of( - new MockRawSnapshotWriter(offsetAndEpoch, consumeSnapshotBuffer(committedOffset, committedEpoch)) - ) - }, - 1024, - MemoryPool.NONE, - Time.SYSTEM, - lastContainedLogTime, - CompressionType.NONE, - MetadataRecordSerde.INSTANCE - ).asScala - } - - def consumeSnapshotBuffer(committedOffset: Long, committedEpoch: Int)(buffer: ByteBuffer): Unit = { - val delta = new MetadataDelta(MetadataImage.EMPTY) - val memoryRecords = MemoryRecords.readableRecords(buffer) - val batchIterator = memoryRecords.batchIterator() - while (batchIterator.hasNext) { - val batch = batchIterator.next() - if (!batch.isControlBatch()) { - batch.forEach(record => { - val recordBuffer = record.value().duplicate() - val messageAndVersion = MetadataRecordSerde.INSTANCE.read( - new ByteBufferAccessor(recordBuffer), recordBuffer.remaining()) - delta.replay(committedOffset, committedEpoch, messageAndVersion.message()) - }) - } - } - image.complete(delta.apply()) - } - } - - class BlockingEvent extends EventQueue.Event { - val latch = new CountDownLatch(1) - override def run(): Unit = latch.await() - } - - @Test - def testCreateSnapshot(): Unit = { - val writerBuilder = new MockSnapshotWriterBuilder() - val snapshotter = new BrokerMetadataSnapshotter(0, Time.SYSTEM, None, writerBuilder) - - try { - val blockingEvent = new BlockingEvent() - val reasons = Set(SnapshotReason.UnknownReason) - - snapshotter.eventQueue.append(blockingEvent) - assertTrue(snapshotter.maybeStartSnapshot(10000L, MetadataImageTest.IMAGE1, reasons)) - assertFalse(snapshotter.maybeStartSnapshot(11000L, MetadataImageTest.IMAGE2, reasons)) - blockingEvent.latch.countDown() - assertEquals(MetadataImageTest.IMAGE1, writerBuilder.image.get()) - } finally { - snapshotter.close() - } - } - - @Test - def testCreateSnapshotMultipleReasons(): Unit = { - val writerBuilder = new MockSnapshotWriterBuilder() - val snapshotter = new BrokerMetadataSnapshotter(0, Time.SYSTEM, None, writerBuilder) - - try { - val blockingEvent = new BlockingEvent() - val reasons = Set(SnapshotReason.MaxBytesExceeded, SnapshotReason.MetadataVersionChanged) - - snapshotter.eventQueue.append(blockingEvent) - assertTrue(snapshotter.maybeStartSnapshot(10000L, MetadataImageTest.IMAGE1, reasons)) - assertFalse(snapshotter.maybeStartSnapshot(11000L, MetadataImageTest.IMAGE2, reasons)) - blockingEvent.latch.countDown() - assertEquals(MetadataImageTest.IMAGE1, writerBuilder.image.get()) - } finally { - snapshotter.close() - } - } - - class MockSnapshotWriter extends SnapshotWriter[ApiMessageAndVersion] { - val batches = new util.ArrayList[util.List[ApiMessageAndVersion]] - override def snapshotId(): OffsetAndEpoch = new OffsetAndEpoch(0, 0) - override def lastContainedLogOffset(): Long = 0 - override def lastContainedLogEpoch(): Int = 0 - override def isFrozen: Boolean = false - override def append(batch: util.List[ApiMessageAndVersion]): Unit = batches.add(batch) - override def freeze(): Unit = {} - override def close(): Unit = {} - } -} diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala index efdee12147b00..a606a84f59363 100755 --- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala +++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala @@ -1224,7 +1224,7 @@ object TestUtils extends Logging { TestUtils.waitUntilTrue( () => { brokers.forall { broker => - val metadataOffset = broker.asInstanceOf[BrokerServer].metadataPublisher.publishedOffset + val metadataOffset = broker.asInstanceOf[BrokerServer].metadataLoader.lastAppliedOffset() metadataOffset >= controllerOffset } }, msg) diff --git a/metadata/src/main/java/org/apache/kafka/controller/AclControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/AclControlManager.java index d3fc0fe76edd8..af94568552df9 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/AclControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/AclControlManager.java @@ -28,7 +28,6 @@ import org.apache.kafka.common.requests.ApiError; import org.apache.kafka.metadata.authorizer.ClusterMetadataAuthorizer; import org.apache.kafka.metadata.authorizer.StandardAcl; -import org.apache.kafka.metadata.authorizer.StandardAclRecordIterator; import org.apache.kafka.metadata.authorizer.StandardAclWithId; import org.apache.kafka.raft.OffsetAndEpoch; import org.apache.kafka.server.authorizer.AclCreateResult; @@ -42,7 +41,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -213,23 +211,8 @@ public void replay(RemoveAccessControlEntryRecord record, } } + // VisibleForTesting Map idToAcl() { return Collections.unmodifiableMap(idToAcl); } - - Iterator> iterator(long epoch) { - Iterator> iterator = idToAcl.entrySet(epoch).iterator(); - return new StandardAclRecordIterator(new Iterator() { - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public StandardAclWithId next() { - Entry entry = iterator.next(); - return new StandardAclWithId(entry.getKey(), entry.getValue()); - } - }); - } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java index 6e8198bd01d33..5a7e8552a52c6 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java @@ -36,11 +36,9 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.NoSuchElementException; import java.util.Objects; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -290,45 +288,4 @@ private ApiError validateEntity(ClientQuotaEntity entity, Map va return ApiError.NONE; } - - class ClientQuotaControlIterator implements Iterator> { - private final long epoch; - private final Iterator>> iterator; - - ClientQuotaControlIterator(long epoch) { - this.epoch = epoch; - this.iterator = clientQuotaData.entrySet(epoch).iterator(); - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public List next() { - if (!hasNext()) throw new NoSuchElementException(); - Entry> entry = iterator.next(); - ClientQuotaEntity entity = entry.getKey(); - List records = new ArrayList<>(); - for (Entry quotaEntry : entry.getValue().entrySet(epoch)) { - ClientQuotaRecord record = new ClientQuotaRecord(); - for (Entry entityEntry : entity.entries().entrySet()) { - record.entity().add(new EntityData(). - setEntityType(entityEntry.getKey()). - setEntityName(entityEntry.getValue())); - } - record.setKey(quotaEntry.getKey()); - record.setValue(quotaEntry.getValue()); - record.setRemove(false); - records.add(new ApiMessageAndVersion(record, - CLIENT_QUOTA_RECORD.highestSupportedVersion())); - } - return records; - } - } - - ClientQuotaControlIterator iterator(long epoch) { - return new ClientQuotaControlIterator(epoch); - } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java index 5bc5bd2c3af28..df1851de12eb1 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java @@ -28,9 +28,7 @@ import org.apache.kafka.common.metadata.FenceBrokerRecord; import org.apache.kafka.common.metadata.RegisterBrokerRecord; import org.apache.kafka.common.metadata.RegisterBrokerRecord.BrokerEndpoint; -import org.apache.kafka.common.metadata.RegisterBrokerRecord.BrokerEndpointCollection; import org.apache.kafka.common.metadata.RegisterBrokerRecord.BrokerFeature; -import org.apache.kafka.common.metadata.RegisterBrokerRecord.BrokerFeatureCollection; import org.apache.kafka.common.metadata.UnfenceBrokerRecord; import org.apache.kafka.common.metadata.UnregisterBrokerRecord; import org.apache.kafka.common.protocol.ApiMessage; @@ -58,7 +56,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.NoSuchElementException; import java.util.Optional; import java.util.OptionalLong; import java.util.Random; @@ -67,7 +64,6 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static java.util.Collections.singletonList; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -632,57 +628,4 @@ public void addReadyBrokersFuture(CompletableFuture future, int minBrokers readyBrokersFuture = Optional.empty(); } } - - class ClusterControlIterator implements Iterator> { - private final Iterator> iterator; - private final MetadataVersion metadataVersion; - - ClusterControlIterator(long epoch) { - this.iterator = brokerRegistrations.entrySet(epoch).iterator(); - this.metadataVersion = featureControl.metadataVersion(); - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public List next() { - if (!hasNext()) throw new NoSuchElementException(); - Entry entry = iterator.next(); - int brokerId = entry.getKey(); - BrokerRegistration registration = entry.getValue(); - BrokerEndpointCollection endpoints = new BrokerEndpointCollection(); - for (Entry endpointEntry : registration.listeners().entrySet()) { - endpoints.add(new BrokerEndpoint().setName(endpointEntry.getKey()). - setHost(endpointEntry.getValue().host()). - setPort(endpointEntry.getValue().port()). - setSecurityProtocol(endpointEntry.getValue().securityProtocol().id)); - } - BrokerFeatureCollection features = new BrokerFeatureCollection(); - for (Entry featureEntry : registration.supportedFeatures().entrySet()) { - features.add(new BrokerFeature().setName(featureEntry.getKey()). - setMaxSupportedVersion(featureEntry.getValue().max()). - setMinSupportedVersion(featureEntry.getValue().min())); - } - RegisterBrokerRecord record = new RegisterBrokerRecord(). - setBrokerId(brokerId). - setIncarnationId(registration.incarnationId()). - setBrokerEpoch(registration.epoch()). - setEndPoints(endpoints). - setFeatures(features). - setRack(registration.rack().orElse(null)). - setFenced(registration.fenced()); - if (metadataVersion.isInControlledShutdownStateSupported()) { - record.setInControlledShutdown(registration.inControlledShutdown()); - } - return singletonList(new ApiMessageAndVersion(record, - metadataVersion.registerBrokerRecordVersion())); - } - } - - ClusterControlIterator iterator(long epoch) { - return new ClusterControlIterator(epoch); - } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java index 5c9a73ca98b09..a52d0c8f13e63 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java @@ -43,7 +43,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; @@ -475,39 +474,4 @@ Map currentControllerConfig() { Map result = configData.get(currentController); return (result == null) ? Collections.emptyMap() : result; } - - class ConfigurationControlIterator implements Iterator> { - private final long epoch; - private final Iterator>> iterator; - - ConfigurationControlIterator(long epoch) { - this.epoch = epoch; - this.iterator = configData.entrySet(epoch).iterator(); - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public List next() { - if (!hasNext()) throw new NoSuchElementException(); - List records = new ArrayList<>(); - Entry> entry = iterator.next(); - ConfigResource resource = entry.getKey(); - for (Entry configEntry : entry.getValue().entrySet(epoch)) { - records.add(new ApiMessageAndVersion(new ConfigRecord(). - setResourceName(resource.name()). - setResourceType(resource.type().id()). - setName(configEntry.getKey()). - setValue(configEntry.getValue()), CONFIG_RECORD.highestSupportedVersion())); - } - return records; - } - } - - ConfigurationControlIterator iterator(long epoch) { - return new ConfigurationControlIterator(epoch); - } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/Controller.java b/metadata/src/main/java/org/apache/kafka/controller/Controller.java index 3622fe225dc8f..ed6c523753304 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/Controller.java +++ b/metadata/src/main/java/org/apache/kafka/controller/Controller.java @@ -316,14 +316,6 @@ CompletableFuture updateFeatures( UpdateFeaturesRequestData request ); - /** - * Begin writing a controller snapshot. If there was already an ongoing snapshot, it - * simply returns information about that snapshot rather than starting a new one. - * - * @return A future yielding the epoch of the snapshot. - */ - CompletableFuture beginWritingSnapshot(); - /** * Create partitions on certain topics. * diff --git a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java index e596b705f4555..c777d9828cdcc 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java @@ -20,11 +20,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map.Entry; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Optional; import java.util.TreeMap; import java.util.function.Consumer; @@ -307,46 +305,4 @@ public void replay(FeatureLevelRecord record) { } } } - - class FeatureControlIterator implements Iterator> { - private final Iterator> iterator; - private final MetadataVersion metadataVersion; - private boolean wroteVersion = false; - - FeatureControlIterator(long epoch) { - this.iterator = finalizedVersions.entrySet(epoch).iterator(); - this.metadataVersion = FeatureControlManager.this.metadataVersion.get(epoch); - } - - @Override - public boolean hasNext() { - return needsWriteMetadataVersion() || iterator.hasNext(); - } - - private boolean needsWriteMetadataVersion() { - return !wroteVersion && metadataVersion.isAtLeast(minimumBootstrapVersion); - } - - @Override - public List next() { - // Write the metadata.version first - if (needsWriteMetadataVersion()) { - wroteVersion = true; - return Collections.singletonList(new ApiMessageAndVersion(new FeatureLevelRecord() - .setName(MetadataVersion.FEATURE_NAME) - .setFeatureLevel(metadataVersion.featureLevel()), FEATURE_LEVEL_RECORD.lowestSupportedVersion())); - } - - // Then write the rest of the features - if (!iterator.hasNext()) throw new NoSuchElementException(); - Entry entry = iterator.next(); - return Collections.singletonList(new ApiMessageAndVersion(new FeatureLevelRecord() - .setName(entry.getKey()) - .setFeatureLevel(entry.getValue()), FEATURE_LEVEL_RECORD.highestSupportedVersion())); - } - } - - FeatureControlIterator iterator(long epoch) { - return new FeatureControlIterator(epoch); - } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ProducerIdControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ProducerIdControlManager.java index 178ef46bdb55e..47e4e1b430fdb 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ProducerIdControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ProducerIdControlManager.java @@ -25,14 +25,10 @@ import org.apache.kafka.timeline.TimelineLong; import org.apache.kafka.timeline.TimelineObject; -import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; -import java.util.List; public class ProducerIdControlManager { - private final ClusterControlManager clusterControlManager; private final TimelineObject nextProducerBlock; private final TimelineLong brokerEpoch; @@ -62,6 +58,11 @@ ControllerResult generateNextProducerId(int brokerId, long bro return ControllerResult.of(Collections.singletonList(new ApiMessageAndVersion(record, (short) 0)), block); } + // VisibleForTesting + ProducerIdsBlock nextProducerBlock() { + return nextProducerBlock.get(); + } + void replay(ProducerIdsRecord record) { long currentNextProducerId = nextProducerBlock.get().firstProducerId(); if (record.nextProducerId() <= currentNextProducerId) { @@ -72,19 +73,4 @@ void replay(ProducerIdsRecord record) { brokerEpoch.set(record.brokerEpoch()); } } - - Iterator> iterator(long epoch) { - List records = new ArrayList<>(1); - - ProducerIdsBlock producerIdBlock = nextProducerBlock.get(epoch); - if (producerIdBlock.firstProducerId() > 0) { - records.add(new ApiMessageAndVersion( - new ProducerIdsRecord() - .setNextProducerId(producerIdBlock.firstProducerId()) - .setBrokerId(producerIdBlock.assignedBrokerId()) - .setBrokerEpoch(brokerEpoch.get(epoch)), - (short) 0)); - } - return Collections.singleton(records).iterator(); - } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index 9aaea73082cd6..c0141ac71b577 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -72,7 +72,6 @@ import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; -import org.apache.kafka.controller.SnapshotGenerator.Section; import org.apache.kafka.metadata.BrokerHeartbeatReply; import org.apache.kafka.metadata.BrokerRegistrationReply; import org.apache.kafka.metadata.FinalizedControllerFeatures; @@ -89,7 +88,6 @@ import org.apache.kafka.raft.LeaderAndEpoch; import org.apache.kafka.raft.OffsetAndEpoch; import org.apache.kafka.raft.RaftClient; -import org.apache.kafka.metadata.util.SnapshotReason; import org.apache.kafka.server.authorizer.AclCreateResult; import org.apache.kafka.server.authorizer.AclDeleteResult; import org.apache.kafka.server.common.ApiMessageAndVersion; @@ -98,7 +96,6 @@ import org.apache.kafka.server.policy.AlterConfigPolicy; import org.apache.kafka.server.policy.CreateTopicPolicy; import org.apache.kafka.snapshot.SnapshotReader; -import org.apache.kafka.snapshot.SnapshotWriter; import org.apache.kafka.timeline.SnapshotRegistry; import org.slf4j.Logger; @@ -167,7 +164,6 @@ static public class Builder { private short defaultReplicationFactor = 3; private int defaultNumPartitions = 1; private ReplicaPlacer replicaPlacer = new StripedReplicaPlacer(new Random()); - private long snapshotMaxNewRecordBytes = Long.MAX_VALUE; private OptionalLong leaderImbalanceCheckIntervalNs = OptionalLong.empty(); private OptionalLong maxIdleIntervalNs = OptionalLong.empty(); private long sessionTimeoutNs = ClusterControlManager.DEFAULT_SESSION_TIMEOUT_NS; @@ -239,11 +235,6 @@ public Builder setReplicaPlacer(ReplicaPlacer replicaPlacer) { return this; } - public Builder setSnapshotMaxNewRecordBytes(long value) { - this.snapshotMaxNewRecordBytes = value; - return this; - } - public Builder setLeaderImbalanceCheckIntervalNs(OptionalLong value) { this.leaderImbalanceCheckIntervalNs = value; return this; @@ -338,7 +329,6 @@ public QuorumController build() throws Exception { defaultReplicationFactor, defaultNumPartitions, replicaPlacer, - snapshotMaxNewRecordBytes, leaderImbalanceCheckIntervalNs, maxIdleIntervalNs, sessionTimeoutNs, @@ -505,110 +495,6 @@ private void appendControlEvent(String name, Runnable handler) { queue.append(event); } - private static final String GENERATE_SNAPSHOT = "generateSnapshot"; - - private static final int MAX_BATCHES_PER_GENERATE_CALL = 10; - - class SnapshotGeneratorManager implements Runnable { - private SnapshotGenerator generator = null; - - void createSnapshotGenerator(long committedOffset, int committedEpoch, long committedTimestamp) { - if (generator != null) { - throw new RuntimeException("Snapshot generator already exists."); - } - if (!snapshotRegistry.hasSnapshot(committedOffset)) { - throw new RuntimeException( - String.format( - "Cannot generate a snapshot at committed offset %d because it does not exists in the snapshot registry.", - committedOffset - ) - ); - } - Optional> writer = raftClient.createSnapshot( - committedOffset, - committedEpoch, - committedTimestamp - ); - if (writer.isPresent()) { - generator = new SnapshotGenerator( - logContext, - writer.get(), - MAX_BATCHES_PER_GENERATE_CALL, - Arrays.asList( - new Section("features", featureControl.iterator(committedOffset)), - new Section("cluster", clusterControl.iterator(committedOffset)), - new Section("replication", replicationControl.iterator(committedOffset)), - new Section("configuration", configurationControl.iterator(committedOffset)), - new Section("clientQuotas", clientQuotaControlManager.iterator(committedOffset)), - new Section("producerIds", producerIdControlManager.iterator(committedOffset)), - new Section("acls", aclControlManager.iterator(committedOffset)) - ) - ); - reschedule(0); - } else { - log.info( - "Skipping generation of snapshot for committed offset {} and epoch {} since it already exists", - committedOffset, - committedEpoch - ); - } - } - - void cancel() { - if (generator == null) return; - log.error("Cancelling snapshot {}", generator.lastContainedLogOffset()); - generator.writer().close(); - generator = null; - - // Delete every in-memory snapshot up to the committed offset. They are not needed since this - // snapshot generation was canceled. - snapshotRegistry.deleteSnapshotsUpTo(lastCommittedOffset); - - queue.cancelDeferred(GENERATE_SNAPSHOT); - } - - void reschedule(long delayNs) { - ControlEvent event = new ControlEvent(GENERATE_SNAPSHOT, this); - queue.scheduleDeferred(event.name, - new EarliestDeadlineFunction(time.nanoseconds() + delayNs), event); - } - - @Override - public void run() { - if (generator == null) { - log.debug("No snapshot is in progress."); - return; - } - OptionalLong nextDelay; - try { - nextDelay = generator.generateBatches(); - } catch (Exception e) { - log.error("Error while generating snapshot {}", generator.lastContainedLogOffset(), e); - generator.writer().close(); - generator = null; - return; - } - if (!nextDelay.isPresent()) { - log.info("Finished generating snapshot {}.", generator.lastContainedLogOffset()); - generator.writer().close(); - generator = null; - - // Delete every in-memory snapshot up to the committed offset. They are not needed since this - // snapshot generation finished. - snapshotRegistry.deleteSnapshotsUpTo(lastCommittedOffset); - return; - } - reschedule(nextDelay.getAsLong()); - } - - OptionalLong snapshotLastOffsetFromLog() { - if (generator == null) { - return OptionalLong.empty(); - } - return OptionalLong.of(generator.lastContainedLogOffset()); - } - } - /** * A controller event that reads the committed internal state in order to expose it * to an API. @@ -931,7 +817,6 @@ public void handleCommit(BatchReader reader) { appendRaftEvent("handleCommit[baseOffset=" + reader.baseOffset() + "]", () -> { try { maybeCompleteAuthorizerInitialLoad(); - long processedRecordsSize = 0; boolean isActive = isActiveController(); while (reader.hasNext()) { Batch batch = reader.next(); @@ -948,10 +833,7 @@ public void handleCommit(BatchReader reader) { purgatory.completeUpTo(offset); // Delete all the in-memory snapshots that we no longer need. - // If we are writing a new snapshot, then we need to keep that around; - // otherwise, we should delete up to the current committed offset. - snapshotRegistry.deleteSnapshotsUpTo( - snapshotGeneratorManager.snapshotLastOffsetFromLog().orElse(offset)); + snapshotRegistry.deleteSnapshotsUpTo(offset); } else { // If the controller is a standby, replay the records that were // created by the active controller. @@ -980,15 +862,9 @@ public void handleCommit(BatchReader reader) { i++; } } - updateLastCommittedState( - offset, - epoch, - batch.appendTimestamp(), - committedBytesSinceLastSnapshot + batch.sizeInBytes() - ); + updateLastCommittedState(offset, epoch, batch.appendTimestamp()); } - maybeGenerateSnapshot(); } finally { reader.close(); } @@ -1040,12 +916,9 @@ public void handleSnapshot(SnapshotReader reader) { i++; } } - updateLastCommittedState( - reader.lastContainedLogOffset(), + updateLastCommittedState(reader.lastContainedLogOffset(), reader.lastContainedLogEpoch(), - reader.lastContainedLogTimestamp(), - 0 - ); + reader.lastContainedLogTimestamp()); snapshotRegistry.getOrCreateSnapshot(lastCommittedOffset); authorizer.ifPresent(a -> a.loadSnapshot(aclControlManager.idToAcl())); } finally { @@ -1201,13 +1074,11 @@ public void processBatchEndOffset(long offset) { private void updateLastCommittedState( long offset, int epoch, - long timestamp, - long bytesSinceLastSnapshot + long timestamp ) { lastCommittedOffset = offset; lastCommittedEpoch = epoch; lastCommittedTimestamp = timestamp; - committedBytesSinceLastSnapshot = bytesSinceLastSnapshot; controllerMetrics.setLastCommittedRecordOffset(offset); if (!isActiveController()) { @@ -1454,42 +1325,12 @@ private void replay(ApiMessage message, Optional snapshotId, lon } } - private void maybeGenerateSnapshot() { - if (committedBytesSinceLastSnapshot >= snapshotMaxNewRecordBytes && - snapshotGeneratorManager.generator == null - ) { - if (!isActiveController()) { - // The active controller creates in-memory snapshot every time an uncommitted - // batch gets appended. The in-active controller can be more efficient and only - // create an in-memory snapshot when needed. - snapshotRegistry.getOrCreateSnapshot(lastCommittedOffset); - } - - log.info( - "Generating a snapshot that includes (epoch={}, offset={}) after {} committed bytes since the last snapshot because, {}.", - lastCommittedEpoch, - lastCommittedOffset, - committedBytesSinceLastSnapshot, - SnapshotReason.MaxBytesExceeded - ); - - snapshotGeneratorManager.createSnapshotGenerator( - lastCommittedOffset, - lastCommittedEpoch, - lastCommittedTimestamp - ); - committedBytesSinceLastSnapshot = 0; - } - } - /** * Clear all data structures and reset all KRaft state. */ private void resetToEmptyState() { - snapshotGeneratorManager.cancel(); snapshotRegistry.reset(); - - updateLastCommittedState(-1, -1, -1, 0); + updateLastCommittedState(-1, -1, -1); } /** @@ -1603,11 +1444,6 @@ private void resetToEmptyState() { */ private final LogReplayTracker logReplayTracker; - /** - * Manages generating controller snapshots. - */ - private final SnapshotGeneratorManager snapshotGeneratorManager = new SnapshotGeneratorManager(); - /** * The interface that we use to mutate the Raft log. */ @@ -1654,16 +1490,6 @@ private void resetToEmptyState() { */ private long writeOffset; - /** - * Maximum number of bytes processed through handling commits before generating a snapshot. - */ - private final long snapshotMaxNewRecordBytes; - - /** - * Number of bytes processed through handling commits since the last snapshot was generated. - */ - private long committedBytesSinceLastSnapshot = 0; - /** * How long to delay partition leader balancing operations. */ @@ -1716,7 +1542,6 @@ private QuorumController( short defaultReplicationFactor, int defaultNumPartitions, ReplicaPlacer replicaPlacer, - long snapshotMaxNewRecordBytes, OptionalLong leaderImbalanceCheckIntervalNs, OptionalLong maxIdleIntervalNs, long sessionTimeoutNs, @@ -1773,7 +1598,6 @@ private QuorumController( setFeatureControlManager(featureControl). build(); this.producerIdControlManager = new ProducerIdControlManager(clusterControl, snapshotRegistry); - this.snapshotMaxNewRecordBytes = snapshotMaxNewRecordBytes; this.leaderImbalanceCheckIntervalNs = leaderImbalanceCheckIntervalNs; this.maxIdleIntervalNs = maxIdleIntervalNs; this.replicationControl = new ReplicationControlManager.Builder(). @@ -2112,29 +1936,6 @@ public CompletableFuture> createPartitions( }); } - @Override - public CompletableFuture beginWritingSnapshot() { - CompletableFuture future = new CompletableFuture<>(); - appendControlEvent("beginWritingSnapshot", () -> { - if (snapshotGeneratorManager.generator == null) { - log.info( - "Generating a snapshot that includes (epoch={}, offset={}) after {} committed bytes since the last snapshot because, {}.", - lastCommittedEpoch, - lastCommittedOffset, - committedBytesSinceLastSnapshot, - SnapshotReason.UnknownReason - ); - snapshotGeneratorManager.createSnapshotGenerator( - lastCommittedOffset, - lastCommittedEpoch, - lastCommittedTimestamp - ); - } - future.complete(snapshotGeneratorManager.generator.lastContainedLogOffset()); - }); - return future; - } - @Override public CompletableFuture> createAcls( ControllerRequestContext context, diff --git a/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java index 6683d5e147635..dab1176013f17 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java @@ -106,7 +106,6 @@ import java.util.ListIterator; import java.util.Map.Entry; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Optional; import java.util.OptionalInt; import java.util.Set; @@ -1911,39 +1910,6 @@ private void listReassigningTopic(ListPartitionReassignmentsResponseData respons setReplicas(Replicas.toList(partition.replicas))); } - class ReplicationControlIterator implements Iterator> { - private final long epoch; - private final Iterator iterator; - - ReplicationControlIterator(long epoch) { - this.epoch = epoch; - this.iterator = topics.values(epoch).iterator(); - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public List next() { - if (!hasNext()) throw new NoSuchElementException(); - TopicControlInfo topic = iterator.next(); - List records = new ArrayList<>(); - records.add(new ApiMessageAndVersion(new TopicRecord(). - setName(topic.name). - setTopicId(topic.id), TOPIC_RECORD.highestSupportedVersion())); - for (Entry entry : topic.parts.entrySet(epoch)) { - records.add(entry.getValue().toRecord(topic.id, entry.getKey())); - } - return records; - } - } - - ReplicationControlIterator iterator(long epoch) { - return new ReplicationControlIterator(epoch); - } - private static final class IneligibleReplica { private final int replicaId; private final String reason; diff --git a/metadata/src/main/java/org/apache/kafka/controller/SnapshotGenerator.java b/metadata/src/main/java/org/apache/kafka/controller/SnapshotGenerator.java deleted file mode 100644 index d34696ef44f11..0000000000000 --- a/metadata/src/main/java/org/apache/kafka/controller/SnapshotGenerator.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kafka.controller; - -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.OptionalLong; - -import org.apache.kafka.common.utils.LogContext; -import org.apache.kafka.server.common.ApiMessageAndVersion; -import org.apache.kafka.snapshot.SnapshotWriter; -import org.slf4j.Logger; - - -final class SnapshotGenerator { - static class Section { - private final String name; - private final Iterator> iterator; - - Section(String name, Iterator> iterator) { - this.name = name; - this.iterator = iterator; - } - - String name() { - return name; - } - - Iterator> iterator() { - return iterator; - } - } - - private final Logger log; - private final SnapshotWriter writer; - private final int maxBatchesPerGenerateCall; - private final List
sections; - private final Iterator
sectionIterator; - private Iterator> batchIterator; - private List batch; - private Section section; - private long numRecords; - - SnapshotGenerator(LogContext logContext, - SnapshotWriter writer, - int maxBatchesPerGenerateCall, - List
sections) { - this.log = logContext.logger(SnapshotGenerator.class); - this.writer = writer; - this.maxBatchesPerGenerateCall = maxBatchesPerGenerateCall; - this.sections = sections; - this.sectionIterator = this.sections.iterator(); - this.batchIterator = Collections.emptyIterator(); - this.batch = null; - this.section = null; - this.numRecords = 0; - } - - /** - * Returns the last offset from the log that will be included in the snapshot. - */ - long lastContainedLogOffset() { - return writer.lastContainedLogOffset(); - } - - SnapshotWriter writer() { - return writer; - } - - /** - * Generate and write the next batch of records. - * - * @return true if the last batch was generated, otherwise false - */ - private boolean generateBatch() throws Exception { - if (batch == null) { - while (!batchIterator.hasNext()) { - if (section != null) { - log.info("Generated {} record(s) for the {} section of snapshot {}.", - numRecords, section.name(), writer.snapshotId()); - section = null; - numRecords = 0; - } - if (!sectionIterator.hasNext()) { - writer.freeze(); - return true; - } - section = sectionIterator.next(); - log.info("Generating records for the {} section of snapshot {}.", - section.name(), writer.snapshotId()); - batchIterator = section.iterator(); - } - batch = batchIterator.next(); - } - - writer.append(batch); - numRecords += batch.size(); - batch = null; - return false; - } - - /** - * Generate the next few batches of records. - * - * @return The number of nanoseconds to delay before rescheduling the - * generateBatches event, or empty if the snapshot is done. - */ - OptionalLong generateBatches() throws Exception { - for (int numBatches = 0; numBatches < maxBatchesPerGenerateCall; numBatches++) { - if (generateBatch()) { - return OptionalLong.empty(); - } - } - return OptionalLong.of(0); - } -} diff --git a/metadata/src/main/java/org/apache/kafka/image/FeaturesDelta.java b/metadata/src/main/java/org/apache/kafka/image/FeaturesDelta.java index 7f431c2d0615f..ac67c87af4bb6 100644 --- a/metadata/src/main/java/org/apache/kafka/image/FeaturesDelta.java +++ b/metadata/src/main/java/org/apache/kafka/image/FeaturesDelta.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Optional; +import java.util.stream.Collectors; /** @@ -34,7 +35,7 @@ public final class FeaturesDelta { private final Map> changes = new HashMap<>(); - private MetadataVersion metadataVersionChange = null; + private Optional metadataVersionChange = Optional.empty(); public FeaturesDelta(FeaturesImage image) { this.image = image; @@ -44,8 +45,8 @@ public Map> changes() { return changes; } - public Optional metadataVersionChange() { - return Optional.ofNullable(metadataVersionChange); + public Optional metadataVersionChange() { + return metadataVersionChange; } public void finishSnapshot() { @@ -54,17 +55,39 @@ public void finishSnapshot() { changes.put(featureName, Optional.empty()); } } + // Note: we do not need to do anything for metadataVersionChange here, since that is + // handled in the constructor of MetadataDelta. + } + + public Optional calculateMetadataVersionChange(FeatureLevelRecord record) { + // Check if the record alters the metadata version. + if (!record.name().equals(MetadataVersion.FEATURE_NAME)) { + return Optional.empty(); + } + // Check if the new version is the same as the previous one. + MetadataVersion oldVersion = metadataVersionChange.isPresent() ? + metadataVersionChange.get().newVersion() : image.metadataVersion(); + MetadataVersion newVersion = MetadataVersion.fromFeatureLevel(record.featureLevel()); + if (oldVersion.equals(newVersion)) return Optional.empty(); + return Optional.of(new MetadataVersionChange(oldVersion, newVersion)); + } + + public void replayMetadataVersionChange(MetadataVersion newVersion) { + if (image.metadataVersion().equals(newVersion)) { + metadataVersionChange = Optional.empty(); + } else { + metadataVersionChange = Optional.of( + new MetadataVersionChange(image.metadataVersion(), newVersion)); + } } public void replay(FeatureLevelRecord record) { if (record.name().equals(MetadataVersion.FEATURE_NAME)) { - metadataVersionChange = MetadataVersion.fromFeatureLevel(record.featureLevel()); + replayMetadataVersionChange(MetadataVersion.fromFeatureLevel(record.featureLevel())); + } else if (record.featureLevel() == 0) { + changes.put(record.name(), Optional.empty()); } else { - if (record.featureLevel() == 0) { - changes.put(record.name(), Optional.empty()); - } else { - changes.put(record.name(), Optional.of(record.featureLevel())); - } + changes.put(record.name(), Optional.of(record.featureLevel())); } } @@ -89,21 +112,18 @@ public FeaturesImage apply() { } } } - - final MetadataVersion metadataVersion; - if (metadataVersionChange == null) { - metadataVersion = image.metadataVersion(); - } else { - metadataVersion = metadataVersionChange; - } - return new FeaturesImage(newFinalizedVersions, metadataVersion); + return new FeaturesImage(newFinalizedVersions, metadataVersionChange.isPresent() ? + metadataVersionChange.get().newVersion() : image.metadataVersion()); } @Override public String toString() { return "FeaturesDelta(" + - "changes=" + changes + - ", metadataVersionChange=" + metadataVersionChange + - ')'; + "changes=" + changes.keySet().stream(). + sorted(). + map(k -> "k=" + changes.get(k)). + collect(Collectors.joining(", ")) + + ", metadataVersionChange=" + metadataVersionChange.toString() + + ")"; } } diff --git a/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java b/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java index a82216769d03f..8df79a1ee6f7e 100644 --- a/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java @@ -34,26 +34,37 @@ import org.apache.kafka.common.metadata.UnfenceBrokerRecord; import org.apache.kafka.common.metadata.UnregisterBrokerRecord; import org.apache.kafka.common.protocol.ApiMessage; -import org.apache.kafka.raft.OffsetAndEpoch; -import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.server.common.MetadataVersion; -import java.util.Iterator; -import java.util.List; import java.util.Optional; /** * A change to the broker metadata image. - * - * This class is thread-safe. */ public final class MetadataDelta { - private final MetadataImage image; + public static class Builder { + private MetadataImage image = MetadataImage.EMPTY; + private MetadataVersion metadataVersion = null; + + public Builder setImage(MetadataImage image) { + this.image = image; + return this; + } - private long highestOffset; + public Builder setMetadataVersion(MetadataVersion metadataVersion) { + this.metadataVersion = metadataVersion; + return this; + } + + public MetadataDelta build() { + if (metadataVersion == null) metadataVersion = image.features().metadataVersion(); + return new MetadataDelta(image, + metadataVersion); + } + } - private int highestEpoch; + private final MetadataImage image; private FeaturesDelta featuresDelta = null; @@ -69,10 +80,24 @@ public final class MetadataDelta { private AclsDelta aclsDelta = null; - public MetadataDelta(MetadataImage image) { + private MetadataDelta( + MetadataImage image, + MetadataVersion metadataVersion + ) { this.image = image; - this.highestOffset = image.highestOffsetAndEpoch().offset(); - this.highestEpoch = image.highestOffsetAndEpoch().epoch(); + if (!image.features().metadataVersion().equals(metadataVersion)) { + getOrCreateFeaturesDelta().replayMetadataVersionChange(metadataVersion); + } + } + + public void clear() { + this.featuresDelta = null; + this.clusterDelta = null; + this.topicsDelta = null; + this.configsDelta = null; + this.clientQuotasDelta = null; + this.producerIdsDelta = null; + this.aclsDelta = null; } public MetadataImage image() { @@ -144,27 +169,7 @@ public AclsDelta getOrCreateAclsDelta() { return aclsDelta; } - public Optional metadataVersionChanged() { - if (featuresDelta == null) { - return Optional.empty(); - } else { - return featuresDelta.metadataVersionChange(); - } - } - - public void read(long highestOffset, int highestEpoch, Iterator> reader) { - while (reader.hasNext()) { - List batch = reader.next(); - for (ApiMessageAndVersion messageAndVersion : batch) { - replay(highestOffset, highestEpoch, messageAndVersion.message()); - } - } - } - - public void replay(long offset, int epoch, ApiMessage record) { - highestOffset = offset; - highestEpoch = epoch; - + public void replay(ApiMessage record) { MetadataRecordType type = MetadataRecordType.fromId(record.apiKey()); switch (type) { case REGISTER_BROKER_RECORD: @@ -223,13 +228,11 @@ public void replay(long offset, int epoch, ApiMessage record) { } public void replay(RegisterBrokerRecord record) { - if (clusterDelta == null) clusterDelta = new ClusterDelta(image.cluster()); - clusterDelta.replay(record); + getOrCreateClusterDelta().replay(record); } public void replay(UnregisterBrokerRecord record) { - if (clusterDelta == null) clusterDelta = new ClusterDelta(image.cluster()); - clusterDelta.replay(record); + getOrCreateClusterDelta().replay(record); } public void replay(TopicRecord record) { @@ -261,17 +264,32 @@ public void replay(RemoveTopicRecord record) { getOrCreateConfigsDelta().replay(record, topicName); } + public void setMetadataVersion(MetadataVersion metadataVersion) { + getOrCreateFeaturesDelta().replay(new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(metadataVersion.featureLevel())); + } + public void replay(FeatureLevelRecord record) { - getOrCreateFeaturesDelta().replay(record); - featuresDelta.metadataVersionChange().ifPresent(changedMetadataVersion -> { - // If any feature flags change, need to immediately check if any metadata needs to be downgraded. - getOrCreateClusterDelta().handleMetadataVersionChange(changedMetadataVersion); - getOrCreateConfigsDelta().handleMetadataVersionChange(changedMetadataVersion); - getOrCreateTopicsDelta().handleMetadataVersionChange(changedMetadataVersion); - getOrCreateClientQuotasDelta().handleMetadataVersionChange(changedMetadataVersion); - getOrCreateProducerIdsDelta().handleMetadataVersionChange(changedMetadataVersion); - getOrCreateAclsDelta().handleMetadataVersionChange(changedMetadataVersion); - }); + FeaturesDelta delta = getOrCreateFeaturesDelta(); + Optional change = delta.calculateMetadataVersionChange(record); + if (change.isPresent()) { + // + // If this record would change the metadata version, and the base image is not empty, + // we throw an exception here. The caller is expected to handle this exception by saving + // the image to a series of records with the new metadata version, and then reloading it. + // + // The rationale for handling it "out-of-band" like this is that: + // 1) the broker and controller need to trigger a save of the metadata image right + // before any version change, so we need to interrupt the normal control flow anyway. + // 2) it is easier to maintain one downgrade/upgrade code path in the image write + // functions than to try to handle every conceivable change that a metadata version + // could bring in an event-driven fashion. + // 3) metadata version changes are rare so the overhead of this is not a problem. + // + throw new MetadataVersionChangeException(change.get()); + } + delta.replay(record); } public void replay(BrokerRegistrationChangeRecord record) { @@ -308,7 +326,7 @@ public void finishSnapshot() { getOrCreateAclsDelta().finishSnapshot(); } - public MetadataImage apply() { + public MetadataImage apply(MetadataProvenance provenance) { FeaturesImage newFeatures; if (featuresDelta == null) { newFeatures = image.features(); @@ -352,7 +370,7 @@ public MetadataImage apply() { newAcls = aclsDelta.apply(); } return new MetadataImage( - new OffsetAndEpoch(highestOffset, highestEpoch), + provenance, newFeatures, newCluster, newTopics, @@ -366,9 +384,7 @@ public MetadataImage apply() { @Override public String toString() { return "MetadataDelta(" + - "highestOffset=" + highestOffset + - ", highestEpoch=" + highestEpoch + - ", featuresDelta=" + featuresDelta + + "featuresDelta=" + featuresDelta + ", clusterDelta=" + clusterDelta + ", topicsDelta=" + topicsDelta + ", configsDelta=" + configsDelta + diff --git a/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java b/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java index 18c836ee3bd66..36db5dab1de6b 100644 --- a/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java @@ -31,7 +31,7 @@ */ public final class MetadataImage { public final static MetadataImage EMPTY = new MetadataImage( - new OffsetAndEpoch(0, 0), + MetadataProvenance.EMPTY, FeaturesImage.EMPTY, ClusterImage.EMPTY, TopicsImage.EMPTY, @@ -40,7 +40,7 @@ public final class MetadataImage { ProducerIdsImage.EMPTY, AclsImage.EMPTY); - private final OffsetAndEpoch highestOffsetAndEpoch; + private final MetadataProvenance provenance; private final FeaturesImage features; @@ -57,7 +57,7 @@ public final class MetadataImage { private final AclsImage acls; public MetadataImage( - OffsetAndEpoch highestOffsetAndEpoch, + MetadataProvenance provenance, FeaturesImage features, ClusterImage cluster, TopicsImage topics, @@ -66,7 +66,7 @@ public MetadataImage( ProducerIdsImage producerIds, AclsImage acls ) { - this.highestOffsetAndEpoch = highestOffsetAndEpoch; + this.provenance = provenance; this.features = features; this.cluster = cluster; this.topics = topics; @@ -86,8 +86,16 @@ public boolean isEmpty() { acls.isEmpty(); } + public MetadataProvenance provenance() { + return provenance; + } + public OffsetAndEpoch highestOffsetAndEpoch() { - return highestOffsetAndEpoch; + return new OffsetAndEpoch(provenance.offset(), provenance.epoch()); + } + + public long offset() { + return provenance.offset(); } public FeaturesImage features() { @@ -135,7 +143,7 @@ public void write(ImageWriter writer, ImageWriterOptions options) { public boolean equals(Object o) { if (o == null || !o.getClass().equals(this.getClass())) return false; MetadataImage other = (MetadataImage) o; - return highestOffsetAndEpoch.equals(other.highestOffsetAndEpoch) && + return provenance.equals(other.provenance) && features.equals(other.features) && cluster.equals(other.cluster) && topics.equals(other.topics) && @@ -147,7 +155,8 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(highestOffsetAndEpoch, + return Objects.hash( + provenance, features, cluster, topics, @@ -159,7 +168,8 @@ public int hashCode() { @Override public String toString() { - return "MetadataImage(highestOffsetAndEpoch=" + highestOffsetAndEpoch + + return "MetadataImage(" + + "provenance=" + provenance + ", features=" + features + ", cluster=" + cluster + ", topics=" + topics + diff --git a/metadata/src/main/java/org/apache/kafka/image/MetadataProvenance.java b/metadata/src/main/java/org/apache/kafka/image/MetadataProvenance.java new file mode 100644 index 0000000000000..6e487768b507a --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataProvenance.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image; + +import java.util.Objects; + + +/** + * Information about the source of a metadata image. + */ +public final class MetadataProvenance { + public static final MetadataProvenance EMPTY = new MetadataProvenance(-1L, 0, 0L); + + private final long offset; + private final int epoch; + private final long lastContainedLogTimeMs; + + public MetadataProvenance( + long offset, + int epoch, + long lastContainedLogTimeMs + ) { + this.offset = offset; + this.epoch = epoch; + this.lastContainedLogTimeMs = lastContainedLogTimeMs; + } + + public long offset() { + return offset; + } + + public int epoch() { + return epoch; + } + + public long lastContainedLogTimeMs() { + return lastContainedLogTimeMs; + } + + public String snapshotName() { + return String.format("snapshot %020d-%010d", offset, epoch); + } + + @Override + public boolean equals(Object o) { + if (o == null || !o.getClass().equals(this.getClass())) return false; + MetadataProvenance other = (MetadataProvenance) o; + return offset == other.offset && + epoch == other.epoch && + lastContainedLogTimeMs == other.lastContainedLogTimeMs; + } + + @Override + public int hashCode() { + return Objects.hash(offset, + epoch, + lastContainedLogTimeMs); + } + + @Override + public String toString() { + return "MetadataProvenance(" + + "offset=" + offset + + ", epoch=" + epoch + + ", lastContainedLogTimeMs=" + lastContainedLogTimeMs + + ")"; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/MetadataVersionChange.java b/metadata/src/main/java/org/apache/kafka/image/MetadataVersionChange.java new file mode 100644 index 0000000000000..4585c2512a3a0 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataVersionChange.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image; + +import org.apache.kafka.server.common.MetadataVersion; + +import java.util.Objects; + + +/** + * A change in the MetadataVersion. + */ +public final class MetadataVersionChange { + private final MetadataVersion oldVersion; + private final MetadataVersion newVersion; + + public MetadataVersionChange( + MetadataVersion oldVersion, + MetadataVersion newVersion + ) { + this.oldVersion = oldVersion; + this.newVersion = newVersion; + } + + public MetadataVersion oldVersion() { + return oldVersion; + } + + public MetadataVersion newVersion() { + return newVersion; + } + + @Override + public boolean equals(Object o) { + if (o == null || !o.getClass().equals(this.getClass())) return false; + MetadataVersionChange other = (MetadataVersionChange) o; + return oldVersion.equals(other.oldVersion) && + newVersion.equals(other.newVersion); + } + + @Override + public int hashCode() { + return Objects.hash(oldVersion, + newVersion); + } + + @Override + public String toString() { + return "MetadataVersionChange(" + + "oldVersion=" + oldVersion + + ", newVersion=" + newVersion + + ")"; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/metadata/util/SnapshotReason.java b/metadata/src/main/java/org/apache/kafka/image/MetadataVersionChangeException.java similarity index 58% rename from metadata/src/main/java/org/apache/kafka/metadata/util/SnapshotReason.java rename to metadata/src/main/java/org/apache/kafka/image/MetadataVersionChangeException.java index 59d95035f2ddd..a4c931f77b7f6 100644 --- a/metadata/src/main/java/org/apache/kafka/metadata/util/SnapshotReason.java +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataVersionChangeException.java @@ -14,21 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.metadata.util; -public enum SnapshotReason { - UnknownReason("unknown reason"), - MaxBytesExceeded("max bytes were exceeded"), - MetadataVersionChanged("metadata version was changed"); +package org.apache.kafka.image; - private final String snapshotReason; - SnapshotReason(String snapshotReason) { - this.snapshotReason = snapshotReason; +/** + * Indicates that the metadata version has changed. + */ +public final class MetadataVersionChangeException extends RuntimeException { + private static final long serialVersionUID = 1L; + private final MetadataVersionChange change; + + public MetadataVersionChangeException(MetadataVersionChange change) { + super("The metadata version is changing from " + change.oldVersion() + " to " + + change.newVersion()); + this.change = change; } - @Override - public String toString() { - return snapshotReason; + public MetadataVersionChange change() { + return change; } } diff --git a/metadata/src/main/java/org/apache/kafka/image/loader/LogDeltaManifest.java b/metadata/src/main/java/org/apache/kafka/image/loader/LogDeltaManifest.java new file mode 100644 index 0000000000000..c3ab72b3b79f2 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/loader/LogDeltaManifest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.loader; + +import org.apache.kafka.image.MetadataProvenance; + +import java.util.Objects; + + +/** + * Contains information about a set of changes that were loaded from the metadata log. + */ +public class LogDeltaManifest { + /** + * The highest offset and epoch included in this delta, inclusive. + */ + private final MetadataProvenance provenance; + + /** + * The number of batches that were loaded. + */ + private final int numBatches; + + /** + * The time in nanoseconds that it took to load the changes. + */ + private final long elapsedNs; + + /** + * The total size of the records in bytes that we read while creating the delta. + */ + private final long numBytes; + + public LogDeltaManifest( + MetadataProvenance provenance, + int numBatches, + long elapsedNs, + long numBytes + ) { + this.provenance = provenance; + this.numBatches = numBatches; + this.elapsedNs = elapsedNs; + this.numBytes = numBytes; + } + + + public MetadataProvenance provenance() { + return provenance; + } + + public int numBatches() { + return numBatches; + } + + public long elapsedNs() { + return elapsedNs; + } + + public long numBytes() { + return numBytes; + } + + @Override + public int hashCode() { + return Objects.hash( + provenance, + numBatches, + elapsedNs, + numBytes); + } + + @Override + public boolean equals(Object o) { + if (o == null || !o.getClass().equals(this.getClass())) return false; + LogDeltaManifest other = (LogDeltaManifest) o; + return provenance.equals(other.provenance) && + numBatches == other.numBatches && + elapsedNs == other.elapsedNs && + numBytes == other.numBytes; + } + + @Override + public String toString() { + return "LogDeltaManifest(" + + "provenance=" + provenance + + ", numBatches=" + numBatches + + ", elapsedNs=" + elapsedNs + + ", numBytes=" + numBytes + + ")"; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java b/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java new file mode 100644 index 0000000000000..47a3fc6b4b604 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java @@ -0,0 +1,504 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.loader; + +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.image.MetadataDelta; +import org.apache.kafka.image.MetadataImage; +import org.apache.kafka.image.MetadataProvenance; +import org.apache.kafka.image.MetadataVersionChange; +import org.apache.kafka.image.MetadataVersionChangeException; +import org.apache.kafka.image.publisher.MetadataPublisher; +import org.apache.kafka.image.writer.ImageReWriter; +import org.apache.kafka.image.writer.ImageWriterOptions; +import org.apache.kafka.queue.KafkaEventQueue; +import org.apache.kafka.raft.Batch; +import org.apache.kafka.raft.BatchReader; +import org.apache.kafka.raft.LeaderAndEpoch; +import org.apache.kafka.raft.RaftClient; +import org.apache.kafka.server.common.ApiMessageAndVersion; +import org.apache.kafka.server.fault.FaultHandler; +import org.apache.kafka.snapshot.SnapshotReader; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + + +/** + * The MetadataLoader follows changes provided by a RaftClient, and packages them into metadata + * deltas and images that can be consumed by publishers. + * + * The Loader maintains its own thread, which is used to make all callbacks into publishers. If a + * publisher A is installed before B, A will receive all callbacks before B. This is also true if + * A and B are installed as part of a list [A, B]. + * + * Publishers should not modify any data structures passed to them. + * + * It is possible to change the list of publishers dynamically over time. Whenever a new publisher is + * added, it receives a catch-up delta which contains the full state. Any publisher installed when the + * loader is closed will itself be closed. + */ +public class MetadataLoader implements RaftClient.Listener, AutoCloseable { + public static class Builder { + private int nodeId = -1; + private Time time = Time.SYSTEM; + private LogContext logContext = null; + private String threadNamePrefix = ""; + private FaultHandler faultHandler = null; + private MetadataLoaderMetrics metrics = new MetadataLoaderMetrics() { + private volatile long lastAppliedOffset = -1L; + + @Override + public void updateBatchProcessingTime(long elapsedNs) { } + + @Override + public void updateBatchSize(int size) { } + + @Override + public void updateLastAppliedImageProvenance(MetadataProvenance provenance) { + this.lastAppliedOffset = provenance.offset(); + } + + @Override + public long lastAppliedOffset() { + return lastAppliedOffset; + } + + @Override + public void close() throws Exception { } + }; + + public Builder setNodeId(int nodeId) { + this.nodeId = nodeId; + return this; + } + + public Builder setTime(Time time) { + this.time = time; + return this; + } + + public Builder setThreadNamePrefix(String threadNamePrefix) { + this.threadNamePrefix = threadNamePrefix; + return this; + } + + public Builder setFaultHandler(FaultHandler faultHandler) { + this.faultHandler = faultHandler; + return this; + } + + public Builder setMetadataLoaderMetrics(MetadataLoaderMetrics metrics) { + this.metrics = metrics; + return this; + } + + public MetadataLoader build() { + if (logContext == null) { + logContext = new LogContext("[MetadataLoader " + nodeId + "] "); + } + if (faultHandler == null) throw new RuntimeException("You must set a fault handler."); + return new MetadataLoader( + time, + logContext, + threadNamePrefix, + faultHandler, + metrics); + } + } + + /** + * The log4j logger for this loader. + */ + private final Logger log; + + /** + * The clock used by this loader. + */ + private final Time time; + + /** + * The fault handler to use if metadata loading fails. + */ + private final FaultHandler faultHandler; + + /** + * Callbacks for updating metrics. + */ + private final MetadataLoaderMetrics metrics; + + /** + * The publishers which should receive cluster metadata updates. + */ + private final List publishers; + + /** + * The current leader and epoch. + */ + private LeaderAndEpoch currentLeaderAndEpoch = LeaderAndEpoch.UNKNOWN; + + /** + * The current metadata image. Accessed only from the event queue thread. + */ + private MetadataImage image; + + /** + * The event queue which runs this loader. + */ + private final KafkaEventQueue eventQueue; + + private MetadataLoader( + Time time, + LogContext logContext, + String threadNamePrefix, + FaultHandler faultHandler, + MetadataLoaderMetrics metrics + ) { + this.log = logContext.logger(MetadataLoader.class); + this.time = time; + this.faultHandler = faultHandler; + this.metrics = metrics; + this.publishers = new ArrayList<>(); + this.image = MetadataImage.EMPTY; + this.eventQueue = new KafkaEventQueue(time, logContext, threadNamePrefix); + } + + @Override + public void handleCommit(BatchReader reader) { + eventQueue.append(() -> { + try { + MetadataDelta delta = new MetadataDelta.Builder(). + setImage(image). + build(); + LogDeltaManifest manifest = loadLogDelta(delta, reader); + try { + image = delta.apply(manifest.provenance()); + } catch (Throwable e) { + faultHandler.handleFault("Error generating new metadata image from " + + "metadata delta between offset " + image.offset() + + " and " + manifest.provenance().offset(), e); + return; + } + for (MetadataPublisher publisher : publishers) { + try { + publisher.publishLogDelta(delta, image, manifest); + } catch (Throwable e) { + faultHandler.handleFault("Unhandled error publishing the new metadata " + + "image ending at " + manifest.provenance().offset() + + " with publisher " + publisher.name(), e); + } + } + metrics.updateLastAppliedImageProvenance(image.provenance()); + } catch (Throwable e) { + // This is a general catch-all block where we don't expect to end up; + // failure-prone operations should have individual try/catch blocks around them. + faultHandler.handleFault("Unhandled fault in MetadataLoader#handleCommit. " + + "Last image offset was " + image.offset(), e); + } finally { + reader.close(); + } + }); + } + + /** + * Load some batches of records from the log. We have to do some bookkeeping here to + * translate between batch offsets and record offsets, and track the number of bytes we + * have read. Additionally, there is the chance that one of the records is a metadata + * version change which needs to be handled differently. + * + * @param delta The metadata delta we are preparing. + * @param reader The reader which yields the batches. + * @return A manifest of what was loaded. + */ + LogDeltaManifest loadLogDelta( + MetadataDelta delta, + BatchReader reader + ) { + long startNs = time.nanoseconds(); + int numBatches = 0; + long numBytes = 0L; + long lastOffset = image.provenance().offset(); + int lastEpoch = image.provenance().epoch(); + long lastContainedLogTimeMs = image.provenance().lastContainedLogTimeMs(); + + while (reader.hasNext()) { + Batch batch = reader.next(); + int indexWithinBatch = 0; + for (ApiMessageAndVersion record : batch.records()) { + try { + delta.replay(record.message()); + } catch (MetadataVersionChangeException e) { + handleMetadataVersionChange(delta, + e.change(), + batch.baseOffset() + indexWithinBatch, + lastEpoch, + lastContainedLogTimeMs); + } catch (Throwable e) { + faultHandler.handleFault("Error loading metadata log record from offset " + + batch.baseOffset() + indexWithinBatch, e); + } + lastEpoch = batch.epoch(); + lastContainedLogTimeMs = batch.appendTimestamp(); + indexWithinBatch++; + } + metrics.updateBatchSize(batch.records().size()); + lastOffset = batch.lastOffset(); + numBytes += batch.sizeInBytes(); + numBatches++; + } + MetadataProvenance provenance = + new MetadataProvenance(lastOffset, lastEpoch, lastContainedLogTimeMs); + long elapsedNs = time.nanoseconds() - startNs; + // TODO: this metric should be renamed something like "delta processing time" + metrics.updateBatchProcessingTime(elapsedNs); + return new LogDeltaManifest(provenance, + numBatches, + elapsedNs, + numBytes); + } + + /** + * Handle a change in the metadata version. + * + * @param delta The metadata delta we're working with. + * @param change The change we're handling. + * @param changeOffset The offset of the change. + * @param preChangeEpoch The log epoch BEFORE the change. + */ + void handleMetadataVersionChange( + MetadataDelta delta, + MetadataVersionChange change, + long changeOffset, + int preChangeEpoch, + long lastContainedLogTimeMs + ) { + // First, we materialize the current metadata image and send it to all the publishers that + // are interested in preVersionChange images. The most important one is the publisher which + // writes snapshots out to disk. + MetadataProvenance provenance = + new MetadataProvenance(changeOffset - 1, preChangeEpoch, lastContainedLogTimeMs); + PreVersionChangeManifest manifest = new PreVersionChangeManifest(provenance, change); + MetadataImage preVersionChangeImage = delta.apply(provenance); + for (MetadataPublisher publisher : publishers) { + try { + publisher.publishPreVersionChangeImage(delta, preVersionChangeImage, manifest); + } catch (Throwable e) { + faultHandler.handleFault("Error publishing pre-version change image at offset " + + provenance.offset() + " with publisher " + publisher.name(), e); + } + } + // Then, we clear the current delta and write out the current image to it in the new format. + // If any metadata was lost, we just log it here. We cannot prevent the losses because the + // decision to change the metadata version was already taken by the controller. + delta.clear(); + ImageReWriter writer = new ImageReWriter(delta); + preVersionChangeImage.write(writer, new ImageWriterOptions.Builder(). + setMetadataVersion(change.newVersion()). + setLossHandler(loss -> { + log.warn("{}", loss.getMessage()); + }). + build()); + } + + @Override + public void handleSnapshot(SnapshotReader reader) { + eventQueue.append(() -> { + try { + MetadataDelta delta = new MetadataDelta.Builder(). + setImage(image). + build(); + SnapshotManifest manifest = loadSnapshot(delta, reader); + try { + image = delta.apply(manifest.provenance()); + } catch (Throwable e) { + faultHandler.handleFault("Error generating new metadata image from " + + "snapshot at offset " + reader.lastContainedLogOffset(), e); + return; + } + for (MetadataPublisher publisher : publishers) { + try { + publisher.publishSnapshot(delta, image, manifest); + } catch (Throwable e) { + faultHandler.handleFault("Unhandled error publishing the new metadata " + + "image from snapshot at offset " + reader.lastContainedLogOffset() + + " with publisher " + publisher.name(), e); + } + } + metrics.updateLastAppliedImageProvenance(image.provenance()); + } catch (Throwable e) { + // This is a general catch-all block where we don't expect to end up; + // failure-prone operations should have individual try/catch blocks around them. + faultHandler.handleFault("Unhandled fault in MetadataLoader#handleSnapshot. " + + "Snapshot offset was " + reader.lastContainedLogOffset(), e); + } finally { + reader.close(); + } + }); + } + + /** + * Load a snapshot. This is relatively straightforward since we don't track as many things as + * we do in loadLogDelta. Additionally, it is an error for a version change to occur unless + * it is the very first record. The main complication here is that we have to maintain an index + * of what record we are processing so that we can give useful error messages. + * + * @param delta The metadata delta we are preparing. + * @param reader The reader which yields the snapshot batches. + * @return A manifest of what was loaded. + */ + SnapshotManifest loadSnapshot( + MetadataDelta delta, + SnapshotReader reader + ) { + long startNs = time.nanoseconds(); + int snapshotIndex = 0; + while (reader.hasNext()) { + Batch batch = reader.next(); + for (ApiMessageAndVersion record : batch.records()) { + try { + delta.replay(record.message()); + } catch (Throwable e) { + faultHandler.handleFault("Error loading metadata log record " + snapshotIndex + + " in snapshot at offset " + reader.lastContainedLogOffset(), e); + } + snapshotIndex++; + } + } + MetadataProvenance provenance = new MetadataProvenance(reader.lastContainedLogOffset(), + reader.lastContainedLogEpoch(), + reader.lastContainedLogTimestamp()); + return new SnapshotManifest(provenance, + time.nanoseconds() - startNs); + } + + @Override + public void handleLeaderChange(LeaderAndEpoch leaderAndEpoch) { + eventQueue.append(() -> { + currentLeaderAndEpoch = leaderAndEpoch; + }); + } + + /** + * Install a list of publishers. When a publisher is installed, we will publish a MetadataDelta + * to it which contains the entire current image. + * + * @param newPublishers The publishers to install. + * + * @return A future which yields null when the publishers have been added, or + * an exception if the installation failed. + */ + public CompletableFuture installPublishers(List newPublishers) { + if (newPublishers.isEmpty()) return CompletableFuture.completedFuture(null); + CompletableFuture future = new CompletableFuture<>(); + eventQueue.beginShutdown("installPublishers", () -> { + try { + installNewPublishers(newPublishers); + } catch (Throwable e) { + future.completeExceptionally(faultHandler.handleFault("Unhandled fault in " + + "MetadataLoader#installPublishers", e)); + } + }); + return future; + } + + void installNewPublishers( + List newPublishers + ) { + long startNs = time.nanoseconds(); + // Publishers can't be re-installed if they're already present. + for (MetadataPublisher publisher : newPublishers) { + if (publishers.contains(publisher)) { + throw faultHandler.handleFault("Attempted to install publisher " + publisher.name() + + ", which is already installed."); + } + } + MetadataDelta delta = new MetadataDelta.Builder(). + setImage(image). + build(); + ImageReWriter writer = new ImageReWriter(delta); + image.write(writer, new ImageWriterOptions.Builder(). + build()); + SnapshotManifest manifest = new SnapshotManifest( + image.provenance(), + time.nanoseconds() - startNs); + for (MetadataPublisher publisher : newPublishers) { + try { + log.info("Publishing initial snapshot at offset {} to {}", + image.highestOffsetAndEpoch().offset(), publisher.name()); + publisher.publishSnapshot(delta, image, manifest); + publishers.add(publisher); + } catch (Throwable e) { + faultHandler.handleFault("Unhandled error publishing the initial metadata " + + "image from snapshot at offset " + image.highestOffsetAndEpoch().offset() + + " with publisher " + publisher.name(), e); + } + } + } + + /** + * Remove a publisher and close it. + * + * @param publisher The publisher to remove and close. + * + * @return A future which yields null when the publisher has been removed + * and closed, or an exception if the removal failed. + */ + public CompletableFuture removeAndClosePublisher(MetadataPublisher publisher) { + CompletableFuture future = new CompletableFuture<>(); + eventQueue.beginShutdown("removePublisher", () -> { + try { + if (!publishers.remove(publisher)) { + throw faultHandler.handleFault("Attempted to remove publisher " + publisher.name() + + ", which is not installed."); + } + publisher.close(); + future.complete(null); + } catch (Throwable e) { + future.completeExceptionally(e); + } + }); + return future; + } + + public long lastAppliedOffset() { + return metrics.lastAppliedOffset(); + } + + @Override + public void beginShutdown() { + eventQueue.beginShutdown("beginShutdown", () -> { + for (MetadataPublisher publisher : publishers) { + try { + publisher.close(); + } catch (Throwable e) { + faultHandler.handleFault("Got unexpected exception while closing " + + "publisher " + publisher.name(), e); + } + } + }); + } + + @Override + public void close() throws Exception { + beginShutdown(); + eventQueue.close(); + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoaderMetrics.java b/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoaderMetrics.java new file mode 100644 index 0000000000000..f933703bba23a --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoaderMetrics.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.loader; + + +import org.apache.kafka.image.MetadataProvenance; + +/** + * An interface for the metadata loader metrics. + */ +public interface MetadataLoaderMetrics extends AutoCloseable { + /** + * Update the batch processing time histogram. + */ + void updateBatchProcessingTime(long elapsedNs); + + /** + * Update the batch size histogram. + */ + void updateBatchSize(int size); + + /** + * Set the provenance of the last image which has been processed by all publishers. + */ + void updateLastAppliedImageProvenance(MetadataProvenance provenance); + + /** + * Retrieve the last offset which has been processed by all publishers. + */ + long lastAppliedOffset(); +} diff --git a/metadata/src/main/java/org/apache/kafka/image/loader/PreVersionChangeManifest.java b/metadata/src/main/java/org/apache/kafka/image/loader/PreVersionChangeManifest.java new file mode 100644 index 0000000000000..87684e72291b4 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/loader/PreVersionChangeManifest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.loader; + +import org.apache.kafka.image.MetadataProvenance; +import org.apache.kafka.image.MetadataVersionChange; + +import java.util.Objects; + + +/** + * Contains information about a snapshot that we want to publish right before changing the metadata + * version. + */ +public class PreVersionChangeManifest { + /** + * The source of this snapshot. + */ + private final MetadataProvenance provenance; + + /** + * The new metadata version. + */ + private final MetadataVersionChange change; + + public PreVersionChangeManifest( + MetadataProvenance provenance, + MetadataVersionChange change + ) { + this.provenance = provenance; + this.change = change; + } + + public MetadataProvenance provenance() { + return provenance; + } + + public MetadataVersionChange change() { + return change; + } + + @Override + public int hashCode() { + return Objects.hash( + provenance, + change); + } + + @Override + public boolean equals(Object o) { + if (o == null || !o.getClass().equals(this.getClass())) return false; + PreVersionChangeManifest other = (PreVersionChangeManifest) o; + return provenance.equals(other.provenance) && + change.equals(other.change); + } + + @Override + public String toString() { + return "PreVersionChangeManifest(" + + "provenance=" + provenance + + ", change=" + change + + ")"; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/loader/SnapshotManifest.java b/metadata/src/main/java/org/apache/kafka/image/loader/SnapshotManifest.java new file mode 100644 index 0000000000000..b6c6dcce4d5ea --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/loader/SnapshotManifest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.loader; + +import org.apache.kafka.image.MetadataProvenance; + +import java.util.Objects; + + +/** + * Contains information about a snapshot that was loaded. + */ +public class SnapshotManifest { + /** + * The source of this snapshot. + */ + private final MetadataProvenance provenance; + + /** + * The time in microseconds that it took to load the snapshot. + */ + private final long elapsedNs; + + public SnapshotManifest( + MetadataProvenance provenance, + long elapsedNs + ) { + this.provenance = provenance; + this.elapsedNs = elapsedNs; + } + + public MetadataProvenance provenance() { + return provenance; + } + + public long elapsedNs() { + return elapsedNs; + } + + @Override + public int hashCode() { + return Objects.hash( + provenance, + elapsedNs); + } + + @Override + public boolean equals(Object o) { + if (o == null || !o.getClass().equals(this.getClass())) return false; + SnapshotManifest other = (SnapshotManifest) o; + return provenance.equals(other.provenance) && + elapsedNs == other.elapsedNs; + } + + @Override + public String toString() { + return "SnapshotManifest(" + + "provenance=" + provenance + + ", elapsedNs=" + elapsedNs + + ")"; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/publisher/MetadataPublisher.java b/metadata/src/main/java/org/apache/kafka/image/publisher/MetadataPublisher.java new file mode 100644 index 0000000000000..2870952104fdf --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/publisher/MetadataPublisher.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.publisher; + +import org.apache.kafka.image.MetadataDelta; +import org.apache.kafka.image.MetadataImage; +import org.apache.kafka.image.loader.LogDeltaManifest; +import org.apache.kafka.image.loader.PreVersionChangeManifest; +import org.apache.kafka.image.loader.SnapshotManifest; + + +/** + * Publishes metadata deltas which we have loaded from the log and snapshots. + * + * Publishers receive a stream of callbacks from the metadata loader which keeps them notified + * of the latest cluster metadata. This interface abstracts away some of the complications of + * following the cluster metadata. For example, if the loader needs to read a snapshot, it will + * present the contents of the snapshot in the form of a delta from the previous state. + */ +public interface MetadataPublisher extends AutoCloseable { + /** + * Returns the name of this publisher. + * + * @return The publisher name. + */ + String name(); + + /** + * Publish a new cluster metadata snapshot that we loaded. + * + * @param delta The delta between the previous state and the new one. + * @param newImage The complete new state. + * @param manifest The contents of what was published. + */ + void publishSnapshot( + MetadataDelta delta, + MetadataImage newImage, + SnapshotManifest manifest + ); + + /** + * Publish a cluster metadata snapshot that we created prior to changing the metadata version. + * + * @param delta The delta between the previous state and the new one. + * @param preVersionChangeImage The complete new state. + * @param manifest The contents of what was published. + */ + default void publishPreVersionChangeImage( + MetadataDelta delta, + MetadataImage preVersionChangeImage, + PreVersionChangeManifest manifest + ) { + } + + /** + * Publish a change to the cluster metadata. + * + * @param delta The delta between the previous state and the new one. + * @param newImage The complete new state. + * @param manifest The contents of what was published. + */ + void publishLogDelta( + MetadataDelta delta, + MetadataImage newImage, + LogDeltaManifest manifest + ); + + /** + * Close this metadata publisher. + */ + void close() throws Exception; +} diff --git a/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotEmitter.java b/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotEmitter.java new file mode 100644 index 0000000000000..39ddee784f57e --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotEmitter.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.publisher; + + +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.image.MetadataImage; +import org.apache.kafka.image.MetadataProvenance; +import org.apache.kafka.image.writer.ImageWriterOptions; +import org.apache.kafka.image.writer.RaftSnapshotWriter; +import org.apache.kafka.raft.RaftClient; +import org.apache.kafka.server.common.ApiMessageAndVersion; +import org.apache.kafka.snapshot.SnapshotWriter; +import org.slf4j.Logger; + +import java.util.Optional; + +public class SnapshotEmitter implements SnapshotGenerator.Emitter { + private final static int DEFAULT_BATCH_SIZE = 100; + + public static class Builder { + private int nodeId = 0; + private RaftClient raftClient = null; + private int batchSize = DEFAULT_BATCH_SIZE; + + public Builder setNodeId(int nodeId) { + this.nodeId = nodeId; + return this; + } + + public Builder setRaftClient(RaftClient raftClient) { + this.raftClient = raftClient; + return this; + } + + public Builder setBatchSize(int batchSize) { + this.batchSize = batchSize; + return this; + } + + public SnapshotEmitter build() { + if (raftClient == null) throw new RuntimeException("You must set the raftClient."); + return new SnapshotEmitter(nodeId, + raftClient, + batchSize); + } + } + + /** + * The slf4j logger to use. + */ + private final Logger log; + + /** + * The RaftClient to use. + */ + private final RaftClient raftClient; + + /** + * The maximum number of records to put in each batch. + */ + private final int batchSize; + + private SnapshotEmitter( + int nodeId, + RaftClient raftClient, + int batchSize + ) { + this.log = new LogContext("[SnapshotEmitter id=" + nodeId + "] ").logger(SnapshotEmitter.class); + this.raftClient = raftClient; + this.batchSize = batchSize; + } + + @Override + public void emit(MetadataImage image) { + MetadataProvenance provenance = image.provenance(); + Optional> snapshotWriter = + raftClient.createSnapshot(provenance.offset(), + provenance.epoch(), + provenance.lastContainedLogTimeMs()); + if (!snapshotWriter.isPresent()) { + log.error("Not generating {} because it already exists.", provenance.snapshotName()); + return; + } + try (RaftSnapshotWriter writer = new RaftSnapshotWriter(snapshotWriter.get(), batchSize)) { + image.write(writer, new ImageWriterOptions.Builder(). + setMetadataVersion(image.features().metadataVersion()). + build()); + writer.close(true); + } catch (Throwable e) { + log.error("Encountered error while writing {}", provenance.snapshotName(), e); + throw e; + } finally { + try { + snapshotWriter.get().close(); + } catch (Throwable e) { + log.error("Error closing SnapshotWriter for {}", provenance.snapshotName(), e); + throw e; + } + } + log.info("Successfully wrote {}", provenance.snapshotName()); + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java b/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java new file mode 100644 index 0000000000000..68f08eca9cb8a --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.publisher; + +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.image.MetadataDelta; +import org.apache.kafka.image.MetadataImage; +import org.apache.kafka.image.loader.LogDeltaManifest; +import org.apache.kafka.image.loader.PreVersionChangeManifest; +import org.apache.kafka.image.loader.SnapshotManifest; +import org.apache.kafka.queue.EventQueue; +import org.apache.kafka.queue.KafkaEventQueue; +import org.apache.kafka.server.fault.FaultHandler; +import org.slf4j.Logger; + +import java.util.concurrent.TimeUnit; + + +/** + * A metadata publisher that generates snapshots when appropriate. + */ +public class SnapshotGenerator implements MetadataPublisher { + public static class Builder { + private int nodeId = 0; + private Time time = Time.SYSTEM; + private Emitter emitter = null; + private FaultHandler faultHandler = (m, e) -> null; + private long minBytesSinceLastSnapshot = 100 * 1024L * 1024L; + private long minTimeSinceLastSnapshotNs = TimeUnit.DAYS.toNanos(1); + + public Builder setNodeId(int nodeId) { + this.nodeId = nodeId; + return this; + } + + public Builder setTime(Time time) { + this.time = time; + return this; + } + + public Builder setEmitter(Emitter emitter) { + this.emitter = emitter; + return this; + } + + public Builder setFaultHandler(FaultHandler faultHandler) { + this.faultHandler = faultHandler; + return this; + } + + public Builder setMinBytesSinceLastSnapshot(long minBytesSinceLastSnapshot) { + this.minBytesSinceLastSnapshot = minBytesSinceLastSnapshot; + return this; + } + + public Builder setMinTimeSinceLastSnapshotNs(long minTimeSinceLastSnapshotNs) { + this.minTimeSinceLastSnapshotNs = minTimeSinceLastSnapshotNs; + return this; + } + + public SnapshotGenerator build() { + if (emitter == null) throw new RuntimeException("You must set a snapshot emitter."); + return new SnapshotGenerator( + nodeId, + time, + emitter, + faultHandler, + minBytesSinceLastSnapshot, + minTimeSinceLastSnapshotNs + ); + } + } + + /** + * The callback which actually generates the snapshot. + */ + public interface Emitter { + void emit(MetadataImage image); + } + + /** + * The node ID. + */ + private final int nodeId; + + /** + * The clock to use. + */ + private final Time time; + + /** + * The emitter callback, which actually generates the snapshot. + */ + private final Emitter emitter; + + /** + * The slf4j logger to use. + */ + private final Logger log; + + /** + * The fault handler to use. + */ + private final FaultHandler faultHandler; + + /** + * The minimum number of bytes we will wait to see before emitting a snapshot. + */ + private final long minBytesSinceLastSnapshot; + + /** + * The minimum amount of time we will wait before emitting a snapshot, or 0 to disable + * time-based snapshotting. + */ + private final long minTimeSinceLastSnapshotNs; + + /** + * If non-null, the reason why snapshots have been disabled. + */ + private volatile String disabledReason; + + /** + * The event queue used to schedule emitting snapshots. + */ + private final EventQueue eventQueue; + + /** + * The log bytes that we have read since the last snapshot. + */ + private long bytesSinceLastSnapshot; + + /** + * The time at which we created the last snapshot. + */ + private long lastSnapshotTimeNs; + + private SnapshotGenerator( + int nodeId, + Time time, + Emitter emitter, + FaultHandler faultHandler, + long minBytesSinceLastSnapshot, + long minTimeSinceLastSnapshotNs + ) { + this.nodeId = nodeId; + this.time = time; + this.emitter = emitter; + this.faultHandler = faultHandler; + this.minBytesSinceLastSnapshot = minBytesSinceLastSnapshot; + this.minTimeSinceLastSnapshotNs = minTimeSinceLastSnapshotNs; + LogContext logContext = new LogContext("[SnapshotGenerator " + nodeId + "] "); + this.log = logContext.logger(SnapshotGenerator.class); + this.disabledReason = null; + this.eventQueue = new KafkaEventQueue(time, logContext, ""); + resetSnapshotCounters(); + } + + @Override + public String name() { + return "SnapshotGenerator"; + } + + void resetSnapshotCounters() { + this.bytesSinceLastSnapshot = 0L; + this.lastSnapshotTimeNs = time.nanoseconds(); + } + + @Override + public void publishSnapshot( + MetadataDelta delta, + MetadataImage newImage, + SnapshotManifest manifest + ) { + // Reset the snapshot counters because we just read a snapshot. + resetSnapshotCounters(); + } + + @Override + public void publishPreVersionChangeImage( + MetadataDelta delta, + MetadataImage preVersionChangeImage, + PreVersionChangeManifest manifest + ) { + scheduleEmit("the metadata version changed", preVersionChangeImage); + } + + @Override + public void publishLogDelta( + MetadataDelta delta, + MetadataImage newImage, + LogDeltaManifest manifest + ) { + bytesSinceLastSnapshot += manifest.numBytes(); + if (bytesSinceLastSnapshot >= minBytesSinceLastSnapshot) { + if (eventQueue.isEmpty()) { + scheduleEmit("we have replayed at least " + minBytesSinceLastSnapshot + + " bytes", newImage); + } + } else if (minTimeSinceLastSnapshotNs != 0 && + (time.nanoseconds() - lastSnapshotTimeNs >= minTimeSinceLastSnapshotNs)) { + if (eventQueue.isEmpty()) { + scheduleEmit("we have waited at least " + + TimeUnit.NANOSECONDS.toMinutes(minTimeSinceLastSnapshotNs) + " minute(s)", newImage); + } + } + } + + void scheduleEmit( + String reason, + MetadataImage image + ) { + resetSnapshotCounters(); + eventQueue.append(() -> { + String currentDisabledReason = disabledReason; + if (currentDisabledReason != null) { + log.error("Not emitting {} despite the fact that {} because snapshots are " + + "disabled; {}", image.provenance().snapshotName(), reason, + currentDisabledReason); + } else { + log.info("Creating new KRaft snapshot file {} because {}.", + image.provenance().snapshotName(), reason); + try { + emitter.emit(image); + } catch (Throwable e) { + faultHandler.handleFault("KRaft snapshot file generation error", e); + } + } + }); + } + + public void beginShutdown() { + this.disabledReason = "we are shutting down"; + eventQueue.beginShutdown("beginShutdown"); + } + + public void disable(String disabledReason) { + log.error("Disabling periodic KRaft snapshot generation because {}.", disabledReason); + this.disabledReason = disabledReason; + } + + @Override + public void close() throws InterruptedException { + eventQueue.beginShutdown("close"); + eventQueue.close(); + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/writer/ImageReWriter.java b/metadata/src/main/java/org/apache/kafka/image/writer/ImageReWriter.java new file mode 100644 index 0000000000000..42a0aaa93a182 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/writer/ImageReWriter.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.writer; + +import org.apache.kafka.image.MetadataDelta; +import org.apache.kafka.image.MetadataImage; +import org.apache.kafka.server.common.ApiMessageAndVersion; + + +/** + * ImageReWriter writes a metadata image out to another metadata image. + * + * There are a few reasons why you might want to do this. One is to obtain a MetadataDelta + * object which contains everything in the image. Another is to translate an image from + * one metadata version to another. + */ +public class ImageReWriter implements ImageWriter { + private final MetadataDelta delta; + private boolean closed = false; + private MetadataImage image = null; + + public ImageReWriter(MetadataDelta delta) { + this.delta = delta; + } + + @Override + public void write(ApiMessageAndVersion record) { + if (closed) throw new ImageWriterClosedException(); + delta.replay(record.message()); + } + + @Override + public void close(boolean complete) { + if (closed) return; + closed = true; + if (complete) { + image = delta.apply(delta.image().provenance()); + } + } + + public MetadataImage image() { + return image; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/writer/ImageWriterOptions.java b/metadata/src/main/java/org/apache/kafka/image/writer/ImageWriterOptions.java index 635a6c77c406e..906d5420b1bf0 100644 --- a/metadata/src/main/java/org/apache/kafka/image/writer/ImageWriterOptions.java +++ b/metadata/src/main/java/org/apache/kafka/image/writer/ImageWriterOptions.java @@ -43,12 +43,16 @@ public Builder setMetadataVersion(MetadataVersion metadataVersion) { return this; } - // Package-private for testing + // Visible for testing public Builder setRawMetadataVersion(MetadataVersion metadataVersion) { this.metadataVersion = metadataVersion; return this; } + public MetadataVersion metadataVersion() { + return metadataVersion; + } + public Builder setLossHandler(Consumer lossHandler) { this.lossHandler = lossHandler; return this; diff --git a/metadata/src/test/java/org/apache/kafka/controller/AclControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/AclControlManagerTest.java index fdc0327645176..566fa4acd5471 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/AclControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/AclControlManagerTest.java @@ -215,11 +215,9 @@ public void testLoadSnapshot() { // Verify that the ACLs stored in the AclControlManager match the ones we expect. Set foundAcls = new HashSet<>(); - for (Iterator> iterator = manager.iterator(Long.MAX_VALUE); - iterator.hasNext(); ) { - for (ApiMessageAndVersion apiMessageAndVersion : iterator.next()) { - assertTrue(foundAcls.add(apiMessageAndVersion)); - } + for (Map.Entry entry : manager.idToAcl().entrySet()) { + foundAcls.add(new ApiMessageAndVersion( + new StandardAclWithId(entry.getKey(), entry.getValue()).toRecord(), (short) 0)); } assertEquals(loadedAcls, foundAcls); @@ -233,7 +231,7 @@ public void testLoadSnapshot() { // a cluster metadata authorizer. snapshotRegistry.revertToSnapshot(0); authorizer.loadSnapshot(manager.idToAcl()); - assertFalse(manager.iterator(Long.MAX_VALUE).hasNext()); + assertTrue(manager.idToAcl().isEmpty()); } @Test @@ -243,11 +241,9 @@ public void testAddAndDelete() { MockClusterMetadataAuthorizer authorizer = new MockClusterMetadataAuthorizer(); authorizer.loadSnapshot(manager.idToAcl()); manager.replay(StandardAclWithIdTest.TEST_ACLS.get(0).toRecord(), Optional.empty()); - assertEquals(new ApiMessageAndVersion(TEST_ACLS.get(0).toRecord(), (short) 0), - manager.iterator(Long.MAX_VALUE).next().get(0)); manager.replay(new RemoveAccessControlEntryRecord(). setId(TEST_ACLS.get(0).id()), Optional.empty()); - assertFalse(manager.iterator(Long.MAX_VALUE).hasNext()); + assertTrue(manager.idToAcl().isEmpty()); } @Test @@ -284,7 +280,7 @@ public void testCreateAclDeleteAcl() { } } RecordTestUtils.replayAll(manager, createResult.records()); - assertTrue(manager.iterator(Long.MAX_VALUE).hasNext()); + assertFalse(manager.idToAcl().isEmpty()); ControllerResult> deleteResult = manager.deleteAcls(Arrays.asList( @@ -308,12 +304,8 @@ public void testCreateAclDeleteAcl() { deleteResult.response().get(1).exception().get().getClass()); RecordTestUtils.replayAll(manager, deleteResult.records()); - Iterator> iterator = manager.iterator(Long.MAX_VALUE); - assertTrue(iterator.hasNext()); - List list = iterator.next(); - assertEquals(1, list.size()); - assertEquals(TEST_ACLS.get(1).toBinding(), StandardAcl.fromRecord( - (AccessControlEntryRecord) list.get(0).message()).toBinding()); + Iterator> iterator = manager.idToAcl().entrySet().iterator(); + assertEquals(TEST_ACLS.get(1).acl(), iterator.next().getValue()); assertFalse(iterator.hasNext()); } diff --git a/metadata/src/test/java/org/apache/kafka/controller/ClientQuotaControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ClientQuotaControlManagerTest.java index b915db3fe0d49..1fb81cbf7a473 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ClientQuotaControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ClientQuotaControlManagerTest.java @@ -205,49 +205,50 @@ public void testEntityTypes() throws Exception { List alters = new ArrayList<>(); quotasToTest.forEach((entity, quota) -> entityQuotaToAlterations(entity, quota, alters::add)); - alterQuotas(alters, manager); - - RecordTestUtils.assertBatchIteratorContains(Arrays.asList( - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + List records = alterQuotas(alters, manager); + List expectedRecords = Arrays.asList( + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName("user-1"), new EntityData().setEntityType("client-id").setEntityName("client-id-1"))). - setKey("request_percentage").setValue(50.5).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(50.5).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName("user-2"), new EntityData().setEntityType("client-id").setEntityName("client-id-1"))). - setKey("request_percentage").setValue(51.51).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(51.51).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName("user-3"), new EntityData().setEntityType("client-id").setEntityName("client-id-2"))). - setKey("request_percentage").setValue(52.52).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(52.52).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName(null), new EntityData().setEntityType("client-id").setEntityName("client-id-1"))). - setKey("request_percentage").setValue(53.53).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(53.53).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName("user-1"), new EntityData().setEntityType("client-id").setEntityName(null))). - setKey("request_percentage").setValue(54.54).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(54.54).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName("user-3"), new EntityData().setEntityType("client-id").setEntityName(null))). - setKey("request_percentage").setValue(55.55).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(55.55).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName("user-1"))). - setKey("request_percentage").setValue(56.56).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(56.56).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName("user-2"))). - setKey("request_percentage").setValue(57.57).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(57.57).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName("user-3"))). - setKey("request_percentage").setValue(58.58).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(58.58).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName(null))). - setKey("request_percentage").setValue(59.59).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(59.59).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("client-id").setEntityName("client-id-2"))). - setKey("request_percentage").setValue(60.60).setRemove(false), (short) 0))), - manager.iterator(Long.MAX_VALUE)); + setKey("request_percentage").setValue(60.60).setRemove(false), (short) 0)); + RecordTestUtils.deepSortRecords(records); + RecordTestUtils.deepSortRecords(expectedRecords); + assertEquals(expectedRecords, records); } static void entityQuotaToAlterations(ClientQuotaEntity entity, Map quota, @@ -258,11 +259,15 @@ static void entityQuotaToAlterations(ClientQuotaEntity entity, Map alterations, ClientQuotaControlManager manager) { + static List alterQuotas( + List alterations, + ClientQuotaControlManager manager + ) { ControllerResult> result = manager.alterClientQuotas(alterations); assertTrue(result.response().values().stream().allMatch(ApiError::isSuccess)); result.records().forEach(apiMessageAndVersion -> manager.replay((ClientQuotaRecord) apiMessageAndVersion.message())); + return result.records(); } static Map quotas(String key, Double value) { diff --git a/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java index ceb12cc62542d..24c9086ed9eb7 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java @@ -34,12 +34,12 @@ import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.image.writer.ImageWriterOptions; import org.apache.kafka.metadata.BrokerRegistration; import org.apache.kafka.metadata.BrokerRegistrationFencingChange; import org.apache.kafka.metadata.BrokerRegistrationInControlledShutdownChange; import org.apache.kafka.metadata.BrokerRegistrationReply; import org.apache.kafka.metadata.FinalizedControllerFeatures; -import org.apache.kafka.metadata.RecordTestUtils; import org.apache.kafka.metadata.VersionRange; import org.apache.kafka.metadata.placement.ClusterDescriber; import org.apache.kafka.metadata.placement.PlacementSpec; @@ -425,7 +425,7 @@ public Iterator usableBrokers() { @ParameterizedTest @EnumSource(value = MetadataVersion.class, names = {"IBP_3_3_IV2", "IBP_3_3_IV3"}) - public void testIterator(MetadataVersion metadataVersion) throws Exception { + public void testRegistrationsToRecords(MetadataVersion metadataVersion) throws Exception { MockTime time = new MockTime(0, 0, 0); SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); FeatureControlManager featureControl = new FeatureControlManager.Builder(). @@ -467,8 +467,12 @@ public void testIterator(MetadataVersion metadataVersion) throws Exception { IN_CONTROLLED_SHUTDOWN.value()); clusterControl.replay(registrationChangeRecord); short expectedVersion = metadataVersion.registerBrokerRecordVersion(); - RecordTestUtils.assertBatchIteratorContains(Arrays.asList( - Arrays.asList(new ApiMessageAndVersion(new RegisterBrokerRecord(). + + ImageWriterOptions options = new ImageWriterOptions.Builder(). + setMetadataVersion(metadataVersion). + setLossHandler(__ -> { }). + build(); + assertEquals(new ApiMessageAndVersion(new RegisterBrokerRecord(). setBrokerEpoch(100).setBrokerId(0).setRack(null). setEndPoints(new BrokerEndpointCollection(Collections.singleton( new BrokerEndpoint().setSecurityProtocol(SecurityProtocol.PLAINTEXT.id). @@ -476,27 +480,28 @@ public void testIterator(MetadataVersion metadataVersion) throws Exception { setName("PLAINTEXT"). setHost("example.com")).iterator())). setInControlledShutdown(metadataVersion.isInControlledShutdownStateSupported()). - setFenced(false), expectedVersion)), - Arrays.asList(new ApiMessageAndVersion(new RegisterBrokerRecord(). + setFenced(false), expectedVersion), + clusterControl.brokerRegistrations().get(0).toRecord(options)); + assertEquals(new ApiMessageAndVersion(new RegisterBrokerRecord(). setBrokerEpoch(100).setBrokerId(1).setRack(null). setEndPoints(new BrokerEndpointCollection(Collections.singleton( new BrokerEndpoint().setSecurityProtocol(SecurityProtocol.PLAINTEXT.id). setPort((short) 9093). setName("PLAINTEXT"). setHost("example.com")).iterator())). - setFenced(false), expectedVersion)), - Arrays.asList(new ApiMessageAndVersion(new RegisterBrokerRecord(). + setFenced(false), expectedVersion), + clusterControl.brokerRegistrations().get(1).toRecord(options)); + assertEquals(new ApiMessageAndVersion(new RegisterBrokerRecord(). setBrokerEpoch(100).setBrokerId(2).setRack(null). setEndPoints(new BrokerEndpointCollection(Collections.singleton( new BrokerEndpoint().setSecurityProtocol(SecurityProtocol.PLAINTEXT.id). setPort((short) 9094). setName("PLAINTEXT"). setHost("example.com")).iterator())). - setFenced(true), expectedVersion))), - clusterControl.iterator(Long.MAX_VALUE)); + setFenced(true), expectedVersion), + clusterControl.brokerRegistrations().get(2).toRecord(options)); } - @Test public void testRegistrationWithUnsupportedMetadataVersion() { SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); diff --git a/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java index 02bd6e4d7e969..bbca7bfbf6b67 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java @@ -135,14 +135,6 @@ public void testReplay() throws Exception { setName("def").setValue("blah")); assertEquals(toMap(entry("abc", "x,y,z"), entry("def", "blah")), manager.getConfigs(MYTOPIC)); - RecordTestUtils.assertBatchIteratorContains(asList( - asList(new ApiMessageAndVersion(new ConfigRecord(). - setResourceType(TOPIC.id()).setResourceName("mytopic"). - setName("abc").setValue("x,y,z"), CONFIG_RECORD.highestSupportedVersion()), - new ApiMessageAndVersion(new ConfigRecord(). - setResourceType(TOPIC.id()).setResourceName("mytopic"). - setName("def").setValue("blah"), CONFIG_RECORD.highestSupportedVersion()))), - manager.iterator(Long.MAX_VALUE)); } @Test diff --git a/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java index 2478f4ce164a4..8345b4e314955 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java @@ -21,6 +21,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -202,25 +203,7 @@ public void testUpdateFeaturesErrorCases() { } @Test - public void testFeatureControlIteratorWithOldMetadataVersion() throws Exception { - // We require minimum of IBP_3_3_IV0 to write metadata version in the snapshot. - - LogContext logContext = new LogContext(); - SnapshotRegistry snapshotRegistry = new SnapshotRegistry(logContext); - FeatureControlManager manager = new FeatureControlManager.Builder() - .setLogContext(logContext) - .setSnapshotRegistry(snapshotRegistry) - .setMetadataVersion(MetadataVersion.IBP_3_2_IV0) - .build(); - - RecordTestUtils.assertBatchIteratorContains( - Collections.emptyList(), - manager.iterator(Long.MAX_VALUE) - ); - } - - @Test - public void testFeatureControlIterator() throws Exception { + public void testReplayRecords() throws Exception { LogContext logContext = new LogContext(); SnapshotRegistry snapshotRegistry = new SnapshotRegistry(logContext); FeatureControlManager manager = new FeatureControlManager.Builder(). @@ -233,17 +216,12 @@ public void testFeatureControlIterator() throws Exception { updateFeatures(updateMap("foo", 5, "bar", 1), Collections.emptyMap(), Collections.emptyMap(), false); RecordTestUtils.replayAll(manager, result.records()); - RecordTestUtils.assertBatchIteratorContains(Arrays.asList( - Arrays.asList(new ApiMessageAndVersion(new FeatureLevelRecord(). - setName("metadata.version"). - setFeatureLevel((short) 4), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new FeatureLevelRecord(). - setName("foo"). - setFeatureLevel((short) 5), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new FeatureLevelRecord(). - setName("bar"). - setFeatureLevel((short) 1), (short) 0))), - manager.iterator(Long.MAX_VALUE)); + assertEquals(MetadataVersion.IBP_3_3_IV0, manager.metadataVersion()); + assertEquals(Optional.of((short) 5), manager.finalizedFeatures(Long.MAX_VALUE).get("foo")); + assertEquals(Optional.of((short) 1), manager.finalizedFeatures(Long.MAX_VALUE).get("bar")); + assertEquals(new HashSet<>(Arrays.asList( + MetadataVersion.FEATURE_NAME, "foo", "bar")), + manager.finalizedFeatures(Long.MAX_VALUE).featureNames()); } private static final FeatureControlManager.Builder TEST_MANAGER_BUILDER1 = diff --git a/metadata/src/test/java/org/apache/kafka/controller/ProducerIdControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ProducerIdControlManagerTest.java index 80c5c505ae0eb..d0c16d1e3b0d7 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ProducerIdControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ProducerIdControlManagerTest.java @@ -26,20 +26,14 @@ import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; -import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.server.common.MetadataVersion; import org.apache.kafka.server.common.ProducerIdsBlock; import org.apache.kafka.timeline.SnapshotRegistry; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.Iterator; -import java.util.List; - import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; public class ProducerIdControlManagerTest { @@ -149,29 +143,11 @@ public void testMaxValue() { } @Test - public void testSnapshotIterator() { - ProducerIdsBlock range = null; + public void testGenerateProducerIds() { for (int i = 0; i < 100; i++) { - range = generateProducerIds(producerIdControlManager, i % 4, 100); - } - - Iterator> snapshotIterator = producerIdControlManager.iterator(Long.MAX_VALUE); - assertTrue(snapshotIterator.hasNext()); - List batch = snapshotIterator.next(); - assertEquals(1, batch.size(), "Producer IDs record batch should only contain a single record"); - assertEquals(range.firstProducerId() + range.size(), ((ProducerIdsRecord) batch.get(0).message()).nextProducerId()); - assertFalse(snapshotIterator.hasNext(), "Producer IDs iterator should only contain a single batch"); - - ProducerIdControlManager newProducerIdManager = new ProducerIdControlManager(clusterControl, snapshotRegistry); - snapshotIterator = producerIdControlManager.iterator(Long.MAX_VALUE); - while (snapshotIterator.hasNext()) { - snapshotIterator.next().forEach(message -> newProducerIdManager.replay((ProducerIdsRecord) message.message())); + generateProducerIds(producerIdControlManager, i % 4, 100); } - - // Verify that after reloading state from this "snapshot", we don't produce any overlapping IDs - long lastProducerID = range.firstProducerId() + range.size() - 1; - range = generateProducerIds(producerIdControlManager, 1, 100); - assertTrue(range.firstProducerId() > lastProducerID); + assertEquals(new ProducerIdsBlock(3, 100000, 1000), producerIdControlManager.nextProducerBlock()); } static ProducerIdsBlock generateProducerIds( diff --git a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java index 90807a2e2a7f6..127f1afa5429a 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java @@ -31,8 +31,6 @@ import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; -import java.util.Spliterator; -import java.util.Spliterators; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -42,7 +40,6 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; -import java.util.stream.StreamSupport; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.BrokerIdNotRegisteredException; @@ -85,27 +82,22 @@ import org.apache.kafka.common.metadata.TopicRecord; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ApiError; -import org.apache.kafka.common.utils.BufferSupplier; import org.apache.kafka.controller.QuorumController.ConfigResourceExistenceChecker; import org.apache.kafka.metadata.BrokerHeartbeatReply; +import org.apache.kafka.metadata.BrokerRegistrationFencingChange; import org.apache.kafka.metadata.BrokerRegistrationReply; import org.apache.kafka.metadata.FinalizedControllerFeatures; -import org.apache.kafka.metadata.MetadataRecordSerde; import org.apache.kafka.metadata.PartitionRegistration; -import org.apache.kafka.metadata.RecordTestUtils; import org.apache.kafka.metadata.authorizer.StandardAuthorizer; import org.apache.kafka.metadata.bootstrap.BootstrapMetadata; import org.apache.kafka.metadata.util.BatchFileWriter; import org.apache.kafka.metalog.LocalLogManager; import org.apache.kafka.metalog.LocalLogManagerTestEnv; -import org.apache.kafka.raft.Batch; import org.apache.kafka.raft.OffsetAndEpoch; import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.server.common.MetadataVersion; import org.apache.kafka.snapshot.FileRawSnapshotReader; -import org.apache.kafka.snapshot.SnapshotReader; import org.apache.kafka.snapshot.RawSnapshotReader; -import org.apache.kafka.snapshot.RecordsSnapshotReader; import org.apache.kafka.snapshot.Snapshots; import org.apache.kafka.test.TestUtils; import org.junit.jupiter.api.Disabled; @@ -128,7 +120,6 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; @Timeout(value = 40) @@ -669,345 +660,80 @@ public void testSnapshotSaveAndLoad() throws Throwable { fooId = fooData.topics().find("foo").topicId(); active.allocateProducerIds(ANONYMOUS_CONTEXT, new AllocateProducerIdsRequestData().setBrokerId(0).setBrokerEpoch(brokerEpochs.get(0))).get(); - long snapshotLogOffset = active.beginWritingSnapshot().get(); - reader = logEnv.waitForSnapshot(snapshotLogOffset); - SnapshotReader snapshot = createSnapshotReader(reader); - assertEquals(snapshotLogOffset, snapshot.lastContainedLogOffset()); - checkSnapshotContent(expectedSnapshotContent(fooId, brokerEpochs), snapshot); + controlEnv.close(); + assertEquals(generateTestRecords(fooId, brokerEpochs), logEnv.allRecords()); } - - try ( - LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv.Builder(3). - setSnapshotReader(reader). - build(); - QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv.Builder(logEnv). - setControllerBuilderInitializer(controllerBuilder -> { - controllerBuilder.setConfigSchema(SCHEMA); - }). - build(); - ) { - QuorumController active = controlEnv.activeController(); - long snapshotLogOffset = active.beginWritingSnapshot().get(); - SnapshotReader snapshot = createSnapshotReader( - logEnv.waitForSnapshot(snapshotLogOffset) - ); - assertEquals(snapshotLogOffset, snapshot.lastContainedLogOffset()); - checkSnapshotContent(expectedSnapshotContent(fooId, brokerEpochs), snapshot); - } - } - - @Test - public void testSnapshotConfiguration() throws Throwable { - final int numBrokers = 4; - final int maxNewRecordBytes = 4; - Map brokerEpochs = new HashMap<>(); - Uuid fooId; - try ( - LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv.Builder(3). - build(); - QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv.Builder(logEnv). - setControllerBuilderInitializer(controllerBuilder -> { - controllerBuilder.setConfigSchema(SCHEMA); - controllerBuilder.setSnapshotMaxNewRecordBytes(maxNewRecordBytes); - controllerBuilder.setBootstrapMetadata(SIMPLE_BOOTSTRAP); - }). - build(); - ) { - QuorumController active = controlEnv.activeController(); - for (int i = 0; i < numBrokers; i++) { - BrokerRegistrationReply reply = active.registerBroker(ANONYMOUS_CONTEXT, - new BrokerRegistrationRequestData(). - setBrokerId(i). - setRack(null). - setClusterId(active.clusterId()). - setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_3_IV3)). - setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB" + i)). - setListeners(new ListenerCollection(Arrays.asList(new Listener(). - setName("PLAINTEXT").setHost("localhost"). - setPort(9092 + i)).iterator()))).get(); - brokerEpochs.put(i, reply.epoch()); - } - for (int i = 0; i < numBrokers - 1; i++) { - assertEquals(new BrokerHeartbeatReply(true, false, false, false), - active.processBrokerHeartbeat(ANONYMOUS_CONTEXT, new BrokerHeartbeatRequestData(). - setWantFence(false).setBrokerEpoch(brokerEpochs.get(i)). - setBrokerId(i).setCurrentMetadataOffset(100000L)).get()); - } - CreateTopicsResponseData fooData = active.createTopics(ANONYMOUS_CONTEXT, - new CreateTopicsRequestData().setTopics( - new CreatableTopicCollection(Collections.singleton( - new CreatableTopic().setName("foo").setNumPartitions(-1). - setReplicationFactor((short) -1). - setAssignments(new CreatableReplicaAssignmentCollection( - Arrays.asList(new CreatableReplicaAssignment(). - setPartitionIndex(0). - setBrokerIds(Arrays.asList(0, 1, 2)), - new CreatableReplicaAssignment(). - setPartitionIndex(1). - setBrokerIds(Arrays.asList(1, 2, 0))). - iterator()))).iterator())), - Collections.singleton("foo")).get(); - fooId = fooData.topics().find("foo").topicId(); - active.allocateProducerIds(ANONYMOUS_CONTEXT, - new AllocateProducerIdsRequestData().setBrokerId(0).setBrokerEpoch(brokerEpochs.get(0))).get(); - - SnapshotReader snapshot = createSnapshotReader(logEnv.waitForLatestSnapshot()); - checkSnapshotSubcontent( - expectedSnapshotContent(fooId, brokerEpochs), - snapshot - ); - } - } - - @Test - public void testSnapshotOnlyAfterConfiguredMinBytes() throws Throwable { - final int numBrokers = 4; - final int maxNewRecordBytes = 1000; - Map brokerEpochs = new HashMap<>(); - try ( - LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv.Builder(3). - build(); - QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv.Builder(logEnv). - setControllerBuilderInitializer(controllerBuilder -> { - controllerBuilder.setConfigSchema(SCHEMA); - controllerBuilder.setSnapshotMaxNewRecordBytes(maxNewRecordBytes); - }). - build(); - ) { - QuorumController active = controlEnv.activeController(); - for (int i = 0; i < numBrokers; i++) { - BrokerRegistrationReply reply = active.registerBroker(ANONYMOUS_CONTEXT, - new BrokerRegistrationRequestData(). - setBrokerId(i). - setRack(null). - setClusterId(active.clusterId()). - setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_3_IV3)). - setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB" + i)). - setListeners(new ListenerCollection(Arrays.asList(new Listener(). - setName("PLAINTEXT").setHost("localhost"). - setPort(9092 + i)).iterator()))).get(); - brokerEpochs.put(i, reply.epoch()); - assertEquals(new BrokerHeartbeatReply(true, false, false, false), - active.processBrokerHeartbeat(ANONYMOUS_CONTEXT, new BrokerHeartbeatRequestData(). - setWantFence(false).setBrokerEpoch(brokerEpochs.get(i)). - setBrokerId(i).setCurrentMetadataOffset(100000L)).get()); - } - - assertTrue(logEnv.appendedBytes() < maxNewRecordBytes, - String.format("%s appended bytes is not less than %s max new record bytes", - logEnv.appendedBytes(), - maxNewRecordBytes)); - - // Keep creating topic until we reached the max bytes limit - int counter = 0; - while (logEnv.appendedBytes() < maxNewRecordBytes) { - counter += 1; - String topicName = String.format("foo-%s", counter); - active.createTopics(ANONYMOUS_CONTEXT, new CreateTopicsRequestData().setTopics( - new CreatableTopicCollection(Collections.singleton( - new CreatableTopic().setName(topicName).setNumPartitions(-1). - setReplicationFactor((short) -1). - setAssignments(new CreatableReplicaAssignmentCollection( - Arrays.asList(new CreatableReplicaAssignment(). - setPartitionIndex(0). - setBrokerIds(Arrays.asList(0, 1, 2)), - new CreatableReplicaAssignment(). - setPartitionIndex(1). - setBrokerIds(Arrays.asList(1, 2, 0))). - iterator()))).iterator())), - Collections.singleton(topicName)).get(60, TimeUnit.SECONDS); - } - logEnv.waitForLatestSnapshot(); - } - } - - @Test - public void testSnapshotAfterRepeatedResign() throws Throwable { - final int numBrokers = 4; - final int maxNewRecordBytes = 1000; - Map brokerEpochs = new HashMap<>(); - try ( - LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv.Builder(1). - build(); - QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv.Builder(logEnv). - setControllerBuilderInitializer(controllerBuilder -> { - controllerBuilder.setConfigSchema(SCHEMA); - controllerBuilder.setSnapshotMaxNewRecordBytes(maxNewRecordBytes); - }). - build(); - ) { - QuorumController active = controlEnv.activeController(); - for (int i = 0; i < numBrokers; i++) { - BrokerRegistrationReply reply = active.registerBroker(ANONYMOUS_CONTEXT, - new BrokerRegistrationRequestData(). - setBrokerId(i). - setRack(null). - setClusterId(active.clusterId()). - setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_3_IV3)). - setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB" + i)). - setListeners(new ListenerCollection(Arrays.asList(new Listener(). - setName("PLAINTEXT").setHost("localhost"). - setPort(9092 + i)).iterator()))).get(); - brokerEpochs.put(i, reply.epoch()); - assertEquals(new BrokerHeartbeatReply(true, false, false, false), - active.processBrokerHeartbeat(ANONYMOUS_CONTEXT, new BrokerHeartbeatRequestData(). - setWantFence(false).setBrokerEpoch(brokerEpochs.get(i)). - setBrokerId(i).setCurrentMetadataOffset(100000L)).get()); - } - - assertTrue(logEnv.appendedBytes() < maxNewRecordBytes, - String.format("%s appended bytes is not less than %s max new record bytes", - logEnv.appendedBytes(), - maxNewRecordBytes)); - - // Keep creating topic and resign leader until we reached the max bytes limit - int counter = 0; - while (logEnv.appendedBytes() < maxNewRecordBytes) { - active = controlEnv.activeController(); - - counter += 1; - String topicName = String.format("foo-%s", counter); - active.createTopics(ANONYMOUS_CONTEXT, new CreateTopicsRequestData().setTopics( - new CreatableTopicCollection(Collections.singleton( - new CreatableTopic().setName(topicName).setNumPartitions(-1). - setReplicationFactor((short) -1). - setAssignments(new CreatableReplicaAssignmentCollection( - Arrays.asList(new CreatableReplicaAssignment(). - setPartitionIndex(0). - setBrokerIds(Arrays.asList(0, 1, 2)), - new CreatableReplicaAssignment(). - setPartitionIndex(1). - setBrokerIds(Arrays.asList(1, 2, 0))). - iterator()))).iterator())), - Collections.singleton(topicName)).get(60, TimeUnit.SECONDS); - - LocalLogManager activeLocalLogManager = logEnv.logManagers().get(active.nodeId()); - activeLocalLogManager.resign(activeLocalLogManager.leaderAndEpoch().epoch()); - } - logEnv.waitForLatestSnapshot(); - } - } - - private SnapshotReader createSnapshotReader(RawSnapshotReader reader) { - return RecordsSnapshotReader.of( - reader, - new MetadataRecordSerde(), - BufferSupplier.create(), - Integer.MAX_VALUE, - true - ); } - private List expectedSnapshotContent(Uuid fooId, Map brokerEpochs) { + private List generateTestRecords(Uuid fooId, Map brokerEpochs) { return Arrays.asList( new ApiMessageAndVersion(new FeatureLevelRecord(). setName(MetadataVersion.FEATURE_NAME). setFeatureLevel(MetadataVersion.IBP_3_3_IV3.featureLevel()), (short) 0), - new ApiMessageAndVersion(new TopicRecord(). - setName("foo").setTopicId(fooId), (short) 0), - new ApiMessageAndVersion(new PartitionRecord().setPartitionId(0). - setTopicId(fooId).setReplicas(Arrays.asList(0, 1, 2)). - setIsr(Arrays.asList(0, 1, 2)).setRemovingReplicas(Collections.emptyList()). - setAddingReplicas(Collections.emptyList()).setLeader(0).setLeaderEpoch(0). - setPartitionEpoch(0), (short) 0), - new ApiMessageAndVersion(new PartitionRecord().setPartitionId(1). - setTopicId(fooId).setReplicas(Arrays.asList(1, 2, 0)). - setIsr(Arrays.asList(1, 2, 0)).setRemovingReplicas(Collections.emptyList()). - setAddingReplicas(Collections.emptyList()).setLeader(1).setLeaderEpoch(0). - setPartitionEpoch(0), (short) 0), new ApiMessageAndVersion(new RegisterBrokerRecord(). setBrokerId(0).setBrokerEpoch(brokerEpochs.get(0)). setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB0")). - setEndPoints( - new BrokerEndpointCollection( - Arrays.asList( - new BrokerEndpoint().setName("PLAINTEXT").setHost("localhost"). - setPort(9092).setSecurityProtocol((short) 0)).iterator())). + setEndPoints(new BrokerEndpointCollection( + Arrays.asList(new BrokerEndpoint().setName("PLAINTEXT").setHost("localhost"). + setPort(9092).setSecurityProtocol((short) 0)).iterator())). setFeatures(registrationFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_3_IV3)). setRack(null). - setFenced(false), (short) 1), + setFenced(true), (short) 1), new ApiMessageAndVersion(new RegisterBrokerRecord(). setBrokerId(1).setBrokerEpoch(brokerEpochs.get(1)). setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB1")). - setEndPoints( - new BrokerEndpointCollection( - Arrays.asList( - new BrokerEndpoint().setName("PLAINTEXT").setHost("localhost"). - setPort(9093).setSecurityProtocol((short) 0)).iterator())). + setEndPoints(new BrokerEndpointCollection( + Arrays.asList(new BrokerEndpoint().setName("PLAINTEXT").setHost("localhost"). + setPort(9093).setSecurityProtocol((short) 0)).iterator())). setFeatures(registrationFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_3_IV3)). setRack(null). - setFenced(false), (short) 1), + setFenced(true), (short) 1), new ApiMessageAndVersion(new RegisterBrokerRecord(). setBrokerId(2).setBrokerEpoch(brokerEpochs.get(2)). setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB2")). - setEndPoints( - new BrokerEndpointCollection( - Arrays.asList( - new BrokerEndpoint().setName("PLAINTEXT").setHost("localhost"). - setPort(9094).setSecurityProtocol((short) 0)).iterator())). + setEndPoints(new BrokerEndpointCollection( + Arrays.asList(new BrokerEndpoint().setName("PLAINTEXT").setHost("localhost"). + setPort(9094).setSecurityProtocol((short) 0)).iterator())). setFeatures(registrationFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_3_IV3)). setRack(null). - setFenced(false), (short) 1), + setFenced(true), (short) 1), new ApiMessageAndVersion(new RegisterBrokerRecord(). setBrokerId(3).setBrokerEpoch(brokerEpochs.get(3)). setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB3")). - setEndPoints(new BrokerEndpointCollection(Arrays.asList( - new BrokerEndpoint().setName("PLAINTEXT").setHost("localhost"). + setEndPoints(new BrokerEndpointCollection( + Arrays.asList(new BrokerEndpoint().setName("PLAINTEXT").setHost("localhost"). setPort(9095).setSecurityProtocol((short) 0)).iterator())). setFeatures(registrationFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_3_IV3)). - setRack(null), (short) 1), + setRack(null). + setFenced(true), (short) 1), + new ApiMessageAndVersion(new BrokerRegistrationChangeRecord(). + setBrokerId(0). + setBrokerEpoch(brokerEpochs.get(0)). + setFenced(BrokerRegistrationFencingChange.UNFENCE.value()), (short) 0), + new ApiMessageAndVersion(new BrokerRegistrationChangeRecord(). + setBrokerId(1). + setBrokerEpoch(brokerEpochs.get(1)). + setFenced(BrokerRegistrationFencingChange.UNFENCE.value()), (short) 0), + new ApiMessageAndVersion(new BrokerRegistrationChangeRecord(). + setBrokerId(2). + setBrokerEpoch(brokerEpochs.get(2)). + setFenced(BrokerRegistrationFencingChange.UNFENCE.value()), (short) 0), + new ApiMessageAndVersion(new TopicRecord(). + setName("foo").setTopicId(fooId), (short) 0), + new ApiMessageAndVersion(new PartitionRecord().setPartitionId(0). + setTopicId(fooId).setReplicas(Arrays.asList(0, 1, 2)). + setIsr(Arrays.asList(0, 1, 2)).setRemovingReplicas(Collections.emptyList()). + setAddingReplicas(Collections.emptyList()).setLeader(0).setLeaderEpoch(0). + setPartitionEpoch(0), (short) 0), + new ApiMessageAndVersion(new PartitionRecord().setPartitionId(1). + setTopicId(fooId).setReplicas(Arrays.asList(1, 2, 0)). + setIsr(Arrays.asList(1, 2, 0)).setRemovingReplicas(Collections.emptyList()). + setAddingReplicas(Collections.emptyList()).setLeader(1).setLeaderEpoch(0). + setPartitionEpoch(0), (short) 0), new ApiMessageAndVersion(new ProducerIdsRecord(). setBrokerId(0). setBrokerEpoch(brokerEpochs.get(0)). - setNextProducerId(1000), (short) 0) - ); - } - - private void checkSnapshotContent( - List expected, - Iterator> iterator - ) throws Exception { - RecordTestUtils.assertBatchIteratorContains( - Arrays.asList(expected), - Arrays.asList( - StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false) - .flatMap(batch -> batch.records().stream()) - .collect(Collectors.toList()) - ).iterator() - ); - } - - /** - * This function checks that the iterator is a subset of the expected list. - * - * This is needed because when generating snapshots through configuration is difficult to control exactly when a - * snapshot will be generated and which committed offset will be included in the snapshot. - */ - private void checkSnapshotSubcontent( - List expected, - Iterator> iterator - ) throws Exception { - RecordTestUtils.deepSortRecords(expected); - - List actual = StreamSupport - .stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false) - .flatMap(batch -> batch.records().stream()) - .collect(Collectors.toList()); - - RecordTestUtils.deepSortRecords(actual); - - int expectedIndex = 0; - for (ApiMessageAndVersion current : actual) { - while (expectedIndex < expected.size() && !expected.get(expectedIndex).equals(current)) { - expectedIndex += 1; - } - - if (expectedIndex >= expected.size()) { - fail("Failed to find record " + current + " in the expected record set: " + expected); - } - - expectedIndex += 1; - } + setNextProducerId(1000), (short) 0)); } /** @@ -1341,7 +1067,7 @@ public void testQuorumControllerCompletesAuthorizerInitialLoad() throws Throwabl }). build() ) { - logEnv.appendInitialRecords(expectedSnapshotContent(FOO_ID, ALL_ZERO_BROKER_EPOCHS)); + logEnv.appendInitialRecords(generateTestRecords(FOO_ID, ALL_ZERO_BROKER_EPOCHS)); logEnv.logManagers().forEach(m -> m.setMaxReadOffset(2)); try ( QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv.Builder(logEnv). diff --git a/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java index c81f25e4c000a..0997792cd90f6 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java @@ -58,7 +58,6 @@ import org.apache.kafka.common.metadata.BrokerRegistrationChangeRecord; import org.apache.kafka.common.metadata.ConfigRecord; import org.apache.kafka.common.metadata.PartitionChangeRecord; -import org.apache.kafka.common.metadata.PartitionRecord; import org.apache.kafka.common.metadata.RegisterBrokerRecord; import org.apache.kafka.common.metadata.RemoveTopicRecord; import org.apache.kafka.common.metadata.TopicRecord; @@ -519,16 +518,6 @@ public void testCreateTopics() throws Exception { setErrorCode(Errors.TOPIC_ALREADY_EXISTS.code()). setErrorMessage("Topic 'foo' already exists.")); assertEquals(expectedResponse4, result4.response()); - Uuid fooId = result3.response().topics().find("foo").topicId(); - RecordTestUtils.assertBatchIteratorContains(asList( - asList(new ApiMessageAndVersion(new PartitionRecord(). - setPartitionId(0).setTopicId(fooId). - setReplicas(asList(1, 2, 0)).setIsr(asList(1, 2, 0)). - setRemovingReplicas(Collections.emptyList()).setAddingReplicas(Collections.emptyList()).setLeader(1). - setLeaderEpoch(0).setPartitionEpoch(0), (short) 0), - new ApiMessageAndVersion(new TopicRecord(). - setTopicId(fooId).setName("foo"), (short) 0))), - ctx.replicationControl.iterator(Long.MAX_VALUE)); } @Test diff --git a/metadata/src/test/java/org/apache/kafka/controller/SnapshotGeneratorTest.java b/metadata/src/test/java/org/apache/kafka/controller/SnapshotGeneratorTest.java deleted file mode 100644 index f7fa18f20a4cf..0000000000000 --- a/metadata/src/test/java/org/apache/kafka/controller/SnapshotGeneratorTest.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kafka.controller; - -import org.apache.kafka.common.Uuid; -import org.apache.kafka.common.config.ConfigResource; -import org.apache.kafka.common.memory.MemoryPool; -import org.apache.kafka.common.metadata.ConfigRecord; -import org.apache.kafka.common.metadata.TopicRecord; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.utils.LogContext; -import org.apache.kafka.common.utils.MockTime; -import org.apache.kafka.controller.SnapshotGenerator.Section; -import org.apache.kafka.metadata.MetadataRecordSerde; -import org.apache.kafka.raft.OffsetAndEpoch; -import org.apache.kafka.server.common.ApiMessageAndVersion; -import org.apache.kafka.snapshot.SnapshotWriter; -import org.apache.kafka.snapshot.MockRawSnapshotWriter; -import org.apache.kafka.snapshot.RawSnapshotWriter; -import org.apache.kafka.snapshot.RecordsSnapshotWriter; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; - -import java.util.Arrays; -import java.util.List; -import java.util.OptionalLong; -import java.util.Optional; - -import static org.apache.kafka.raft.KafkaRaftClient.MAX_BATCH_SIZE_BYTES; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - - -@Timeout(40) -public class SnapshotGeneratorTest { - private static final List> BATCHES; - - static { - BATCHES = Arrays.asList( - Arrays.asList(new ApiMessageAndVersion(new TopicRecord(). - setName("foo").setTopicId(Uuid.randomUuid()), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new TopicRecord(). - setName("bar").setTopicId(Uuid.randomUuid()), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new TopicRecord(). - setName("baz").setTopicId(Uuid.randomUuid()), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ConfigRecord(). - setResourceName("foo").setResourceType(ConfigResource.Type.TOPIC.id()). - setName("retention.ms").setValue("10000000"), (short) 0), - new ApiMessageAndVersion(new ConfigRecord(). - setResourceName("foo").setResourceType(ConfigResource.Type.TOPIC.id()). - setName("max.message.bytes").setValue("100000000"), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ConfigRecord(). - setResourceName("bar").setResourceType(ConfigResource.Type.TOPIC.id()). - setName("retention.ms").setValue("5000000"), (short) 0))); - } - - @Test - public void testGenerateBatches() throws Exception { - SnapshotWriter writer = createSnapshotWriter(123, 0); - List
sections = Arrays.asList(new Section("replication", - Arrays.asList(BATCHES.get(0), BATCHES.get(1), BATCHES.get(2)).iterator()), - new Section("configuration", - Arrays.asList(BATCHES.get(3), BATCHES.get(4)).iterator())); - SnapshotGenerator generator = new SnapshotGenerator(new LogContext(), - writer, 2, sections); - assertFalse(writer.isFrozen()); - assertEquals(123L, generator.lastContainedLogOffset()); - assertEquals(writer, generator.writer()); - assertEquals(OptionalLong.of(0L), generator.generateBatches()); - assertEquals(OptionalLong.of(0L), generator.generateBatches()); - assertFalse(writer.isFrozen()); - assertEquals(OptionalLong.empty(), generator.generateBatches()); - assertTrue(writer.isFrozen()); - } - - private SnapshotWriter createSnapshotWriter( - long committedOffset, - long lastContainedLogTime - ) { - return RecordsSnapshotWriter.createWithHeader( - () -> createNewSnapshot(new OffsetAndEpoch(committedOffset + 1, 1)), - MAX_BATCH_SIZE_BYTES, - MemoryPool.NONE, - new MockTime(), - lastContainedLogTime, - CompressionType.NONE, - new MetadataRecordSerde() - ).get(); - } - - private Optional createNewSnapshot( - OffsetAndEpoch snapshotId - ) { - return Optional.of(new MockRawSnapshotWriter(snapshotId, buffer -> { })); - } -} diff --git a/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java b/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java index 8c63cbc516d48..f4707a49890d6 100644 --- a/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java @@ -30,7 +30,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import static org.apache.kafka.server.common.MetadataVersion.IBP_3_3_IV0; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -49,6 +51,10 @@ public class FeaturesImageTest { IMAGE1 = new FeaturesImage(map1, MetadataVersion.latest()); DELTA1_RECORDS = new ArrayList<>(); + DELTA1_RECORDS.add(new ApiMessageAndVersion(new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(MetadataVersion.latest().featureLevel()), + (short) 0)); DELTA1_RECORDS.add(new ApiMessageAndVersion(new FeatureLevelRecord(). setName("foo").setFeatureLevel((short) 3), (short) 0)); @@ -95,4 +101,29 @@ private void testToImageAndBack(FeaturesImage image) throws Throwable { FeaturesImage nextImage = delta.apply(); assertEquals(image, nextImage); } + + private static final FeatureLevelRecord MV_LATEST_RECORD = new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(MetadataVersion.latest().featureLevel()); + + private static final FeatureLevelRecord MV_3_3_IV0_RECORD = new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(IBP_3_3_IV0.featureLevel()); + + @Test + public void testCalculateMetadataVersionChange() throws Throwable { + FeaturesDelta delta = new FeaturesDelta(IMAGE1); + assertEquals(Optional.empty(), delta.calculateMetadataVersionChange(MV_LATEST_RECORD)); + assertEquals(Optional.of(new MetadataVersionChange( + MetadataVersion.latest(), IBP_3_3_IV0)), delta.calculateMetadataVersionChange(MV_3_3_IV0_RECORD)); + } + + @Test + public void testReplayMetadataVersionChangeRecord() throws Throwable { + FeaturesDelta delta = new FeaturesDelta(IMAGE1); + delta.replay(MV_LATEST_RECORD); + assertEquals(MetadataVersion.latest(), delta.apply().metadataVersion()); + delta.replay(MV_3_3_IV0_RECORD); + assertEquals(IBP_3_3_IV0, delta.apply().metadataVersion()); + } } diff --git a/metadata/src/test/java/org/apache/kafka/image/ImageDowngradeTest.java b/metadata/src/test/java/org/apache/kafka/image/ImageDowngradeTest.java index 5a7c2e6c0b605..ce6cfe3da4cf9 100644 --- a/metadata/src/test/java/org/apache/kafka/image/ImageDowngradeTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/ImageDowngradeTest.java @@ -133,9 +133,11 @@ private static void writeWithExpectedLosses( List expectedOutputs ) { MockLossConsumer lossConsumer = new MockLossConsumer(metadataVersion); - MetadataDelta delta = new MetadataDelta(MetadataImage.EMPTY); + MetadataDelta delta = new MetadataDelta.Builder(). + setMetadataVersion(findMetadataVersion(inputs)). + build(); RecordTestUtils.replayAll(delta, inputs); - MetadataImage image = delta.apply(); + MetadataImage image = delta.apply(MetadataProvenance.EMPTY); RecordListWriter writer = new RecordListWriter(); image.write(writer, new ImageWriterOptions.Builder(). setRawMetadataVersion(metadataVersion). @@ -144,4 +146,16 @@ private static void writeWithExpectedLosses( assertEquals(expectedLosses, lossConsumer.losses, "Failed to get expected metadata losses."); assertEquals(expectedOutputs, writer.records(), "Failed to get expected output records."); } + + private static MetadataVersion findMetadataVersion(List records) { + for (ApiMessageAndVersion messageAndVersion : records) { + if (messageAndVersion.message() instanceof FeatureLevelRecord) { + FeatureLevelRecord record = (FeatureLevelRecord) messageAndVersion.message(); + if (record.name().equals(MetadataVersion.FEATURE_NAME)) { + return MetadataVersion.fromFeatureLevel(record.featureLevel()); + } + } + } + return MetadataVersion.MINIMUM_KRAFT_VERSION; + } } diff --git a/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java b/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java index 290214369ccf6..d15e88fb20000 100644 --- a/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java @@ -20,7 +20,6 @@ import org.apache.kafka.image.writer.ImageWriterOptions; import org.apache.kafka.image.writer.RecordListWriter; import org.apache.kafka.metadata.RecordTestUtils; -import org.apache.kafka.raft.OffsetAndEpoch; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -37,7 +36,7 @@ public class MetadataImageTest { static { IMAGE1 = new MetadataImage( - new OffsetAndEpoch(100, 4), + new MetadataProvenance(100, 4, 2000), FeaturesImageTest.IMAGE1, ClusterImageTest.IMAGE1, TopicsImageTest.IMAGE1, @@ -46,17 +45,19 @@ public class MetadataImageTest { ProducerIdsImageTest.IMAGE1, AclsImageTest.IMAGE1); - DELTA1 = new MetadataDelta(IMAGE1); - RecordTestUtils.replayAll(DELTA1, 200, 5, FeaturesImageTest.DELTA1_RECORDS); - RecordTestUtils.replayAll(DELTA1, 200, 5, ClusterImageTest.DELTA1_RECORDS); - RecordTestUtils.replayAll(DELTA1, 200, 5, TopicsImageTest.DELTA1_RECORDS); - RecordTestUtils.replayAll(DELTA1, 200, 5, ConfigurationsImageTest.DELTA1_RECORDS); - RecordTestUtils.replayAll(DELTA1, 200, 5, ClientQuotasImageTest.DELTA1_RECORDS); - RecordTestUtils.replayAll(DELTA1, 200, 5, ProducerIdsImageTest.DELTA1_RECORDS); - RecordTestUtils.replayAll(DELTA1, 200, 5, AclsImageTest.DELTA1_RECORDS); + DELTA1 = new MetadataDelta.Builder(). + setImage(IMAGE1). + build(); + RecordTestUtils.replayAll(DELTA1, FeaturesImageTest.DELTA1_RECORDS); + RecordTestUtils.replayAll(DELTA1, ClusterImageTest.DELTA1_RECORDS); + RecordTestUtils.replayAll(DELTA1, TopicsImageTest.DELTA1_RECORDS); + RecordTestUtils.replayAll(DELTA1, ConfigurationsImageTest.DELTA1_RECORDS); + RecordTestUtils.replayAll(DELTA1, ClientQuotasImageTest.DELTA1_RECORDS); + RecordTestUtils.replayAll(DELTA1, ProducerIdsImageTest.DELTA1_RECORDS); + RecordTestUtils.replayAll(DELTA1, AclsImageTest.DELTA1_RECORDS); IMAGE2 = new MetadataImage( - new OffsetAndEpoch(200, 5), + new MetadataProvenance(200, 5, 4000), FeaturesImageTest.IMAGE2, ClusterImageTest.IMAGE2, TopicsImageTest.IMAGE2, @@ -78,7 +79,7 @@ public void testImage1RoundTrip() throws Throwable { @Test public void testApplyDelta1() throws Throwable { - assertEquals(IMAGE2, DELTA1.apply()); + assertEquals(IMAGE2, DELTA1.apply(IMAGE2.provenance())); } @Test @@ -88,13 +89,14 @@ public void testImage2RoundTrip() throws Throwable { private void testToImageAndBack(MetadataImage image) throws Throwable { RecordListWriter writer = new RecordListWriter(); - image.write(writer, new ImageWriterOptions.Builder().build()); - MetadataDelta delta = new MetadataDelta(MetadataImage.EMPTY); - RecordTestUtils.replayAll(delta, - image.highestOffsetAndEpoch().offset(), - image.highestOffsetAndEpoch().epoch(), - writer.records()); - MetadataImage nextImage = delta.apply(); + image.write(writer, new ImageWriterOptions.Builder(). + setMetadataVersion(image.features().metadataVersion()). + build()); + MetadataDelta delta = new MetadataDelta.Builder(). + setMetadataVersion(image.features().metadataVersion()). + build(); + RecordTestUtils.replayAll(delta, writer.records()); + MetadataImage nextImage = delta.apply(image.provenance()); assertEquals(image, nextImage); } } diff --git a/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotEmitterTest.java b/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotEmitterTest.java new file mode 100644 index 0000000000000..62a1d15c3fa82 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotEmitterTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.publisher; + +import org.apache.kafka.image.MetadataImageTest; +import org.apache.kafka.raft.LeaderAndEpoch; +import org.apache.kafka.raft.OffsetAndEpoch; +import org.apache.kafka.raft.RaftClient; +import org.apache.kafka.server.common.ApiMessageAndVersion; +import org.apache.kafka.snapshot.SnapshotWriter; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.OptionalLong; +import java.util.concurrent.CompletableFuture; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +@Timeout(value = 40) +public class SnapshotEmitterTest { + static class MockRaftClient implements RaftClient { + Map writers = new HashMap<>(); + + @Override + public void initialize() { + // nothing to do + } + + @Override + public void register(Listener listener) { + // nothing to do + } + + @Override + public void unregister(Listener listener) { + // nothing to do + } + + @Override + public OptionalLong highWatermark() { + return OptionalLong.empty(); + } + + @Override + public LeaderAndEpoch leaderAndEpoch() { + return LeaderAndEpoch.UNKNOWN; + } + + @Override + public OptionalInt nodeId() { + return OptionalInt.empty(); + } + + @Override + public long scheduleAppend(int epoch, List records) { + return 0; + } + + @Override + public long scheduleAtomicAppend(int epoch, List records) { + return 0; + } + + @Override + public CompletableFuture shutdown(int timeoutMs) { + return CompletableFuture.completedFuture(null); + } + + @Override + public void resign(int epoch) { + // nothing to do + } + + @Override + public Optional> createSnapshot( + long committedOffset, + int committedEpoch, + long lastContainedLogTime + ) { + OffsetAndEpoch snapshotId = new OffsetAndEpoch(committedOffset, committedEpoch); + if (writers.containsKey(snapshotId)) { + return Optional.empty(); + } + MockSnapshotWriter writer = new MockSnapshotWriter(snapshotId); + writers.put(snapshotId, writer); + return Optional.of(writer); + } + + @Override + public void close() throws Exception { + // nothing to do + } + } + + static class MockSnapshotWriter implements SnapshotWriter { + private final OffsetAndEpoch snapshotId; + private boolean frozen = false; + private boolean closed = false; + private final List> batches; + + MockSnapshotWriter(OffsetAndEpoch snapshotId) { + this.snapshotId = snapshotId; + this.batches = new ArrayList<>(); + } + + @Override + public OffsetAndEpoch snapshotId() { + return snapshotId; + } + + @Override + public long lastContainedLogOffset() { + return snapshotId.offset(); + } + + @Override + public int lastContainedLogEpoch() { + return snapshotId.epoch(); + } + + @Override + public boolean isFrozen() { + return frozen; + } + + @Override + public void append(List records) { + batches.add(records); + } + + List> batches() { + List> results = new ArrayList<>(); + batches.forEach(batch -> results.add(new ArrayList<>(batch))); + return results; + } + + @Override + public void freeze() { + frozen = true; + } + + @Override + public void close() { + closed = true; + } + + boolean isClosed() { + return closed; + } + } + + @Test + public void testEmit() throws Exception { + MockRaftClient mockRaftClient = new MockRaftClient(); + SnapshotEmitter emitter = new SnapshotEmitter.Builder(). + setBatchSize(2). + setRaftClient(mockRaftClient). + build(); + emitter.emit(MetadataImageTest.IMAGE1); + MockSnapshotWriter writer = mockRaftClient.writers.get( + MetadataImageTest.IMAGE1.highestOffsetAndEpoch()); + assertNotNull(writer); + assertEquals(MetadataImageTest.IMAGE1.highestOffsetAndEpoch().offset(), + writer.lastContainedLogOffset()); + assertEquals(MetadataImageTest.IMAGE1.highestOffsetAndEpoch().epoch(), + writer.lastContainedLogEpoch()); + assertTrue(writer.isFrozen()); + assertTrue(writer.isClosed()); + + // Second call to emit does nothing because we already have a snapshot at that offset and epoch. + emitter.emit(MetadataImageTest.IMAGE1); + assertEquals(1, mockRaftClient.writers.size()); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotGeneratorTest.java b/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotGeneratorTest.java new file mode 100644 index 0000000000000..8f022004ac7cb --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotGeneratorTest.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.publisher; + +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.image.MetadataDelta; +import org.apache.kafka.image.MetadataImage; +import org.apache.kafka.image.MetadataProvenance; +import org.apache.kafka.image.MetadataVersionChange; +import org.apache.kafka.image.loader.LogDeltaManifest; +import org.apache.kafka.image.loader.PreVersionChangeManifest; +import org.apache.kafka.metadata.RecordTestUtils; +import org.apache.kafka.server.fault.FaultHandlerException; +import org.apache.kafka.server.fault.MockFaultHandler; +import org.apache.kafka.test.TestUtils; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.apache.kafka.server.common.MetadataVersion.IBP_3_3_IV2; +import static org.apache.kafka.server.common.MetadataVersion.IBP_3_3_IV3; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +@Timeout(value = 40) +public class SnapshotGeneratorTest { + static class MockEmitter implements SnapshotGenerator.Emitter { + private final CountDownLatch latch = new CountDownLatch(1); + private final List images = new ArrayList<>(); + private RuntimeException problem = null; + + MockEmitter setReady() { + latch.countDown(); + return this; + } + + synchronized MockEmitter setProblem(RuntimeException problem) { + this.problem = problem; + return this; + } + + @Override + public synchronized void emit(MetadataImage image) { + RuntimeException currentProblem = problem; + if (currentProblem != null) { + throw currentProblem; + } + try { + latch.await(); + } catch (Throwable e) { + throw new RuntimeException(e); + } + images.add(image); + } + + synchronized List images() { + return new ArrayList<>(images); + } + } + + private final static MetadataDelta TEST_DELTA; + + static { + TEST_DELTA = new MetadataDelta.Builder(). + setImage(MetadataImage.EMPTY). + setMetadataVersion(IBP_3_3_IV2). + build(); + TEST_DELTA.replay(RecordTestUtils.testRecord(0).message()); + } + + private final static MetadataImage TEST_IMAGE = TEST_DELTA.apply(MetadataProvenance.EMPTY); + + @Test + public void testCreateSnapshot() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("SnapshotGenerator"); + MockEmitter emitter = new MockEmitter(); + try (SnapshotGenerator generator = new SnapshotGenerator.Builder(). + setEmitter(emitter). + setFaultHandler(faultHandler). + setMinBytesSinceLastSnapshot(200). + setMinTimeSinceLastSnapshotNs(TimeUnit.DAYS.toNanos(10)). + build()) { + // Publish a log delta batch. This one will not trigger a snapshot yet. + generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, + new LogDeltaManifest(MetadataProvenance.EMPTY, 1, 100, 100)); + // Publish a log delta batch. This will trigger a snapshot. + generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, + new LogDeltaManifest(MetadataProvenance.EMPTY, 1, 100, 100)); + // Pre-version change images are always queued for writing, even if there is another + // image in progress. + for (int i = 0; i < 2; i++) { + generator.publishPreVersionChangeImage(TEST_DELTA, TEST_IMAGE, + new PreVersionChangeManifest(MetadataProvenance.EMPTY, + new MetadataVersionChange(IBP_3_3_IV2, IBP_3_3_IV3))); + } + // Publish a log delta batch. This one will be ignored because there are other images + // queued for writing. + generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, + new LogDeltaManifest(MetadataProvenance.EMPTY, 1, 100, 2000)); + assertEquals(Collections.emptyList(), emitter.images()); + emitter.setReady(); + } + assertEquals(Arrays.asList(TEST_IMAGE, TEST_IMAGE, TEST_IMAGE), emitter.images()); + faultHandler.maybeRethrowFirstException(); + } + + @Test + public void testSnapshotsDisabled() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("SnapshotGenerator"); + MockEmitter emitter = new MockEmitter().setReady(); + try (SnapshotGenerator generator = new SnapshotGenerator.Builder(). + setEmitter(emitter). + setFaultHandler(faultHandler). + setMinBytesSinceLastSnapshot(1). + setMinTimeSinceLastSnapshotNs(TimeUnit.DAYS.toNanos(10)). + build()) { + generator.disable("we are testing disable()"); + // No snapshots are generated because snapshots are disabled. + generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, + new LogDeltaManifest(MetadataProvenance.EMPTY, 1, 100, 100)); + generator.publishPreVersionChangeImage(TEST_DELTA, TEST_IMAGE, + new PreVersionChangeManifest(MetadataProvenance.EMPTY, + new MetadataVersionChange(IBP_3_3_IV2, IBP_3_3_IV3))); + } + assertEquals(Collections.emptyList(), emitter.images()); + faultHandler.maybeRethrowFirstException(); + } + + @Test + public void testTimeBasedSnapshots() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("SnapshotGenerator"); + MockEmitter emitter = new MockEmitter().setReady(); + MockTime mockTime = new MockTime(); + try (SnapshotGenerator generator = new SnapshotGenerator.Builder(). + setTime(mockTime). + setEmitter(emitter). + setFaultHandler(faultHandler). + setMinBytesSinceLastSnapshot(200). + setMinTimeSinceLastSnapshotNs(TimeUnit.MINUTES.toNanos(30)). + build()) { + // This image isn't published yet. + generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, + new LogDeltaManifest(MetadataProvenance.EMPTY, 1, 100, 50)); + assertEquals(Collections.emptyList(), emitter.images()); + mockTime.sleep(TimeUnit.MINUTES.toNanos(40)); + // Next image is published because of the time delay. + generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, + new LogDeltaManifest(MetadataProvenance.EMPTY, 1, 100, 50)); + TestUtils.waitForCondition(() -> emitter.images().size() == 1, "images.size == 1"); + // bytesSinceLastSnapshot was reset to 0 by the previous snapshot, + // so this does not trigger a new snapshot. + generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, + new LogDeltaManifest(MetadataProvenance.EMPTY, 1, 100, 150)); + } + assertEquals(Arrays.asList(TEST_IMAGE), emitter.images()); + faultHandler.maybeRethrowFirstException(); + } + + @Test + public void testEmitterProblem() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("SnapshotGenerator"); + MockEmitter emitter = new MockEmitter().setProblem(new RuntimeException("oops")); + try (SnapshotGenerator generator = new SnapshotGenerator.Builder(). + setEmitter(emitter). + setFaultHandler(faultHandler). + setMinBytesSinceLastSnapshot(200). + build()) { + for (int i = 0; i < 2; i++) { + generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, + new LogDeltaManifest(MetadataProvenance.EMPTY, 1, 10000, 50000)); + } + } + assertEquals(Collections.emptyList(), emitter.images()); + assertNotNull(faultHandler.firstException()); + assertEquals(FaultHandlerException.class, faultHandler.firstException().getClass()); + assertEquals("SnapshotGenerator: KRaft snapshot file generation error: oops", + faultHandler.firstException().getMessage()); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/image/writer/ImageReWriterTest.java b/metadata/src/test/java/org/apache/kafka/image/writer/ImageReWriterTest.java new file mode 100644 index 0000000000000..640924fe076be --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/image/writer/ImageReWriterTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.writer; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.metadata.TopicRecord; +import org.apache.kafka.image.MetadataDelta; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import static org.apache.kafka.metadata.RecordTestUtils.testRecord; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + + +@Timeout(value = 40) +public class ImageReWriterTest { + @Test + public void testWrite() { + MetadataDelta delta = new MetadataDelta.Builder().build(); + ImageReWriter writer = new ImageReWriter(delta); + writer.write(testRecord(0)); + writer.write(testRecord(1)); + writer.close(true); + assertEquals(2, delta.getOrCreateTopicsDelta().changedTopics().size()); + assertEquals(2, writer.image().topics().topicsById().size()); + } + + @Test + public void testCloseWithoutFreeze() { + MetadataDelta delta = new MetadataDelta.Builder().build(); + ImageReWriter writer = new ImageReWriter(delta); + writer.close(); + assertNull(writer.image()); + } + + @Test + public void testWriteAfterClose() { + MetadataDelta delta = new MetadataDelta.Builder().build(); + ImageReWriter writer = new ImageReWriter(delta); + writer.close(true); + assertThrows(ImageWriterClosedException.class, () -> + writer.write(0, new TopicRecord(). + setName("foo"). + setTopicId(Uuid.fromString("3B134hrsQgKtz8Sp6QBIfg")))); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/image/writer/ImageWriterOptionsTest.java b/metadata/src/test/java/org/apache/kafka/image/writer/ImageWriterOptionsTest.java new file mode 100644 index 0000000000000..07ddd4ff8cbba --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/image/writer/ImageWriterOptionsTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.writer; + +import org.apache.kafka.server.common.MetadataVersion; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + + +@Timeout(value = 40) +public class ImageWriterOptionsTest { + @Test + public void testDefaultLossHandler() { + ImageWriterOptions options = new ImageWriterOptions.Builder().build(); + assertEquals("stuff", assertThrows(UnwritableMetadataException.class, + () -> options.handleLoss("stuff")).loss()); + } + + @Test + public void testSetMetadataVersion() { + for (int i = MetadataVersion.MINIMUM_KRAFT_VERSION.ordinal(); + i < MetadataVersion.VERSIONS.length; + i++) { + MetadataVersion version = MetadataVersion.VERSIONS[i]; + ImageWriterOptions.Builder options = new ImageWriterOptions.Builder(). + setMetadataVersion(version); + if (i < MetadataVersion.MINIMUM_BOOTSTRAP_VERSION.ordinal()) { + assertEquals(MetadataVersion.MINIMUM_KRAFT_VERSION, options.metadataVersion()); + } else { + assertEquals(version, options.metadataVersion()); + } + } + } +} diff --git a/metadata/src/test/java/org/apache/kafka/metadata/RecordTestUtils.java b/metadata/src/test/java/org/apache/kafka/metadata/RecordTestUtils.java index 7fd2a29d5b13c..04da77b0fc327 100644 --- a/metadata/src/test/java/org/apache/kafka/metadata/RecordTestUtils.java +++ b/metadata/src/test/java/org/apache/kafka/metadata/RecordTestUtils.java @@ -23,7 +23,6 @@ import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.ObjectSerializationCache; import org.apache.kafka.common.utils.ImplicitLinkedHashCollection; -import org.apache.kafka.image.MetadataDelta; import org.apache.kafka.raft.Batch; import org.apache.kafka.raft.BatchReader; import org.apache.kafka.raft.internals.MemoryBatchReader; @@ -57,14 +56,6 @@ public class RecordTestUtils { */ public static void replayAll(Object target, List recordsAndVersions) { - if (target instanceof MetadataDelta) { - MetadataDelta delta = (MetadataDelta) target; - replayAll(delta, - delta.image().highestOffsetAndEpoch().offset(), - delta.image().highestOffsetAndEpoch().epoch(), - recordsAndVersions); - return; - } for (ApiMessageAndVersion recordAndVersion : recordsAndVersions) { ApiMessage record = recordAndVersion.message(); try { @@ -96,26 +87,6 @@ public static void replayAll(Object target, } } - /** - * Replay a list of records to the metadata delta. - * - * @param delta the metadata delta on which to replay the records - * @param highestOffset highest offset from the list of records - * @param highestEpoch highest epoch from the list of records - * @param recordsAndVersions list of records - */ - public static void replayAll( - MetadataDelta delta, - long highestOffset, - int highestEpoch, - List recordsAndVersions - ) { - for (ApiMessageAndVersion recordAndVersion : recordsAndVersions) { - ApiMessage record = recordAndVersion.message(); - delta.replay(highestOffset, highestEpoch, record); - } - } - /** * Replay a list of record batches. * @@ -129,25 +100,6 @@ public static void replayAllBatches(Object target, } } - /** - * Replay a list of record batches to the metadata delta. - * - * @param delta the metadata delta on which to replay the records - * @param highestOffset highest offset from the list of record batches - * @param highestEpoch highest epoch from the list of record batches - * @param batches list of batches of records - */ - public static void replayAllBatches( - MetadataDelta delta, - long highestOffset, - int highestEpoch, - List> batches - ) { - for (List batch : batches) { - replayAll(delta, highestOffset, highestEpoch, batch); - } - } - /** * Materialize the output of an iterator into a set. * diff --git a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java index 6822ea72eb9d6..0edce4d818552 100644 --- a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java +++ b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java @@ -46,6 +46,7 @@ import org.slf4j.LoggerFactory; import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.IdentityHashMap; @@ -318,9 +319,9 @@ synchronized void addSnapshot(RawSnapshotReader newSnapshot) { } /** - * Returns the snapshot whos last offset is the committed offset. + * Returns the snapshot whose last offset is the committed offset. * - * If such snapshot doesn't exists, it waits until it does. + * If such snapshot doesn't exist, it waits until it does. */ synchronized RawSnapshotReader waitForSnapshot(long committedOffset) throws InterruptedException { while (true) { @@ -371,6 +372,20 @@ public SharedLogData setInitialMaxReadOffset(long initialMaxReadOffset) { public long initialMaxReadOffset() { return initialMaxReadOffset; } + + /** + * Return all records in the log as a list. + */ + public synchronized List allRecords() { + List allRecords = new ArrayList<>(); + for (LocalBatch batch : batches.values()) { + if (batch instanceof LocalRecordBatch) { + LocalRecordBatch recordBatch = (LocalRecordBatch) batch; + allRecords.addAll(recordBatch.records); + } + } + return allRecords; + } } private static class MetaLogListenerData { diff --git a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManagerTestEnv.java b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManagerTestEnv.java index 1693b62be1aec..d72b7557b48e0 100644 --- a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManagerTestEnv.java +++ b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManagerTestEnv.java @@ -140,6 +140,13 @@ private LocalLogManagerTestEnv( this.logManagers = newLogManagers; } + /** + * Return all records in the log as a list. + */ + public List allRecords() { + return shared.allRecords(); + } + /** * Append some records to the log. This method is meant to be called before the * controllers are started, to simulate a pre-existing metadata log. diff --git a/server-common/src/main/java/org/apache/kafka/queue/EventQueue.java b/server-common/src/main/java/org/apache/kafka/queue/EventQueue.java index c282917bb6ec0..10165bb89f6ab 100644 --- a/server-common/src/main/java/org/apache/kafka/queue/EventQueue.java +++ b/server-common/src/main/java/org/apache/kafka/queue/EventQueue.java @@ -248,6 +248,11 @@ default void beginShutdown(String source, Event cleanupEvent) { */ void beginShutdown(String source, Event cleanupEvent, long timeSpan, TimeUnit timeUnit); + /** + * @return True only if there are no events scheduled in the queue. + */ + boolean isEmpty(); + /** * This method is used during unit tests where MockTime is in use. * It is used to alert the queue that the mock time has changed. diff --git a/server-common/src/main/java/org/apache/kafka/queue/KafkaEventQueue.java b/server-common/src/main/java/org/apache/kafka/queue/KafkaEventQueue.java index 05642f6bac8b5..39ae8bbe350bf 100644 --- a/server-common/src/main/java/org/apache/kafka/queue/KafkaEventQueue.java +++ b/server-common/src/main/java/org/apache/kafka/queue/KafkaEventQueue.java @@ -340,6 +340,17 @@ void wakeUp() { lock.unlock(); } } + + boolean isEmpty() { + lock.lock(); + try { + if (head != head.next) return false; + if (!deadlineMap.isEmpty()) return false; + return true; + } finally { + lock.unlock(); + } + } } private final Time time; @@ -412,6 +423,11 @@ public void beginShutdown(String source, Event newCleanupEvent, } } + @Override + public boolean isEmpty() { + return eventHandler.isEmpty(); + } + @Override public void wakeup() { eventHandler.wakeUp(); diff --git a/server-common/src/main/java/org/apache/kafka/server/fault/ProcessExitingFaultHandler.java b/server-common/src/main/java/org/apache/kafka/server/fault/ProcessExitingFaultHandler.java index b7c0d241a2ad3..fd72a8b1c0cda 100644 --- a/server-common/src/main/java/org/apache/kafka/server/fault/ProcessExitingFaultHandler.java +++ b/server-common/src/main/java/org/apache/kafka/server/fault/ProcessExitingFaultHandler.java @@ -28,6 +28,15 @@ */ public class ProcessExitingFaultHandler implements FaultHandler { private static final Logger log = LoggerFactory.getLogger(ProcessExitingFaultHandler.class); + private final Runnable action; + + public ProcessExitingFaultHandler() { + this.action = () -> { }; + } + + public ProcessExitingFaultHandler(Runnable action) { + this.action = action; + } @Override public RuntimeException handleFault(String failureMessage, Throwable cause) { @@ -36,6 +45,11 @@ public RuntimeException handleFault(String failureMessage, Throwable cause) { } else { log.error("Encountered fatal fault: {}", failureMessage, cause); } + try { + action.run(); + } catch (Throwable e) { + log.error("Failed to run ProcessExitingFaultHandler action.", e); + } Exit.exit(1); return null; } diff --git a/server-common/src/test/java/org/apache/kafka/queue/KafkaEventQueueTest.java b/server-common/src/test/java/org/apache/kafka/queue/KafkaEventQueueTest.java index c3ee62a6ac26d..8dcc02191633d 100644 --- a/server-common/src/test/java/org/apache/kafka/queue/KafkaEventQueueTest.java +++ b/server-common/src/test/java/org/apache/kafka/queue/KafkaEventQueueTest.java @@ -31,12 +31,14 @@ import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.test.TestUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; @Timeout(value = 60) @@ -240,4 +242,29 @@ public void handleException(Throwable e) { assertEquals(RejectedExecutionException.class, assertThrows( ExecutionException.class, () -> future.get()).getCause().getClass()); } -} + + @Test + public void testEmpty() throws Exception { + KafkaEventQueue queue = new KafkaEventQueue(Time.SYSTEM, new LogContext(), + "testEmpty"); + assertTrue(queue.isEmpty()); + CompletableFuture future = new CompletableFuture<>(); + queue.append(() -> future.get()); + assertFalse(queue.isEmpty()); + future.complete(null); + TestUtils.waitForCondition(() -> queue.isEmpty(), "Failed to see the queue become empty."); + queue.scheduleDeferred("later", + __ -> OptionalLong.of(Time.SYSTEM.nanoseconds() + TimeUnit.HOURS.toNanos(1)), + () -> { }); + assertFalse(queue.isEmpty()); + queue.scheduleDeferred("soon", + __ -> OptionalLong.of(Time.SYSTEM.nanoseconds() + TimeUnit.MILLISECONDS.toNanos(1)), + () -> { }); + assertFalse(queue.isEmpty()); + queue.cancelDeferred("later"); + queue.cancelDeferred("soon"); + assertTrue(queue.isEmpty()); + queue.close(); + assertTrue(queue.isEmpty()); + } +} \ No newline at end of file From 831d4fecfcdadaa2bcd65b72dc63923de776d1c7 Mon Sep 17 00:00:00 2001 From: "Colin P. McCabe" Date: Tue, 1 Nov 2022 20:36:58 -1000 Subject: [PATCH 02/15] some fixes --- .../scala/kafka/server/BrokerServer.scala | 8 +- .../scala/kafka/server/ControllerServer.scala | 23 +- .../kafka/server/DynamicBrokerConfig.scala | 4 +- .../metadata/BrokerMetadataPublisher.scala | 65 +-- .../ControllerMetadataPublisher.scala | 126 ------ .../metadata/DynamicConfigPublisher.scala | 125 ++++++ .../kafka/testkit/KafkaClusterTestKit.java | 2 + .../kafka/server/QuorumTestHarness.scala | 2 + .../server/DynamicConfigChangeTest.scala | 3 +- .../BrokerMetadataPublisherTest.scala | 28 +- .../kafka/image/MetadataProvenance.java | 6 + .../kafka/image/loader/MetadataLoader.java | 37 +- .../image/loader/MetadataLoaderTest.java | 425 ++++++++++++++++++ 13 files changed, 631 insertions(+), 223 deletions(-) delete mode 100644 core/src/main/scala/kafka/server/metadata/ControllerMetadataPublisher.scala create mode 100644 core/src/main/scala/kafka/server/metadata/DynamicConfigPublisher.scala create mode 100644 metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java diff --git a/core/src/main/scala/kafka/server/BrokerServer.scala b/core/src/main/scala/kafka/server/BrokerServer.scala index 59eccdf447a73..34845f749e951 100644 --- a/core/src/main/scala/kafka/server/BrokerServer.scala +++ b/core/src/main/scala/kafka/server/BrokerServer.scala @@ -29,7 +29,7 @@ import kafka.log.LogManager import kafka.network.{DataPlaneAcceptor, SocketServer} import kafka.raft.RaftManager import kafka.security.CredentialProvider -import kafka.server.metadata.{BrokerMetadataPublisher, ClientQuotaMetadataManager, KRaftMetadataCache} +import kafka.server.metadata.{BrokerMetadataPublisher, ClientQuotaMetadataManager, DynamicConfigPublisher, KRaftMetadataCache} import kafka.utils.{CoreUtils, KafkaScheduler} import org.apache.kafka.common.feature.SupportedVersionRange import org.apache.kafka.common.message.ApiMessageType.ListenerType @@ -388,6 +388,7 @@ class BrokerServer( DataPlaneAcceptor.ThreadPrefix) // Block until we've caught up with the latest metadata from the controller quorum. + info("Waiting for broker metadata to catch up.") try { lifecycleManager.initialCatchUpFuture.get() } catch { @@ -403,7 +404,10 @@ class BrokerServer( groupCoordinator, transactionCoordinator, clientQuotaMetadataManager, - dynamicConfigHandlers.toMap, + new DynamicConfigPublisher( + config, + metadataPublishingFaultHandler, + dynamicConfigHandlers.toMap), authorizer, fatalFaultHandler, metadataPublishingFaultHandler) diff --git a/core/src/main/scala/kafka/server/ControllerServer.scala b/core/src/main/scala/kafka/server/ControllerServer.scala index f221640495ece..99007421b4b89 100644 --- a/core/src/main/scala/kafka/server/ControllerServer.scala +++ b/core/src/main/scala/kafka/server/ControllerServer.scala @@ -29,7 +29,7 @@ import kafka.security.CredentialProvider import kafka.server.KafkaConfig.{AlterConfigPolicyClassNameProp, CreateTopicPolicyClassNameProp} import kafka.server.KafkaRaftServer.BrokerRole import kafka.server.QuotaFactory.QuotaManagers -import kafka.server.metadata.ControllerMetadataPublisher +import kafka.server.metadata.DynamicConfigPublisher import kafka.utils.{CoreUtils, Logging} import org.apache.kafka.clients.ApiVersions import org.apache.kafka.common.message.ApiMessageType.ListenerType @@ -95,8 +95,7 @@ class ControllerServer( var quotaManagers: QuotaManagers = _ var controllerApis: ControllerApis = _ var controllerApisHandlerPool: KafkaRequestHandlerPool = _ - var dynamicConfigHandlers: immutable.Map[String, ConfigHandler] = _ - var controllerMetadataPublisher: ControllerMetadataPublisher = _ + var dynamicConfigPublisher: DynamicConfigPublisher = _ def kafkaYammerMetrics: KafkaYammerMetrics = KafkaYammerMetrics.INSTANCE private def maybeChangeStatus(from: ProcessStatus, to: ProcessStatus): Boolean = { @@ -265,13 +264,17 @@ class ControllerServer( throw e } config.dynamicConfig.addReconfigurables(this) - dynamicConfigHandlers = immutable.Map[String, ConfigHandler]( - ConfigType.Broker -> new BrokerConfigHandler(config, quotaManagers)) - controllerMetadataPublisher = new ControllerMetadataPublisher( - config, - dynamicConfigHandlers, - fatalFaultHandler) - metadataLoader.installPublishers(List().asJava) + if (!config.processRoles.contains(BrokerRole)) { + // In standalone mode, install a DynamicConfigPublisher. + // In combined mode, we can rely on the broker's DynamicConfigPublisher. + val dynamicConfigHandlers = immutable.Map[String, ConfigHandler]( + ConfigType.Broker -> new BrokerConfigHandler(config, quotaManagers)) + dynamicConfigPublisher = new DynamicConfigPublisher( + config, + fatalFaultHandler, + dynamicConfigHandlers) + metadataLoader.installPublishers(List(dynamicConfigPublisher).asJava) + } } def shutdown(): Unit = { diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala index 30f7db1b1146a..e99d72cf2f8cb 100755 --- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala +++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala @@ -279,8 +279,8 @@ class DynamicBrokerConfig(private val kafkaConfig: KafkaConfig) extends Logging case _ => } if (!kafkaConfig.processRoles.contains(BrokerRole)) { - // In combined mode, these elements are shared between broker and controller, and therefore - // don't need to be updated here (since the broker will do it.) + // In standalone mode, add these reconfigurables. + // In combined mode, we can rely on the broker to add them. addReconfigurable(controller.kafkaYammerMetrics) addReconfigurable(new DynamicMetricsReporters( kafkaConfig.brokerId, controller.config, controller.metrics, controller.clusterId)) diff --git a/core/src/main/scala/kafka/server/metadata/BrokerMetadataPublisher.scala b/core/src/main/scala/kafka/server/metadata/BrokerMetadataPublisher.scala index 70432fdc1cc14..c98e007f01d4a 100644 --- a/core/src/main/scala/kafka/server/metadata/BrokerMetadataPublisher.scala +++ b/core/src/main/scala/kafka/server/metadata/BrokerMetadataPublisher.scala @@ -21,11 +21,9 @@ import java.util.Properties import kafka.coordinator.group.GroupCoordinator import kafka.coordinator.transaction.TransactionCoordinator import kafka.log.{LogManager, UnifiedLog} -import kafka.server.ConfigAdminManager.toLoggableProps -import kafka.server.{ConfigEntityName, ConfigHandler, ConfigType, KafkaConfig, ReplicaManager, RequestLocal} +import kafka.server.{KafkaConfig, ReplicaManager, RequestLocal} import kafka.utils.Logging import org.apache.kafka.common.TopicPartition -import org.apache.kafka.common.config.ConfigResource.Type.{BROKER, TOPIC} import org.apache.kafka.common.internals.Topic import org.apache.kafka.image.loader.{LogDeltaManifest, SnapshotManifest} import org.apache.kafka.image.publisher.MetadataPublisher @@ -104,7 +102,7 @@ class BrokerMetadataPublisher( groupCoordinator: GroupCoordinator, txnCoordinator: TransactionCoordinator, clientQuotaMetadataManager: ClientQuotaMetadataManager, - dynamicConfigHandlers: Map[String, ConfigHandler], + var dynamicConfigPublisher: DynamicConfigPublisher, private val _authorizer: Option[Authorizer], fatalFaultHandler: FaultHandler, metadataPublishingFaultHandler: FaultHandler @@ -226,61 +224,10 @@ class BrokerMetadataPublisher( } // Apply configuration deltas. - Option(delta.configsDelta()).foreach { configsDelta => - configsDelta.changes().keySet().forEach { resource => - val props = newImage.configs().configProperties(resource) - resource.`type`() match { - case TOPIC => - try { - // Apply changes to a topic's dynamic configuration. - info(s"Updating topic ${resource.name()} with new configuration : " + - toLoggableProps(resource, props).mkString(",")) - dynamicConfigHandlers(ConfigType.Topic). - processConfigChanges(resource.name(), props) - } catch { - case t: Throwable => metadataPublishingFaultHandler.handleFault("Error updating topic " + - s"${resource.name()} with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + - s"in ${deltaName}", t) - } - case BROKER => - if (resource.name().isEmpty) { - try { - // Apply changes to "cluster configs" (also known as default BROKER configs). - // These are stored in KRaft with an empty name field. - info("Updating cluster configuration : " + - toLoggableProps(resource, props).mkString(",")) - dynamicConfigHandlers(ConfigType.Broker). - processConfigChanges(ConfigEntityName.Default, props) - } catch { - case t: Throwable => metadataPublishingFaultHandler.handleFault("Error updating " + - s"cluster with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + - s"in ${deltaName}", t) - } - } else if (resource.name() == brokerId.toString) { - try { - // Apply changes to this broker's dynamic configuration. - info(s"Updating broker $brokerId with new configuration : " + - toLoggableProps(resource, props).mkString(",")) - dynamicConfigHandlers(ConfigType.Broker). - processConfigChanges(resource.name(), props) - // When applying a per broker config (not a cluster config), we also - // reload any associated file. For example, if the ssl.keystore is still - // set to /tmp/foo, we still want to reload /tmp/foo in case its contents - // have changed. This doesn't apply to topic configs or cluster configs. - reloadUpdatedFilesWithoutConfigChange(props) - } catch { - case t: Throwable => metadataPublishingFaultHandler.handleFault("Error updating " + - s"broker with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + - s"in ${deltaName}", t) - } - } - case _ => // nothing to do - } - } - } + dynamicConfigPublisher.publish(delta, newImage) + // Apply client quotas delta. try { - // Apply client quotas delta. Option(delta.clientQuotasDelta()).foreach { clientQuotasDelta => clientQuotaMetadataManager.update(clientQuotasDelta) } @@ -442,9 +389,9 @@ class BrokerMetadataPublisher( case t: Throwable => metadataPublishingFaultHandler.handleFault("Error starting high " + "watermark checkpoint thread during startup", t) } -} + } override def close(): Unit = { - // nothing to do + dynamicConfigPublisher.close() } } diff --git a/core/src/main/scala/kafka/server/metadata/ControllerMetadataPublisher.scala b/core/src/main/scala/kafka/server/metadata/ControllerMetadataPublisher.scala deleted file mode 100644 index 17134ceb8ebc6..0000000000000 --- a/core/src/main/scala/kafka/server/metadata/ControllerMetadataPublisher.scala +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.server.metadata - -import kafka.server.ConfigAdminManager.toLoggableProps -import kafka.server.{ConfigEntityName, ConfigHandler, ConfigType, KafkaConfig} -import kafka.utils.Logging -import org.apache.kafka.common.config.ConfigResource.Type.BROKER -import org.apache.kafka.image.loader.{LogDeltaManifest, SnapshotManifest} -import org.apache.kafka.image.publisher.MetadataPublisher -import org.apache.kafka.image.{MetadataDelta, MetadataImage} -import org.apache.kafka.server.fault.FaultHandler - - -class ControllerMetadataPublisher( - conf: KafkaConfig, - dynamicConfigHandlers: Map[String, ConfigHandler], - fatalFaultHandler: FaultHandler, -) extends MetadataPublisher with Logging { - logIdent = s"[ControllerMetadataPublisher id=${conf.nodeId}] " - var firstPublish = true - - val nodeId: Int = conf.nodeId - - override def name(): String = "ControllerMetadataPublisher" - - /** - * Publish a new cluster metadata snapshot that we loaded. - * - * @param delta The delta between the previous state and the new one. - * @param newImage The complete new state. - * @param manifest The contents of what was published. - */ - override def publishSnapshot( - delta: MetadataDelta, - newImage: MetadataImage, - manifest: SnapshotManifest - ): Unit = { - publish(delta, newImage) - } - - /** - * Publish a change to the cluster metadata. - * - * @param delta The delta between the previous state and the new one. - * @param newImage The complete new state. - * @param manifest The contents of what was published. - */ - override def publishLogDelta( - delta: MetadataDelta, - newImage: MetadataImage, - manifest: LogDeltaManifest - ): Unit = { - publish(delta, newImage) - } - - def publish(delta: MetadataDelta, newImage: MetadataImage): Unit = { - val deltaName = if (firstPublish) { - s"initial MetadataDelta up to ${newImage.highestOffsetAndEpoch().offset}" - } else { - s"MetadataDelta up to ${newImage.highestOffsetAndEpoch().offset}" - } - firstPublish = false - - // Apply configuration deltas. - Option(delta.configsDelta()).foreach { configsDelta => - configsDelta.changes().keySet().forEach { resource => - val props = newImage.configs().configProperties(resource) - resource.`type`() match { - case BROKER => - if (resource.name().isEmpty) { - try { - // Apply changes to "cluster configs" (also known as default BROKER configs). - // These are stored in KRaft with an empty name field. - info("Updating cluster configuration : " + - toLoggableProps(resource, props).mkString(",")) - dynamicConfigHandlers(ConfigType.Broker). - processConfigChanges(ConfigEntityName.Default, props) - } catch { - case t: Throwable => fatalFaultHandler.handleFault("Error updating " + - s"cluster with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + - s"in ${deltaName}", t) - } - } else if (resource.name() == nodeId.toString) { - try { - // Apply changes to this broker's dynamic configuration. - info(s"Updating controller $nodeId with new configuration : " + - toLoggableProps(resource, props).mkString(",")) - dynamicConfigHandlers(ConfigType.Broker). - processConfigChanges(resource.name(), props) - // When applying a per broker config (not a cluster config), we also - // reload any associated file. For example, if the ssl.keystore is still - // set to /tmp/foo, we still want to reload /tmp/foo in case its contents - // have changed. This doesn't apply to topic configs or cluster configs. - conf.dynamicConfig.reloadUpdatedFilesWithoutConfigChange(props) - } catch { - case t: Throwable => fatalFaultHandler.handleFault("Error updating the " + - s"controller with a new configuration: ${toLoggableProps(resource, props).mkString(",")} " + - s"in ${deltaName}", t) - } - } - case _ => // nothing to do - } - } - } - } - - override def close(): Unit = { - // nothing to do - } -} diff --git a/core/src/main/scala/kafka/server/metadata/DynamicConfigPublisher.scala b/core/src/main/scala/kafka/server/metadata/DynamicConfigPublisher.scala new file mode 100644 index 0000000000000..723ddbd37d246 --- /dev/null +++ b/core/src/main/scala/kafka/server/metadata/DynamicConfigPublisher.scala @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.server.metadata + +import java.util.Properties +import kafka.server.ConfigAdminManager.toLoggableProps +import kafka.server.{ConfigEntityName, ConfigHandler, ConfigType, KafkaConfig} +import kafka.utils.Logging +import org.apache.kafka.common.config.ConfigResource.Type.{BROKER, TOPIC} +import org.apache.kafka.image.loader.{LogDeltaManifest, SnapshotManifest} +import org.apache.kafka.image.publisher.MetadataPublisher +import org.apache.kafka.image.{MetadataDelta, MetadataImage} +import org.apache.kafka.server.fault.FaultHandler + + +class DynamicConfigPublisher( + conf: KafkaConfig, + faultHandler: FaultHandler, + dynamicConfigHandlers: Map[String, ConfigHandler], +) extends MetadataPublisher with Logging { + logIdent = s"[DynamicConfigPublisher id=${conf.nodeId}] " + + override def name(): String = "DynamicConfigPublisher" + + override def publishSnapshot( + delta: MetadataDelta, + newImage: MetadataImage, + manifest: SnapshotManifest + ): Unit = { + publish(delta, newImage) + } + + override def publishLogDelta( + delta: MetadataDelta, + newImage: MetadataImage, + manifest: LogDeltaManifest + ): Unit = { + publish(delta, newImage) + } + + def publish(delta: MetadataDelta, newImage: MetadataImage): Unit = { + val deltaName = s"MetadataDelta up to ${newImage.highestOffsetAndEpoch().offset}" + try { + // Apply configuration deltas. + Option(delta.configsDelta()).foreach { configsDelta => + configsDelta.changes().keySet().forEach { resource => + val props = newImage.configs().configProperties(resource) + resource.`type`() match { + case TOPIC => + try { + // Apply changes to a topic's dynamic configuration. + info(s"Updating topic ${resource.name()} with new configuration : " + + toLoggableProps(resource, props).mkString(",")) + dynamicConfigHandlers(ConfigType.Topic). + processConfigChanges(resource.name(), props) + } catch { + case t: Throwable => faultHandler.handleFault("Error updating topic " + + s"${resource.name()} with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + + s"in ${deltaName}", t) + } + case BROKER => + if (resource.name().isEmpty) { + try { + // Apply changes to "cluster configs" (also known as default BROKER configs). + // These are stored in KRaft with an empty name field. + info("Updating cluster configuration : " + + toLoggableProps(resource, props).mkString(",")) + dynamicConfigHandlers(ConfigType.Broker). + processConfigChanges(ConfigEntityName.Default, props) + } catch { + case t: Throwable => faultHandler.handleFault("Error updating " + + s"cluster with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + + s"in ${deltaName}", t) + } + } else if (resource.name() == conf.nodeId.toString) { + try { + // Apply changes to this broker's dynamic configuration. + info(s"Updating node ${conf.nodeId} with new configuration : " + + toLoggableProps(resource, props).mkString(",")) + dynamicConfigHandlers(ConfigType.Broker). + processConfigChanges(resource.name(), props) + // When applying a per broker config (not a cluster config), we also + // reload any associated file. For example, if the ssl.keystore is still + // set to /tmp/foo, we still want to reload /tmp/foo in case its contents + // have changed. This doesn't apply to topic configs or cluster configs. + reloadUpdatedFilesWithoutConfigChange(props) + } catch { + case t: Throwable => faultHandler.handleFault("Error updating " + + s"node with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + + s"in ${deltaName}", t) + } + } + case _ => // nothing to do + } + } + } + } catch { + case t: Throwable => faultHandler.handleFault("Uncaught exception while " + + s"publishing dynamic configuration changes from ${deltaName}", t) + } + } + + def reloadUpdatedFilesWithoutConfigChange(props: Properties): Unit = { + conf.dynamicConfig.reloadUpdatedFilesWithoutConfigChange(props) + } + + override def close(): Unit = { + // nothing to do + } +} diff --git a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java index 33617c50986b6..f9a7305ea5b61 100644 --- a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java +++ b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java @@ -200,6 +200,7 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS setThreadNamePrefix(threadNamePrefix). setFaultHandler(fatalFaultHandler). build(); + raftManager.register(metadataLoader); controller = new ControllerServer( nodes.controllerProperties(node.id()), config, @@ -279,6 +280,7 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS setThreadNamePrefix(threadNamePrefix). setFaultHandler(fatalFaultHandler). build(); + raftManager.register(metadataLoader); broker = new BrokerServer( config, nodes.brokerProperties(node.id()), diff --git a/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala b/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala index 0351ebb2077e7..d7bb80c468037 100755 --- a/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala +++ b/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala @@ -107,6 +107,7 @@ class KRaftQuorumImplementation( setNodeId(config.nodeId). setTime(time). setThreadNamePrefix(threadNamePrefix). + setFaultHandler(faultHandler). build() broker = new BrokerServer(config = config, metaProps = new MetaProperties(clusterId, config.nodeId), @@ -330,6 +331,7 @@ abstract class QuorumTestHarness extends Logging { setNodeId(config.nodeId). setTime(time). setThreadNamePrefix(threadNamePrefix). + setFaultHandler(faultHandler). build() controllerServer = new ControllerServer( metaProperties = metaProperties, diff --git a/core/src/test/scala/unit/kafka/server/DynamicConfigChangeTest.scala b/core/src/test/scala/unit/kafka/server/DynamicConfigChangeTest.scala index 84d6f5a2ef93d..9e7e0d2f17d0f 100644 --- a/core/src/test/scala/unit/kafka/server/DynamicConfigChangeTest.scala +++ b/core/src/test/scala/unit/kafka/server/DynamicConfigChangeTest.scala @@ -43,7 +43,7 @@ import org.apache.kafka.common.record.{CompressionType, RecordVersion} import org.apache.kafka.common.security.auth.KafkaPrincipal import org.apache.kafka.server.common.MetadataVersion.IBP_3_0_IV1 import org.junit.jupiter.api.Assertions._ -import org.junit.jupiter.api.Test +import org.junit.jupiter.api.{Test, Timeout} import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.ValueSource import org.mockito.ArgumentMatchers.{any, anyString} @@ -53,6 +53,7 @@ import scala.annotation.nowarn import scala.collection.{Map, Seq} import scala.jdk.CollectionConverters._ +@Timeout(60) class DynamicConfigChangeTest extends KafkaServerTestHarness { def generateConfigs = List(KafkaConfig.fromProps(TestUtils.createBrokerConfig(0, zkConnectOrNull))) diff --git a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala b/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala index 7bb8b86e06fda..603b38ca76499 100644 --- a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala +++ b/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala @@ -35,7 +35,7 @@ import org.apache.kafka.metadata.LeaderRecoveryState import org.apache.kafka.metadata.PartitionRegistration import org.apache.kafka.server.fault.{FaultHandler, MockFaultHandler} import org.junit.jupiter.api.Assertions.{assertEquals, assertNotNull, assertTrue} -import org.junit.jupiter.api.{AfterEach, BeforeEach, Disabled, Test} +import org.junit.jupiter.api.{AfterEach, BeforeEach, Disabled, Test, Timeout} import org.mockito.ArgumentMatchers.any import org.mockito.Mockito import org.mockito.Mockito.doThrow @@ -44,6 +44,7 @@ import org.mockito.stubbing.Answer import scala.jdk.CollectionConverters._ +@Timeout(60) class BrokerMetadataPublisherTest { val exitException = new AtomicReference[Throwable](null) @@ -176,23 +177,13 @@ class BrokerMetadataPublisherTest { new TopicsImage(idsMap.asJava, namesMap.asJava) } - private def newMockPublisher( + private def newMockDynamicConfigPublisher( broker: BrokerServer, errorHandler: FaultHandler = new MockFaultHandler("publisher") - ): BrokerMetadataPublisher = { - Mockito.spy(new BrokerMetadataPublisher( - conf = broker.config, - metadataCache = broker.metadataCache, - logManager = broker.logManager, - replicaManager = broker.replicaManager, - groupCoordinator = broker.groupCoordinator, - txnCoordinator = broker.transactionCoordinator, - clientQuotaMetadataManager = broker.clientQuotaMetadataManager, - dynamicConfigHandlers = broker.dynamicConfigHandlers.toMap, - _authorizer = Option.empty, - errorHandler, - errorHandler - )) + ): DynamicConfigPublisher = { + Mockito.spy(new DynamicConfigPublisher(conf = broker.config, + faultHandler = errorHandler, + dynamicConfigHandlers = broker.dynamicConfigHandlers.toMap)) } @Disabled @@ -207,14 +198,13 @@ class BrokerMetadataPublisherTest { cluster.startup() cluster.waitForReadyBrokers() val broker = cluster.brokers().values().iterator().next() - val publisher = newMockPublisher(broker) + val publisher = newMockDynamicConfigPublisher(broker) val numTimesReloadCalled = new AtomicInteger(0) Mockito.when(publisher.reloadUpdatedFilesWithoutConfigChange(any[Properties]())). thenAnswer(new Answer[Unit]() { override def answer(invocation: InvocationOnMock): Unit = numTimesReloadCalled.addAndGet(1) }) - broker.metadataLoader.removeAndClosePublisher(broker.metadataPublisher).get() - broker.metadataLoader.installPublishers(Collections.singletonList(publisher)).get() + broker.metadataPublisher.dynamicConfigPublisher = publisher val admin = Admin.create(cluster.clientProperties()) try { assertEquals(0, numTimesReloadCalled.get()) diff --git a/metadata/src/main/java/org/apache/kafka/image/MetadataProvenance.java b/metadata/src/main/java/org/apache/kafka/image/MetadataProvenance.java index 6e487768b507a..09ec10eb9d038 100644 --- a/metadata/src/main/java/org/apache/kafka/image/MetadataProvenance.java +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataProvenance.java @@ -17,6 +17,8 @@ package org.apache.kafka.image; +import org.apache.kafka.raft.OffsetAndEpoch; + import java.util.Objects; @@ -40,6 +42,10 @@ public MetadataProvenance( this.lastContainedLogTimeMs = lastContainedLogTimeMs; } + public OffsetAndEpoch offsetAndEpoch() { + return new OffsetAndEpoch(offset, epoch); + } + public long offset() { return offset; } diff --git a/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java b/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java index 47a3fc6b4b604..244b75a5f475d 100644 --- a/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java +++ b/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; /** @@ -197,6 +198,7 @@ public void handleCommit(BatchReader reader) { " and " + manifest.provenance().offset(), e); return; } + log.debug("Publishing new image with provenance {}.", image.provenance()); for (MetadataPublisher publisher : publishers) { try { publisher.publishLogDelta(delta, image, manifest); @@ -261,6 +263,8 @@ LogDeltaManifest loadLogDelta( } metrics.updateBatchSize(batch.records().size()); lastOffset = batch.lastOffset(); + lastEpoch = batch.epoch(); + lastContainedLogTimeMs = batch.appendTimestamp(); numBytes += batch.sizeInBytes(); numBatches++; } @@ -290,6 +294,14 @@ void handleMetadataVersionChange( int preChangeEpoch, long lastContainedLogTimeMs ) { + if (image.isEmpty()) { + // If the previous image was empty, this is not a true metadata version transition. + // It just indicates that we hadn't loaded any records before, and now we have. + // Print out the metadata version and return to the loading process. + log.info("Loaded initial metadata version as {}.", change.newVersion()); + delta.setMetadataVersion(change.newVersion()); + return; + } // First, we materialize the current metadata image and send it to all the publishers that // are interested in preVersionChange images. The most important one is the publisher which // writes snapshots out to disk. @@ -297,6 +309,7 @@ void handleMetadataVersionChange( new MetadataProvenance(changeOffset - 1, preChangeEpoch, lastContainedLogTimeMs); PreVersionChangeManifest manifest = new PreVersionChangeManifest(provenance, change); MetadataImage preVersionChangeImage = delta.apply(provenance); + log.info("Publishing pre-version change image with provenance {}.", image.provenance()); for (MetadataPublisher publisher : publishers) { try { publisher.publishPreVersionChangeImage(delta, preVersionChangeImage, manifest); @@ -309,6 +322,7 @@ void handleMetadataVersionChange( // If any metadata was lost, we just log it here. We cannot prevent the losses because the // decision to change the metadata version was already taken by the controller. delta.clear(); + delta.setMetadataVersion(change.newVersion()); ImageReWriter writer = new ImageReWriter(delta); preVersionChangeImage.write(writer, new ImageWriterOptions.Builder(). setMetadataVersion(change.newVersion()). @@ -333,6 +347,7 @@ public void handleSnapshot(SnapshotReader reader) { "snapshot at offset " + reader.lastContainedLogOffset(), e); return; } + log.debug("Publishing new snapshot image with provenance {}.", image.provenance()); for (MetadataPublisher publisher : publishers) { try { publisher.publishSnapshot(delta, image, manifest); @@ -356,8 +371,7 @@ public void handleSnapshot(SnapshotReader reader) { /** * Load a snapshot. This is relatively straightforward since we don't track as many things as - * we do in loadLogDelta. Additionally, it is an error for a version change to occur unless - * it is the very first record. The main complication here is that we have to maintain an index + * we do in loadLogDelta. The main complication here is that we have to maintain an index * of what record we are processing so that we can give useful error messages. * * @param delta The metadata delta we are preparing. @@ -375,6 +389,12 @@ SnapshotManifest loadSnapshot( for (ApiMessageAndVersion record : batch.records()) { try { delta.replay(record.message()); + } catch (MetadataVersionChangeException e) { + handleMetadataVersionChange(delta, + e.change(), + reader.lastContainedLogOffset(), + image.provenance().epoch(), + image.provenance().lastContainedLogTimeMs()); } catch (Throwable e) { faultHandler.handleFault("Error loading metadata log record " + snapshotIndex + " in snapshot at offset " + reader.lastContainedLogOffset(), e); @@ -408,9 +428,10 @@ public void handleLeaderChange(LeaderAndEpoch leaderAndEpoch) { public CompletableFuture installPublishers(List newPublishers) { if (newPublishers.isEmpty()) return CompletableFuture.completedFuture(null); CompletableFuture future = new CompletableFuture<>(); - eventQueue.beginShutdown("installPublishers", () -> { + eventQueue.append(() -> { try { installNewPublishers(newPublishers); + future.complete(null); } catch (Throwable e) { future.completeExceptionally(faultHandler.handleFault("Unhandled fault in " + "MetadataLoader#installPublishers", e)); @@ -435,6 +456,7 @@ void installNewPublishers( build(); ImageReWriter writer = new ImageReWriter(delta); image.write(writer, new ImageWriterOptions.Builder(). + setMetadataVersion(image.features().metadataVersion()). build()); SnapshotManifest manifest = new SnapshotManifest( image.provenance(), @@ -453,6 +475,13 @@ void installNewPublishers( } } + // VisibleForTesting + void waitForAllEventsToBeHandled() throws Exception { + CompletableFuture future = new CompletableFuture<>(); + eventQueue.append(() -> future.complete(null)); + future.get(); + } + /** * Remove a publisher and close it. * @@ -463,7 +492,7 @@ void installNewPublishers( */ public CompletableFuture removeAndClosePublisher(MetadataPublisher publisher) { CompletableFuture future = new CompletableFuture<>(); - eventQueue.beginShutdown("removePublisher", () -> { + eventQueue.append(() -> { try { if (!publishers.remove(publisher)) { throw faultHandler.handleFault("Attempted to remove publisher " + publisher.name() + diff --git a/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java b/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java new file mode 100644 index 0000000000000..0e5da5ec2bc71 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java @@ -0,0 +1,425 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.loader; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.metadata.FeatureLevelRecord; +import org.apache.kafka.common.metadata.RemoveTopicRecord; +import org.apache.kafka.common.metadata.TopicRecord; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.image.MetadataDelta; +import org.apache.kafka.image.MetadataImage; +import org.apache.kafka.image.MetadataProvenance; +import org.apache.kafka.image.publisher.MetadataPublisher; +import org.apache.kafka.raft.Batch; +import org.apache.kafka.raft.BatchReader; +import org.apache.kafka.raft.OffsetAndEpoch; +import org.apache.kafka.server.common.ApiMessageAndVersion; +import org.apache.kafka.server.common.MetadataVersion; +import org.apache.kafka.server.fault.MockFaultHandler; +import org.apache.kafka.snapshot.SnapshotReader; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.OptionalLong; +import java.util.concurrent.ExecutionException; + +import static java.util.Arrays.asList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +@Timeout(value = 40) +public class MetadataLoaderTest { + @Test + public void testCreateAndClose() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("testCreateAndClose"); + try (MetadataLoader loader = new MetadataLoader.Builder(). + setFaultHandler(faultHandler). + build()) { + assertEquals(-1L, loader.lastAppliedOffset()); + } + faultHandler.maybeRethrowFirstException(); + } + + static class MockPublisher implements MetadataPublisher { + MetadataDelta latestDelta = null; + MetadataImage latestImage = null; + LogDeltaManifest latestLogDeltaManifest = null; + SnapshotManifest latestSnapshotManifest = null; + MetadataDelta latestPreVersionChangeDelta = null; + MetadataImage latestPreVersionChangeImage = null; + PreVersionChangeManifest latestPreVersionChangeManifest = null; + boolean closed = false; + + @Override + public String name() { + return "MockPublisher"; + } + + @Override + public void publishSnapshot( + MetadataDelta delta, + MetadataImage newImage, + SnapshotManifest manifest + ) { + latestDelta = delta; + latestImage = newImage; + latestSnapshotManifest = manifest; + } + + @Override + public void publishPreVersionChangeImage( + MetadataDelta delta, + MetadataImage preVersionChangeImage, + PreVersionChangeManifest manifest + ) { + latestPreVersionChangeDelta = delta; + latestPreVersionChangeImage = preVersionChangeImage; + latestPreVersionChangeManifest = manifest; + } + + @Override + public void publishLogDelta( + MetadataDelta delta, + MetadataImage newImage, + LogDeltaManifest manifest + ) { + latestDelta = delta; + latestImage = newImage; + latestLogDeltaManifest = manifest; + } + + @Override + public void close() throws Exception { + closed = true; + } + } + + /** + * Install 2 publishers and check that the publishers that were installed are closed when the + * loader is closed. + */ + @Test + public void testInstallPublishers() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("testInstallPublishers"); + List publishers = asList(new MockPublisher(), + new MockPublisher(), + new MockPublisher()); + try (MetadataLoader loader = new MetadataLoader.Builder(). + setFaultHandler(faultHandler). + build()) { + loader.installPublishers(publishers.subList(0, 2)).get(); + } + assertTrue(publishers.get(0).closed); + assertEquals(MetadataImage.EMPTY, publishers.get(0).latestImage); + assertTrue(publishers.get(1).closed); + assertEquals(MetadataImage.EMPTY, publishers.get(1).latestImage); + assertFalse(publishers.get(2).closed); + assertNull(publishers.get(2).latestImage); + faultHandler.maybeRethrowFirstException(); + } + + /** + * Test that a publisher cannot be installed more than once. + */ + @Test + public void testPublisherCannotBeInstalledMoreThanOnce() throws Exception { + MockFaultHandler faultHandler = + new MockFaultHandler("testPublisherCannotBeInstalledMoreThanOnce"); + MockPublisher publisher = new MockPublisher(); + try (MetadataLoader loader = new MetadataLoader.Builder(). + setFaultHandler(faultHandler). + build()) { + loader.installPublishers(asList(publisher)).get(); + assertEquals("testPublisherCannotBeInstalledMoreThanOnce: Attempted to install " + + "publisher MockPublisher, which is already installed.", + assertThrows(ExecutionException.class, + () -> loader.installPublishers(asList(publisher)).get()). + getCause().getMessage()); + } + } + + static class MockSnapshotReader implements SnapshotReader { + private final MetadataProvenance provenance; + private final Iterator> iterator; + private MockTime time = null; + boolean closed = false; + + static MockSnapshotReader fromRecordLists( + MetadataProvenance provenance, + List> lists + ) { + List> batches = new ArrayList<>(); + lists.forEach(records -> batches.add(Batch.data( + provenance.offset(), + provenance.epoch(), + provenance.lastContainedLogTimeMs(), + 0, + records))); + return new MockSnapshotReader(provenance, batches); + } + + MockSnapshotReader( + MetadataProvenance provenance, + List> batches + ) { + this.provenance = provenance; + this.iterator = batches.iterator(); + } + + MockSnapshotReader setTime(MockTime time) { + this.time = time; + return this; + } + + @Override + public OffsetAndEpoch snapshotId() { + return provenance.offsetAndEpoch(); + } + + @Override + public long lastContainedLogOffset() { + return provenance.offset(); + } + + @Override + public int lastContainedLogEpoch() { + return provenance.epoch(); + } + + @Override + public long lastContainedLogTimestamp() { + return provenance.lastContainedLogTimeMs(); + } + + @Override + public void close() { + closed = true; + } + + @Override + public boolean hasNext() { + if (time != null) time.sleep(1); + return iterator.hasNext(); + } + + @Override + public Batch next() { + if (time != null) time.sleep(1); + return iterator.next(); + } + } + + /** + * Install 2 publishers and remove one. + */ + @Test + public void testRemovePublisher() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("testRemovePublisher"); + List publishers = asList(new MockPublisher(), + new MockPublisher(), + new MockPublisher()); + try (MetadataLoader loader = new MetadataLoader.Builder(). + setFaultHandler(faultHandler). + build()) { + loader.installPublishers(publishers.subList(0, 2)).get(); + loader.removeAndClosePublisher(publishers.get(1)).get(); + MockSnapshotReader snapshotReader = MockSnapshotReader.fromRecordLists( + new MetadataProvenance(100, 50, 2000), + asList(asList(new ApiMessageAndVersion( + new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(MetadataVersion.IBP_3_3_IV2.featureLevel()), (short) 0)))); + assertFalse(snapshotReader.closed); + loader.handleSnapshot(snapshotReader); + loader.waitForAllEventsToBeHandled(); + assertTrue(snapshotReader.closed); + loader.removeAndClosePublisher(publishers.get(0)).get(); + } + assertTrue(publishers.get(0).closed); + assertEquals(MetadataVersion.IBP_3_3_IV2, + publishers.get(0).latestImage.features().metadataVersion()); + assertTrue(publishers.get(1).closed); + assertEquals(MetadataImage.EMPTY, publishers.get(1).latestImage); + assertFalse(publishers.get(2).closed); + assertNull(publishers.get(2).latestImage); + faultHandler.maybeRethrowFirstException(); + } + + /** + * Test loading a snapshot with 0 records. + */ + @Test + public void testLoadEmptySnapshot() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("testLoadEmptySnapshot"); + MockTime time = new MockTime(); + List publishers = asList(new MockPublisher()); + try (MetadataLoader loader = new MetadataLoader.Builder(). + setFaultHandler(faultHandler). + setTime(time). + build()) { + loader.installPublishers(publishers).get(); + MockSnapshotReader snapshotReader = new MockSnapshotReader( + new MetadataProvenance(200, 100, 4000), + asList(Batch.control(200, 100, 4000, 10, 200))). + setTime(time); + loader.handleSnapshot(snapshotReader); + loader.waitForAllEventsToBeHandled(); + assertEquals(200L, loader.lastAppliedOffset()); + assertEquals(new SnapshotManifest(new MetadataProvenance(200, 100, 4000), 3000000L), + publishers.get(0).latestSnapshotManifest); + } + assertTrue(publishers.get(0).closed); + assertEquals(MetadataVersion.IBP_3_0_IV1, + publishers.get(0).latestImage.features().metadataVersion()); + assertTrue(publishers.get(0).latestImage.isEmpty()); + faultHandler.maybeRethrowFirstException(); + } + + static class MockBatchReader implements BatchReader { + private final long baseOffset; + private final Iterator> iterator; + private boolean closed = false; + private MockTime time = null; + + static Batch newBatch( + long batchBaseOffset, + int epoch, + List records + ) { + return Batch.data(batchBaseOffset, epoch, 0, 0, records); + } + + MockBatchReader( + long baseOffset, + List> batches + ) { + this.baseOffset = baseOffset; + this.iterator = batches.iterator(); + } + + private MockBatchReader setTime(MockTime time) { + this.time = time; + return this; + } + + @Override + public long baseOffset() { + return baseOffset; + } + + @Override + public OptionalLong lastOffset() { + return OptionalLong.empty(); + } + + @Override + public void close() { + this.closed = true; + } + + @Override + public boolean hasNext() { + if (time != null) time.sleep(1); + return iterator.hasNext(); + } + + @Override + public Batch next() { + if (time != null) time.sleep(1); + return iterator.next(); + } + } + + /** + * Test loading a batch with 0 records. + */ + @Test + public void testLoadEmptyBatch() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("testLoadEmptyBatch"); + MockTime time = new MockTime(); + List publishers = asList(new MockPublisher()); + try (MetadataLoader loader = new MetadataLoader.Builder(). + setFaultHandler(faultHandler). + setTime(time). + build()) { + loader.installPublishers(publishers).get(); + MockBatchReader batchReader = new MockBatchReader(300, asList( + Batch.control(300, 100, 4000, 10, 400))). + setTime(time); + loader.handleCommit(batchReader); + loader.waitForAllEventsToBeHandled(); + assertTrue(batchReader.closed); + assertEquals(400L, loader.lastAppliedOffset()); + } + assertTrue(publishers.get(0).closed); + assertEquals(new LogDeltaManifest(new MetadataProvenance(400, 100, 4000), 1, 3000000L, 10), + publishers.get(0).latestLogDeltaManifest); + assertEquals(MetadataVersion.IBP_3_0_IV1, + publishers.get(0).latestImage.features().metadataVersion()); + assertTrue(publishers.get(0).latestImage.isEmpty()); + faultHandler.maybeRethrowFirstException(); + } + + /** + * Test that the lastAppliedOffset moves forward as expected. + */ + @Test + public void testLastAppliedOffset() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("testRemovePublisher"); + List publishers = asList(new MockPublisher(), + new MockPublisher()); + try (MetadataLoader loader = new MetadataLoader.Builder(). + setFaultHandler(faultHandler). + build()) { + loader.installPublishers(publishers).get(); + loader.handleSnapshot(MockSnapshotReader.fromRecordLists( + new MetadataProvenance(200, 100, 4000), asList( + asList(new ApiMessageAndVersion(new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(MetadataVersion.IBP_3_3_IV1.featureLevel()), (short) 0)), + asList(new ApiMessageAndVersion(new TopicRecord(). + setName("foo"). + setTopicId(Uuid.fromString("Uum7sfhHQP-obSvfywmNUA")), (short) 0)) + ))); + loader.waitForAllEventsToBeHandled(); + assertEquals(200L, loader.lastAppliedOffset()); + loader.handleCommit(new MockBatchReader(201, asList( + MockBatchReader.newBatch(201, 100, asList( + new ApiMessageAndVersion(new RemoveTopicRecord(). + setTopicId(Uuid.fromString("Uum7sfhHQP-obSvfywmNUA")), (short) 0)))))); + loader.waitForAllEventsToBeHandled(); + assertEquals(201L, loader.lastAppliedOffset()); + } + for (int i = 0; i < 2; i++) { + assertTrue(publishers.get(i).closed); + assertTrue(publishers.get(i).closed); + assertEquals(MetadataVersion.IBP_3_3_IV1, + publishers.get(i).latestImage.features().metadataVersion()); + } + faultHandler.maybeRethrowFirstException(); + } +} From 247db57f575a897a16616f60fa6aaffa1eb825eb Mon Sep 17 00:00:00 2001 From: "Colin P. McCabe" Date: Wed, 2 Nov 2022 01:07:49 -0700 Subject: [PATCH 03/15] address review comments --- checkstyle/suppressions.xml | 4 +- .../kafka/server/DynamicBrokerConfig.scala | 9 ++- .../main/scala/kafka/server/KafkaConfig.scala | 5 +- .../scala/kafka/server/KafkaRaftServer.scala | 7 +-- .../kafka/testkit/KafkaClusterTestKit.java | 8 +-- .../kafka/image/loader/MetadataLoader.java | 62 +++++++++++-------- .../image/publisher/SnapshotEmitter.java | 16 +++-- .../image/publisher/SnapshotGenerator.java | 29 +++++---- .../controller/ClusterControlManagerTest.java | 2 +- .../image/loader/MetadataLoaderTest.java | 1 - .../image/publisher/SnapshotEmitterTest.java | 4 +- .../publisher/SnapshotGeneratorTest.java | 14 ++--- 12 files changed, 87 insertions(+), 74 deletions(-) diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 1d60fde34d20b..debc77ccd5f5d 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -38,9 +38,7 @@ - - + { })). - setEmitter(snapshotEmitter). setMinBytesSinceLastSnapshot(config.metadataSnapshotMaxNewRecordBytes). setMinTimeSinceLastSnapshotNs(TimeUnit.MILLISECONDS.toNanos(config.metadataSnapshotMaxIntervalMsProp)). build() @@ -105,7 +104,7 @@ class KafkaRaftServer( private val metadataLoader: MetadataLoader = { val builder = new MetadataLoader.Builder(). - setTime(Time.SYSTEM). + setTime(time). setNodeId(config.nodeId) if (config.processRoles.contains(ControllerRole)) { // Controllers and combined nodes must exit immediately if there is a metadata loading error. diff --git a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java index f9a7305ea5b61..6c21b28f3985a 100644 --- a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java +++ b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java @@ -218,8 +218,8 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS metadataLoader); } catch (Throwable e) { log.error("Error creating controller {}", node.id(), e); - metadataLoader.close(); - controller.shutdown(); + if (metadataLoader != null) metadataLoader.close(); + if (controller != null) controller.shutdown(); throw e; } controllers.put(node.id(), controller); @@ -295,8 +295,8 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS metadataLoader); } catch (Throwable e) { log.error("Error creating broker {}", node.id(), e); - metadataLoader.close(); - broker.shutdown(); + if (metadataLoader != null) metadataLoader.close(); + if (broker != null) broker.shutdown(); throw e; } brokers.put(node.id(), broker); diff --git a/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java b/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java index 244b75a5f475d..a7e2de83d65f1 100644 --- a/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java +++ b/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java @@ -40,7 +40,8 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; /** @@ -190,6 +191,11 @@ public void handleCommit(BatchReader reader) { setImage(image). build(); LogDeltaManifest manifest = loadLogDelta(delta, reader); + if (log.isDebugEnabled()) { + log.debug("Generated a metadata delta between {} and {} from {} batch(es) " + + "in {} us.", image.offset(), manifest.provenance().offset(), + manifest.numBatches(), NANOSECONDS.toMicros(manifest.elapsedNs())); + } try { image = delta.apply(manifest.provenance()); } catch (Throwable e) { @@ -248,17 +254,23 @@ LogDeltaManifest loadLogDelta( try { delta.replay(record.message()); } catch (MetadataVersionChangeException e) { - handleMetadataVersionChange(delta, - e.change(), - batch.baseOffset() + indexWithinBatch, - lastEpoch, - lastContainedLogTimeMs); + MetadataProvenance preChangeProvenance; + if (indexWithinBatch == 0) { + preChangeProvenance = new MetadataProvenance( + lastOffset, + lastEpoch, + lastContainedLogTimeMs); + } else { + preChangeProvenance = new MetadataProvenance( + batch.baseOffset() + indexWithinBatch - 1, + batch.epoch(), + batch.appendTimestamp()); + } + handleMetadataVersionChange(delta, e.change(), preChangeProvenance); } catch (Throwable e) { faultHandler.handleFault("Error loading metadata log record from offset " + batch.baseOffset() + indexWithinBatch, e); } - lastEpoch = batch.epoch(); - lastContainedLogTimeMs = batch.appendTimestamp(); indexWithinBatch++; } metrics.updateBatchSize(batch.records().size()); @@ -271,7 +283,6 @@ LogDeltaManifest loadLogDelta( MetadataProvenance provenance = new MetadataProvenance(lastOffset, lastEpoch, lastContainedLogTimeMs); long elapsedNs = time.nanoseconds() - startNs; - // TODO: this metric should be renamed something like "delta processing time" metrics.updateBatchProcessingTime(elapsedNs); return new LogDeltaManifest(provenance, numBatches, @@ -282,17 +293,15 @@ LogDeltaManifest loadLogDelta( /** * Handle a change in the metadata version. * - * @param delta The metadata delta we're working with. - * @param change The change we're handling. - * @param changeOffset The offset of the change. - * @param preChangeEpoch The log epoch BEFORE the change. + * @param delta The metadata delta we're working with. + * @param change The change we're handling. + * @param preChangeProvenance The pre-change provenance. This will be the offset, epoch, + * timestamp, etc. right before the MV change. */ void handleMetadataVersionChange( MetadataDelta delta, MetadataVersionChange change, - long changeOffset, - int preChangeEpoch, - long lastContainedLogTimeMs + MetadataProvenance preChangeProvenance ) { if (image.isEmpty()) { // If the previous image was empty, this is not a true metadata version transition. @@ -305,17 +314,15 @@ void handleMetadataVersionChange( // First, we materialize the current metadata image and send it to all the publishers that // are interested in preVersionChange images. The most important one is the publisher which // writes snapshots out to disk. - MetadataProvenance provenance = - new MetadataProvenance(changeOffset - 1, preChangeEpoch, lastContainedLogTimeMs); - PreVersionChangeManifest manifest = new PreVersionChangeManifest(provenance, change); - MetadataImage preVersionChangeImage = delta.apply(provenance); - log.info("Publishing pre-version change image with provenance {}.", image.provenance()); + PreVersionChangeManifest manifest = new PreVersionChangeManifest(preChangeProvenance, change); + MetadataImage preVersionChangeImage = delta.apply(preChangeProvenance); + log.info("Publishing pre-version change image with provenance {}.", preChangeProvenance); for (MetadataPublisher publisher : publishers) { try { publisher.publishPreVersionChangeImage(delta, preVersionChangeImage, manifest); } catch (Throwable e) { faultHandler.handleFault("Error publishing pre-version change image at offset " + - provenance.offset() + " with publisher " + publisher.name(), e); + preChangeProvenance.offset() + " with publisher " + publisher.name(), e); } } // Then, we clear the current delta and write out the current image to it in the new format. @@ -340,6 +347,11 @@ public void handleSnapshot(SnapshotReader reader) { setImage(image). build(); SnapshotManifest manifest = loadSnapshot(delta, reader); + if (log.isDebugEnabled()) { + log.debug("Generated a metadata delta from a snapshot at offset {} " + + "in {} us.", manifest.provenance().offset(), + NANOSECONDS.toMicros(manifest.elapsedNs())); + } try { image = delta.apply(manifest.provenance()); } catch (Throwable e) { @@ -390,11 +402,7 @@ SnapshotManifest loadSnapshot( try { delta.replay(record.message()); } catch (MetadataVersionChangeException e) { - handleMetadataVersionChange(delta, - e.change(), - reader.lastContainedLogOffset(), - image.provenance().epoch(), - image.provenance().lastContainedLogTimeMs()); + handleMetadataVersionChange(delta, e.change(), image.provenance()); } catch (Throwable e) { faultHandler.handleFault("Error loading metadata log record " + snapshotIndex + " in snapshot at offset " + reader.lastContainedLogOffset(), e); diff --git a/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotEmitter.java b/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotEmitter.java index 39ddee784f57e..1dc95d873c5bd 100644 --- a/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotEmitter.java +++ b/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotEmitter.java @@ -17,8 +17,8 @@ package org.apache.kafka.image.publisher; - import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Utils; import org.apache.kafka.image.MetadataImage; import org.apache.kafka.image.MetadataProvenance; import org.apache.kafka.image.writer.ImageWriterOptions; @@ -30,6 +30,7 @@ import java.util.Optional; + public class SnapshotEmitter implements SnapshotGenerator.Emitter { private final static int DEFAULT_BATCH_SIZE = 100; @@ -87,7 +88,7 @@ private SnapshotEmitter( } @Override - public void emit(MetadataImage image) { + public void maybeEmit(MetadataImage image) { MetadataProvenance provenance = image.provenance(); Optional> snapshotWriter = raftClient.createSnapshot(provenance.offset(), @@ -97,7 +98,8 @@ public void emit(MetadataImage image) { log.error("Not generating {} because it already exists.", provenance.snapshotName()); return; } - try (RaftSnapshotWriter writer = new RaftSnapshotWriter(snapshotWriter.get(), batchSize)) { + RaftSnapshotWriter writer = new RaftSnapshotWriter(snapshotWriter.get(), batchSize); + try { image.write(writer, new ImageWriterOptions.Builder(). setMetadataVersion(image.features().metadataVersion()). build()); @@ -106,12 +108,8 @@ public void emit(MetadataImage image) { log.error("Encountered error while writing {}", provenance.snapshotName(), e); throw e; } finally { - try { - snapshotWriter.get().close(); - } catch (Throwable e) { - log.error("Error closing SnapshotWriter for {}", provenance.snapshotName(), e); - throw e; - } + Utils.closeQuietly(writer, "RaftSnapshotWriter"); + Utils.closeQuietly(snapshotWriter.get(), "SnapshotWriter"); } log.info("Successfully wrote {}", provenance.snapshotName()); } diff --git a/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java b/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java index 68f08eca9cb8a..d0d294970a8a1 100644 --- a/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java +++ b/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java @@ -29,6 +29,7 @@ import org.apache.kafka.server.fault.FaultHandler; import org.slf4j.Logger; +import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -37,13 +38,17 @@ */ public class SnapshotGenerator implements MetadataPublisher { public static class Builder { + private final Emitter emitter; private int nodeId = 0; private Time time = Time.SYSTEM; - private Emitter emitter = null; private FaultHandler faultHandler = (m, e) -> null; private long minBytesSinceLastSnapshot = 100 * 1024L * 1024L; private long minTimeSinceLastSnapshotNs = TimeUnit.DAYS.toNanos(1); + public Builder(Emitter emitter) { + this.emitter = emitter; + } + public Builder setNodeId(int nodeId) { this.nodeId = nodeId; return this; @@ -54,11 +59,6 @@ public Builder setTime(Time time) { return this; } - public Builder setEmitter(Emitter emitter) { - this.emitter = emitter; - return this; - } - public Builder setFaultHandler(FaultHandler faultHandler) { this.faultHandler = faultHandler; return this; @@ -75,7 +75,6 @@ public Builder setMinTimeSinceLastSnapshotNs(long minTimeSinceLastSnapshotNs) { } public SnapshotGenerator build() { - if (emitter == null) throw new RuntimeException("You must set a snapshot emitter."); return new SnapshotGenerator( nodeId, time, @@ -91,7 +90,15 @@ public SnapshotGenerator build() { * The callback which actually generates the snapshot. */ public interface Emitter { - void emit(MetadataImage image); + /** + * Emit a snapshot for the given image. + * + * Note: if a snapshot has already been emitted for the given offset and epoch pair, this + * function will not recreate it. + * + * @param image The metadata image to emit. + */ + void maybeEmit(MetadataImage image); } /** @@ -167,7 +174,7 @@ private SnapshotGenerator( LogContext logContext = new LogContext("[SnapshotGenerator " + nodeId + "] "); this.log = logContext.logger(SnapshotGenerator.class); this.disabledReason = null; - this.eventQueue = new KafkaEventQueue(time, logContext, ""); + this.eventQueue = new KafkaEventQueue(time, logContext, "SnapshotGenerator" + nodeId); resetSnapshotCounters(); } @@ -236,7 +243,7 @@ void scheduleEmit( log.info("Creating new KRaft snapshot file {} because {}.", image.provenance().snapshotName(), reason); try { - emitter.emit(image); + emitter.maybeEmit(image); } catch (Throwable e) { faultHandler.handleFault("KRaft snapshot file generation error", e); } @@ -251,7 +258,7 @@ public void beginShutdown() { public void disable(String disabledReason) { log.error("Disabling periodic KRaft snapshot generation because {}.", disabledReason); - this.disabledReason = disabledReason; + this.disabledReason = Objects.requireNonNull(disabledReason); } @Override diff --git a/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java index 24c9086ed9eb7..7a73788cf3627 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java @@ -498,7 +498,7 @@ public void testRegistrationsToRecords(MetadataVersion metadataVersion) throws E setPort((short) 9094). setName("PLAINTEXT"). setHost("example.com")).iterator())). - setFenced(true), expectedVersion), + setFenced(true), expectedVersion), clusterControl.brokerRegistrations().get(2).toRecord(options)); } diff --git a/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java b/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java index 0e5da5ec2bc71..e05d33bb437dd 100644 --- a/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java @@ -37,7 +37,6 @@ import org.junit.jupiter.api.Timeout; import java.util.ArrayList; -import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.OptionalLong; diff --git a/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotEmitterTest.java b/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotEmitterTest.java index 62a1d15c3fa82..9bdcedd87d77d 100644 --- a/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotEmitterTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotEmitterTest.java @@ -180,7 +180,7 @@ public void testEmit() throws Exception { setBatchSize(2). setRaftClient(mockRaftClient). build(); - emitter.emit(MetadataImageTest.IMAGE1); + emitter.maybeEmit(MetadataImageTest.IMAGE1); MockSnapshotWriter writer = mockRaftClient.writers.get( MetadataImageTest.IMAGE1.highestOffsetAndEpoch()); assertNotNull(writer); @@ -192,7 +192,7 @@ public void testEmit() throws Exception { assertTrue(writer.isClosed()); // Second call to emit does nothing because we already have a snapshot at that offset and epoch. - emitter.emit(MetadataImageTest.IMAGE1); + emitter.maybeEmit(MetadataImageTest.IMAGE1); assertEquals(1, mockRaftClient.writers.size()); } } diff --git a/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotGeneratorTest.java b/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotGeneratorTest.java index 8f022004ac7cb..edacba05c3d21 100644 --- a/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotGeneratorTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotGeneratorTest.java @@ -62,7 +62,7 @@ synchronized MockEmitter setProblem(RuntimeException problem) { } @Override - public synchronized void emit(MetadataImage image) { + public synchronized void maybeEmit(MetadataImage image) { RuntimeException currentProblem = problem; if (currentProblem != null) { throw currentProblem; @@ -96,8 +96,7 @@ synchronized List images() { public void testCreateSnapshot() throws Exception { MockFaultHandler faultHandler = new MockFaultHandler("SnapshotGenerator"); MockEmitter emitter = new MockEmitter(); - try (SnapshotGenerator generator = new SnapshotGenerator.Builder(). - setEmitter(emitter). + try (SnapshotGenerator generator = new SnapshotGenerator.Builder(emitter). setFaultHandler(faultHandler). setMinBytesSinceLastSnapshot(200). setMinTimeSinceLastSnapshotNs(TimeUnit.DAYS.toNanos(10)). @@ -130,8 +129,7 @@ public void testCreateSnapshot() throws Exception { public void testSnapshotsDisabled() throws Exception { MockFaultHandler faultHandler = new MockFaultHandler("SnapshotGenerator"); MockEmitter emitter = new MockEmitter().setReady(); - try (SnapshotGenerator generator = new SnapshotGenerator.Builder(). - setEmitter(emitter). + try (SnapshotGenerator generator = new SnapshotGenerator.Builder(emitter). setFaultHandler(faultHandler). setMinBytesSinceLastSnapshot(1). setMinTimeSinceLastSnapshotNs(TimeUnit.DAYS.toNanos(10)). @@ -153,9 +151,8 @@ public void testTimeBasedSnapshots() throws Exception { MockFaultHandler faultHandler = new MockFaultHandler("SnapshotGenerator"); MockEmitter emitter = new MockEmitter().setReady(); MockTime mockTime = new MockTime(); - try (SnapshotGenerator generator = new SnapshotGenerator.Builder(). + try (SnapshotGenerator generator = new SnapshotGenerator.Builder(emitter). setTime(mockTime). - setEmitter(emitter). setFaultHandler(faultHandler). setMinBytesSinceLastSnapshot(200). setMinTimeSinceLastSnapshotNs(TimeUnit.MINUTES.toNanos(30)). @@ -182,8 +179,7 @@ public void testTimeBasedSnapshots() throws Exception { public void testEmitterProblem() throws Exception { MockFaultHandler faultHandler = new MockFaultHandler("SnapshotGenerator"); MockEmitter emitter = new MockEmitter().setProblem(new RuntimeException("oops")); - try (SnapshotGenerator generator = new SnapshotGenerator.Builder(). - setEmitter(emitter). + try (SnapshotGenerator generator = new SnapshotGenerator.Builder(emitter). setFaultHandler(faultHandler). setMinBytesSinceLastSnapshot(200). build()) { From fd60e8e475e552be8ceb012d8c7469d692a7136f Mon Sep 17 00:00:00 2001 From: "Colin P. McCabe" Date: Wed, 2 Nov 2022 10:33:23 -0700 Subject: [PATCH 04/15] Get snapshots tests with KafkaClusterTestKit working - Rename SnapshotGenerator#minTimeSinceLastSnapshotNs to maxTimeSinceLastSnapshotNs. It is a maximum, not a minimum. - KafkaClusterTestKit should create a SnapshotGenerator if either the time-based or bytes-based config is set (but not otherwise). --- .../scala/kafka/server/KafkaRaftServer.scala | 2 +- .../kafka/testkit/KafkaClusterTestKit.java | 46 ++++++++++++++++++- .../image/publisher/SnapshotGenerator.java | 22 ++++----- .../publisher/SnapshotGeneratorTest.java | 6 +-- 4 files changed, 59 insertions(+), 17 deletions(-) diff --git a/core/src/main/scala/kafka/server/KafkaRaftServer.scala b/core/src/main/scala/kafka/server/KafkaRaftServer.scala index ad3db428611cd..7036f7709ccf9 100644 --- a/core/src/main/scala/kafka/server/KafkaRaftServer.scala +++ b/core/src/main/scala/kafka/server/KafkaRaftServer.scala @@ -93,7 +93,7 @@ class KafkaRaftServer( setNodeId(config.nodeId). setFaultHandler(new LoggingFaultHandler("snapshot generation", () => { })). setMinBytesSinceLastSnapshot(config.metadataSnapshotMaxNewRecordBytes). - setMinTimeSinceLastSnapshotNs(TimeUnit.MILLISECONDS.toNanos(config.metadataSnapshotMaxIntervalMsProp)). + setMaxTimeSinceLastSnapshotNs(TimeUnit.MILLISECONDS.toNanos(config.metadataSnapshotMaxIntervalMsProp)). build() private val brokerMetrics: Option[BrokerServerMetrics] = if (config.processRoles.contains(BrokerRole)) { diff --git a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java index 6c21b28f3985a..4a0632530496c 100644 --- a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java +++ b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java @@ -37,8 +37,11 @@ import org.apache.kafka.controller.Controller; import org.apache.kafka.controller.MockControllerMetrics; import org.apache.kafka.image.loader.MetadataLoader; +import org.apache.kafka.image.publisher.SnapshotEmitter; +import org.apache.kafka.image.publisher.SnapshotGenerator; import org.apache.kafka.metadata.MetadataRecordSerde; import org.apache.kafka.metadata.bootstrap.BootstrapMetadata; +import org.apache.kafka.raft.KafkaRaftClient; import org.apache.kafka.raft.RaftConfig; import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.server.common.MetadataVersion; @@ -58,12 +61,14 @@ import java.nio.file.Paths; import java.util.AbstractMap.SimpleImmutableEntry; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.Properties; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; @@ -193,6 +198,7 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS Time.SYSTEM, new Metrics(), Option.apply(threadNamePrefix), connectFutureManager.future); MetadataLoader metadataLoader = null; ControllerServer controller = null; + SnapshotGenerator snapshotGenerator = null; try { metadataLoader = new MetadataLoader.Builder(). setNodeId(node.id()). @@ -201,6 +207,10 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS setFaultHandler(fatalFaultHandler). build(); raftManager.register(metadataLoader); + snapshotGenerator = maybeCreateSnapshotGenerator(node.id(), raftManager.client()); + if (snapshotGenerator != null) { + metadataLoader.installPublishers(Arrays.asList(snapshotGenerator)); + } controller = new ControllerServer( nodes.controllerProperties(node.id()), config, @@ -218,7 +228,8 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS metadataLoader); } catch (Throwable e) { log.error("Error creating controller {}", node.id(), e); - if (metadataLoader != null) metadataLoader.close(); + Utils.closeQuietly(metadataLoader, "metadataLoader"); + Utils.closeQuietly(snapshotGenerator, "snapshotGenerator"); if (controller != null) controller.shutdown(); throw e; } @@ -273,6 +284,7 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS Metrics metrics = new Metrics(); MetadataLoader metadataLoader = null; BrokerServer broker = null; + SnapshotGenerator snapshotGenerator = null; try { metadataLoader = new MetadataLoader.Builder(). setNodeId(node.id()). @@ -281,6 +293,10 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS setFaultHandler(fatalFaultHandler). build(); raftManager.register(metadataLoader); + snapshotGenerator = maybeCreateSnapshotGenerator(node.id(), raftManager.client()); + if (snapshotGenerator != null) { + metadataLoader.installPublishers(Arrays.asList(snapshotGenerator)); + } broker = new BrokerServer( config, nodes.brokerProperties(node.id()), @@ -295,7 +311,8 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS metadataLoader); } catch (Throwable e) { log.error("Error creating broker {}", node.id(), e); - if (metadataLoader != null) metadataLoader.close(); + Utils.closeQuietly(metadataLoader, "metadataLoader"); + Utils.closeQuietly(snapshotGenerator, "snapshotGenerator"); if (broker != null) broker.shutdown(); throw e; } @@ -326,6 +343,31 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS metadataFaultHandler, fatalFaultHandler); } + private SnapshotGenerator maybeCreateSnapshotGenerator( + int nodeId, + KafkaRaftClient raftClient + ) { + String minBytes = configProps.get(KafkaConfig$.MODULE$.MetadataSnapshotMaxNewRecordBytesProp()); + String maxIntervalMs = configProps.get(KafkaConfig$.MODULE$.MetadataSnapshotMaxIntervalMsProp()); + if (minBytes == null && maxIntervalMs == null) { + return null; + } + SnapshotEmitter emitter = new SnapshotEmitter.Builder(). + setNodeId(nodeId). + setRaftClient(raftClient). + build(); + SnapshotGenerator.Builder builder = new SnapshotGenerator.Builder(emitter). + setTime(Time.SYSTEM). + setFaultHandler(fatalFaultHandler); + if (minBytes != null) { + builder.setMinBytesSinceLastSnapshot(Long.parseLong(minBytes)); + } + if (maxIntervalMs != null) { + builder.setMaxTimeSinceLastSnapshotNs(Long.parseLong(maxIntervalMs)); + } + return builder.build(); + } + private String listeners(int node) { if (nodes.isCoResidentNode(node)) { return "EXTERNAL://localhost:0,CONTROLLER://localhost:0"; diff --git a/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java b/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java index d0d294970a8a1..61fb31025680f 100644 --- a/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java +++ b/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java @@ -43,7 +43,7 @@ public static class Builder { private Time time = Time.SYSTEM; private FaultHandler faultHandler = (m, e) -> null; private long minBytesSinceLastSnapshot = 100 * 1024L * 1024L; - private long minTimeSinceLastSnapshotNs = TimeUnit.DAYS.toNanos(1); + private long maxTimeSinceLastSnapshotNs = TimeUnit.DAYS.toNanos(1); public Builder(Emitter emitter) { this.emitter = emitter; @@ -69,8 +69,8 @@ public Builder setMinBytesSinceLastSnapshot(long minBytesSinceLastSnapshot) { return this; } - public Builder setMinTimeSinceLastSnapshotNs(long minTimeSinceLastSnapshotNs) { - this.minTimeSinceLastSnapshotNs = minTimeSinceLastSnapshotNs; + public Builder setMaxTimeSinceLastSnapshotNs(long maxTimeSinceLastSnapshotNs) { + this.maxTimeSinceLastSnapshotNs = maxTimeSinceLastSnapshotNs; return this; } @@ -81,7 +81,7 @@ public SnapshotGenerator build() { emitter, faultHandler, minBytesSinceLastSnapshot, - minTimeSinceLastSnapshotNs + maxTimeSinceLastSnapshotNs ); } } @@ -132,10 +132,10 @@ public interface Emitter { private final long minBytesSinceLastSnapshot; /** - * The minimum amount of time we will wait before emitting a snapshot, or 0 to disable + * The maximum amount of time we will wait before emitting a snapshot, or 0 to disable * time-based snapshotting. */ - private final long minTimeSinceLastSnapshotNs; + private final long maxTimeSinceLastSnapshotNs; /** * If non-null, the reason why snapshots have been disabled. @@ -163,14 +163,14 @@ private SnapshotGenerator( Emitter emitter, FaultHandler faultHandler, long minBytesSinceLastSnapshot, - long minTimeSinceLastSnapshotNs + long maxTimeSinceLastSnapshotNs ) { this.nodeId = nodeId; this.time = time; this.emitter = emitter; this.faultHandler = faultHandler; this.minBytesSinceLastSnapshot = minBytesSinceLastSnapshot; - this.minTimeSinceLastSnapshotNs = minTimeSinceLastSnapshotNs; + this.maxTimeSinceLastSnapshotNs = maxTimeSinceLastSnapshotNs; LogContext logContext = new LogContext("[SnapshotGenerator " + nodeId + "] "); this.log = logContext.logger(SnapshotGenerator.class); this.disabledReason = null; @@ -219,11 +219,11 @@ public void publishLogDelta( scheduleEmit("we have replayed at least " + minBytesSinceLastSnapshot + " bytes", newImage); } - } else if (minTimeSinceLastSnapshotNs != 0 && - (time.nanoseconds() - lastSnapshotTimeNs >= minTimeSinceLastSnapshotNs)) { + } else if (maxTimeSinceLastSnapshotNs != 0 && + (time.nanoseconds() - lastSnapshotTimeNs >= maxTimeSinceLastSnapshotNs)) { if (eventQueue.isEmpty()) { scheduleEmit("we have waited at least " + - TimeUnit.NANOSECONDS.toMinutes(minTimeSinceLastSnapshotNs) + " minute(s)", newImage); + TimeUnit.NANOSECONDS.toMinutes(maxTimeSinceLastSnapshotNs) + " minute(s)", newImage); } } } diff --git a/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotGeneratorTest.java b/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotGeneratorTest.java index edacba05c3d21..1c21c06e6f640 100644 --- a/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotGeneratorTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotGeneratorTest.java @@ -99,7 +99,7 @@ public void testCreateSnapshot() throws Exception { try (SnapshotGenerator generator = new SnapshotGenerator.Builder(emitter). setFaultHandler(faultHandler). setMinBytesSinceLastSnapshot(200). - setMinTimeSinceLastSnapshotNs(TimeUnit.DAYS.toNanos(10)). + setMaxTimeSinceLastSnapshotNs(TimeUnit.DAYS.toNanos(10)). build()) { // Publish a log delta batch. This one will not trigger a snapshot yet. generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, @@ -132,7 +132,7 @@ public void testSnapshotsDisabled() throws Exception { try (SnapshotGenerator generator = new SnapshotGenerator.Builder(emitter). setFaultHandler(faultHandler). setMinBytesSinceLastSnapshot(1). - setMinTimeSinceLastSnapshotNs(TimeUnit.DAYS.toNanos(10)). + setMaxTimeSinceLastSnapshotNs(0). build()) { generator.disable("we are testing disable()"); // No snapshots are generated because snapshots are disabled. @@ -155,7 +155,7 @@ public void testTimeBasedSnapshots() throws Exception { setTime(mockTime). setFaultHandler(faultHandler). setMinBytesSinceLastSnapshot(200). - setMinTimeSinceLastSnapshotNs(TimeUnit.MINUTES.toNanos(30)). + setMaxTimeSinceLastSnapshotNs(TimeUnit.MINUTES.toNanos(30)). build()) { // This image isn't published yet. generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, From 8750b7cc83fb162f99b96bff339851576510c6be Mon Sep 17 00:00:00 2001 From: "Colin P. McCabe" Date: Wed, 2 Nov 2022 14:39:04 -0700 Subject: [PATCH 05/15] DynamicConfigPublisher: ignore handlers that aren't present Ignore handlers that aren't present (like the TOPIC handler on the controller) --- .../metadata/DynamicConfigPublisher.scala | 75 ++++++++++--------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/core/src/main/scala/kafka/server/metadata/DynamicConfigPublisher.scala b/core/src/main/scala/kafka/server/metadata/DynamicConfigPublisher.scala index 723ddbd37d246..44ca513c6a6a4 100644 --- a/core/src/main/scala/kafka/server/metadata/DynamicConfigPublisher.scala +++ b/core/src/main/scala/kafka/server/metadata/DynamicConfigPublisher.scala @@ -62,49 +62,50 @@ class DynamicConfigPublisher( val props = newImage.configs().configProperties(resource) resource.`type`() match { case TOPIC => - try { - // Apply changes to a topic's dynamic configuration. - info(s"Updating topic ${resource.name()} with new configuration : " + - toLoggableProps(resource, props).mkString(",")) - dynamicConfigHandlers(ConfigType.Topic). - processConfigChanges(resource.name(), props) - } catch { - case t: Throwable => faultHandler.handleFault("Error updating topic " + - s"${resource.name()} with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + - s"in ${deltaName}", t) - } - case BROKER => - if (resource.name().isEmpty) { + dynamicConfigHandlers.get(ConfigType.Topic).foreach(topicConfigHandler => try { - // Apply changes to "cluster configs" (also known as default BROKER configs). - // These are stored in KRaft with an empty name field. - info("Updating cluster configuration : " + + // Apply changes to a topic's dynamic configuration. + info(s"Updating topic ${resource.name()} with new configuration : " + toLoggableProps(resource, props).mkString(",")) - dynamicConfigHandlers(ConfigType.Broker). - processConfigChanges(ConfigEntityName.Default, props) + topicConfigHandler.processConfigChanges(resource.name(), props) } catch { - case t: Throwable => faultHandler.handleFault("Error updating " + - s"cluster with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + + case t: Throwable => faultHandler.handleFault("Error updating topic " + + s"${resource.name()} with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + s"in ${deltaName}", t) } - } else if (resource.name() == conf.nodeId.toString) { - try { - // Apply changes to this broker's dynamic configuration. - info(s"Updating node ${conf.nodeId} with new configuration : " + - toLoggableProps(resource, props).mkString(",")) - dynamicConfigHandlers(ConfigType.Broker). - processConfigChanges(resource.name(), props) - // When applying a per broker config (not a cluster config), we also - // reload any associated file. For example, if the ssl.keystore is still - // set to /tmp/foo, we still want to reload /tmp/foo in case its contents - // have changed. This doesn't apply to topic configs or cluster configs. - reloadUpdatedFilesWithoutConfigChange(props) - } catch { - case t: Throwable => faultHandler.handleFault("Error updating " + - s"node with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + - s"in ${deltaName}", t) + ) + case BROKER => + dynamicConfigHandlers.get(ConfigType.Broker).foreach(nodeConfigHandler => + if (resource.name().isEmpty) { + try { + // Apply changes to "cluster configs" (also known as default BROKER configs). + // These are stored in KRaft with an empty name field. + info("Updating cluster configuration : " + + toLoggableProps(resource, props).mkString(",")) + nodeConfigHandler.processConfigChanges(ConfigEntityName.Default, props) + } catch { + case t: Throwable => faultHandler.handleFault("Error updating " + + s"cluster with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + + s"in ${deltaName}", t) + } + } else if (resource.name() == conf.nodeId.toString) { + try { + // Apply changes to this broker's dynamic configuration. + info(s"Updating node ${conf.nodeId} with new configuration : " + + toLoggableProps(resource, props).mkString(",")) + nodeConfigHandler.processConfigChanges(resource.name(), props) + // When applying a per broker config (not a cluster config), we also + // reload any associated file. For example, if the ssl.keystore is still + // set to /tmp/foo, we still want to reload /tmp/foo in case its contents + // have changed. This doesn't apply to topic configs or cluster configs. + reloadUpdatedFilesWithoutConfigChange(props) + } catch { + case t: Throwable => faultHandler.handleFault("Error updating " + + s"node with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + + s"in ${deltaName}", t) + } } - } + ) case _ => // nothing to do } } From 0838d35a3de74339129a2d4e49d59808913bd993 Mon Sep 17 00:00:00 2001 From: "Colin P. McCabe" Date: Wed, 2 Nov 2022 15:50:16 -0700 Subject: [PATCH 06/15] Add testMetadataVersionChange --- .../image/loader/MetadataLoaderTest.java | 113 +++++++++++++++++- 1 file changed, 108 insertions(+), 5 deletions(-) diff --git a/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java b/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java index e05d33bb437dd..6b26c1c60e2a6 100644 --- a/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java @@ -25,6 +25,7 @@ import org.apache.kafka.image.MetadataDelta; import org.apache.kafka.image.MetadataImage; import org.apache.kafka.image.MetadataProvenance; +import org.apache.kafka.image.MetadataVersionChange; import org.apache.kafka.image.publisher.MetadataPublisher; import org.apache.kafka.raft.Batch; import org.apache.kafka.raft.BatchReader; @@ -37,14 +38,19 @@ import org.junit.jupiter.api.Timeout; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.OptionalLong; import java.util.concurrent.ExecutionException; import static java.util.Arrays.asList; +import static org.apache.kafka.server.common.MetadataVersion.IBP_3_3_IV1; +import static org.apache.kafka.server.common.MetadataVersion.IBP_3_3_IV2; +import static org.apache.kafka.server.common.MetadataVersion.IBP_3_3_IV3; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -251,7 +257,7 @@ public void testRemovePublisher() throws Exception { asList(asList(new ApiMessageAndVersion( new FeatureLevelRecord(). setName(MetadataVersion.FEATURE_NAME). - setFeatureLevel(MetadataVersion.IBP_3_3_IV2.featureLevel()), (short) 0)))); + setFeatureLevel(IBP_3_3_IV2.featureLevel()), (short) 0)))); assertFalse(snapshotReader.closed); loader.handleSnapshot(snapshotReader); loader.waitForAllEventsToBeHandled(); @@ -259,7 +265,7 @@ public void testRemovePublisher() throws Exception { loader.removeAndClosePublisher(publishers.get(0)).get(); } assertTrue(publishers.get(0).closed); - assertEquals(MetadataVersion.IBP_3_3_IV2, + assertEquals(IBP_3_3_IV2, publishers.get(0).latestImage.features().metadataVersion()); assertTrue(publishers.get(1).closed); assertEquals(MetadataImage.EMPTY, publishers.get(1).latestImage); @@ -388,7 +394,7 @@ public void testLoadEmptyBatch() throws Exception { */ @Test public void testLastAppliedOffset() throws Exception { - MockFaultHandler faultHandler = new MockFaultHandler("testRemovePublisher"); + MockFaultHandler faultHandler = new MockFaultHandler("testLastAppliedOffset"); List publishers = asList(new MockPublisher(), new MockPublisher()); try (MetadataLoader loader = new MetadataLoader.Builder(). @@ -399,7 +405,7 @@ public void testLastAppliedOffset() throws Exception { new MetadataProvenance(200, 100, 4000), asList( asList(new ApiMessageAndVersion(new FeatureLevelRecord(). setName(MetadataVersion.FEATURE_NAME). - setFeatureLevel(MetadataVersion.IBP_3_3_IV1.featureLevel()), (short) 0)), + setFeatureLevel(IBP_3_3_IV1.featureLevel()), (short) 0)), asList(new ApiMessageAndVersion(new TopicRecord(). setName("foo"). setTopicId(Uuid.fromString("Uum7sfhHQP-obSvfywmNUA")), (short) 0)) @@ -416,9 +422,106 @@ public void testLastAppliedOffset() throws Exception { for (int i = 0; i < 2; i++) { assertTrue(publishers.get(i).closed); assertTrue(publishers.get(i).closed); - assertEquals(MetadataVersion.IBP_3_3_IV1, + assertEquals(IBP_3_3_IV1, publishers.get(i).latestImage.features().metadataVersion()); } faultHandler.maybeRethrowFirstException(); } + + /** + * Test metadata version changes + */ + @Test + public void testMetadataVersionChange() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("testMetadataVersionChange"); + MockTime time = new MockTime(); + MockPublisher publisher = new MockPublisher(); + try (MetadataLoader loader = new MetadataLoader.Builder(). + setFaultHandler(faultHandler). + setTime(time). + build()) { + // Starting with an empty metadata image and loading a snapshot should not trigger + // the publishPreVersionChangeImage callbacks. + loader.installPublishers(Arrays.asList(publisher)).get(); + loader.handleSnapshot(MockSnapshotReader.fromRecordLists( + new MetadataProvenance(200, 100, 4000), asList( + asList(new ApiMessageAndVersion(new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(IBP_3_3_IV1.featureLevel()), (short) 0)), + asList(new ApiMessageAndVersion(new TopicRecord(). + setName("foo"). + setTopicId(Uuid.fromString("Uum7sfhHQP-obSvfywmNUA")), (short) 0)) + )).setTime(time)); + loader.waitForAllEventsToBeHandled(); + assertEquals(200L, loader.lastAppliedOffset()); + assertNull(publisher.latestPreVersionChangeDelta); + assertNull(publisher.latestPreVersionChangeImage); + assertNull(publisher.latestPreVersionChangeManifest); + + // Loading a snapshot with a new metadata version should trigger the + // publishPreVersionChangeImage callbacks. + loader.handleSnapshot(MockSnapshotReader.fromRecordLists( + new MetadataProvenance(201, 101, 5000), asList( + asList(new ApiMessageAndVersion(new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(IBP_3_3_IV2.featureLevel()), (short) 0)), + asList(new ApiMessageAndVersion(new TopicRecord(). + setName("foo"). + setTopicId(Uuid.fromString("Uum7sfhHQP-obSvfywmNUA")), (short) 0)) + )).setTime(time)); + loader.waitForAllEventsToBeHandled(); + assertEquals(201L, loader.lastAppliedOffset()); + assertNotNull(publisher.latestPreVersionChangeDelta); + assertNotNull(publisher.latestPreVersionChangeImage); + assertEquals(IBP_3_3_IV1, publisher.latestPreVersionChangeImage.features().metadataVersion()); + assertEquals(new PreVersionChangeManifest( + new MetadataProvenance(200, 100, 4000), + new MetadataVersionChange(IBP_3_3_IV1, IBP_3_3_IV2)), + publisher.latestPreVersionChangeManifest); + + // Loading a batch of log messages with a new metadata version should also trigger the + // publishPreVersionChangeImage callbacks. + loader.handleCommit(new MockBatchReader(203, asList( + MockBatchReader.newBatch(203, 102, asList( + new ApiMessageAndVersion(new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(IBP_3_3_IV3.featureLevel()), (short) 0), + new ApiMessageAndVersion(new TopicRecord(). + setName("bar"). + setTopicId(Uuid.fromString("aN0Qt3hRQgiaxkx_raX92Q")), (short) 0))))). + setTime(time)); + loader.waitForAllEventsToBeHandled(); + assertEquals(204L, loader.lastAppliedOffset()); + assertTrue(publisher.latestPreVersionChangeDelta.topicsDelta().changedTopics(). + containsKey(Uuid.fromString("aN0Qt3hRQgiaxkx_raX92Q"))); + assertEquals(IBP_3_3_IV2, publisher.latestPreVersionChangeImage.features().metadataVersion()); + assertEquals(new PreVersionChangeManifest( + new MetadataProvenance(201, 101, 5000), + new MetadataVersionChange(IBP_3_3_IV2, IBP_3_3_IV3)), + publisher.latestPreVersionChangeManifest); + + + // Loading a batch of log messages with a new metadata version should also trigger the + // publishPreVersionChangeImage callbacks. + loader.handleCommit(new MockBatchReader(203, asList( + Batch.data(206, 103, 6000, 20, asList( + new ApiMessageAndVersion(new TopicRecord(). + setName("quux"). + setTopicId(Uuid.fromString("Q1gR0H5AQsmU_Ojh66TnAg")), (short) 0), + new ApiMessageAndVersion(new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(IBP_3_3_IV2.featureLevel()), (short) 0))))). + setTime(time)); + loader.waitForAllEventsToBeHandled(); + assertEquals(207L, loader.lastAppliedOffset()); + assertNotNull(publisher.latestPreVersionChangeDelta); + assertNotNull(publisher.latestPreVersionChangeImage); + assertEquals(IBP_3_3_IV3, publisher.latestPreVersionChangeImage.features().metadataVersion()); + assertEquals(new PreVersionChangeManifest( + new MetadataProvenance(206, 103, 6000), + new MetadataVersionChange(IBP_3_3_IV3, IBP_3_3_IV2)), + publisher.latestPreVersionChangeManifest); + } + faultHandler.maybeRethrowFirstException(); + } } From 0699322b3e1dbaa97197bc3ebde9c56dbf02c50f Mon Sep 17 00:00:00 2001 From: "Colin P. McCabe" Date: Wed, 2 Nov 2022 15:50:33 -0700 Subject: [PATCH 07/15] KafkaClusterTestKit: fix thread name prefixes --- core/src/test/java/kafka/testkit/KafkaClusterTestKit.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java index 4a0632530496c..3d65f377c6e65 100644 --- a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java +++ b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java @@ -188,7 +188,7 @@ public KafkaClusterTestKit build() throws Exception { setupNodeDirectories(baseDirectory, node.metadataDirectory(), Collections.emptyList()); KafkaConfig config = new KafkaConfig(props, false, Option.empty()); - String threadNamePrefix = String.format("controller%d_", node.id()); + String threadNamePrefix = String.format("controller%d", node.id()); MetaProperties metaProperties = MetaProperties.apply(nodes.clusterId().toString(), node.id()); TopicPartition metadataPartition = new TopicPartition(KafkaRaftServer.MetadataTopic(), 0); BootstrapMetadata bootstrapMetadata = BootstrapMetadata. From af85374ae4d97862e1a6661228d69453fa1f504f Mon Sep 17 00:00:00 2001 From: "Colin P. McCabe" Date: Wed, 2 Nov 2022 16:18:50 -0700 Subject: [PATCH 08/15] KRaftClusterTest: set higher values for max.connections.per.ip --- .../scala/integration/kafka/server/KRaftClusterTest.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala b/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala index f52c2a72da25c..36996314df32e 100644 --- a/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala +++ b/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala @@ -559,10 +559,10 @@ class KRaftClusterTest { assertEquals(Seq(ApiError.NONE), incrementalAlter(admin, Seq( (new ConfigResource(Type.BROKER, ""), Seq( new AlterConfigOp(new ConfigEntry("log.roll.ms", "1234567"), OpType.SET), - new AlterConfigOp(new ConfigEntry("max.connections.per.ip", "6"), OpType.SET)))))) + new AlterConfigOp(new ConfigEntry("max.connections.per.ip", "600"), OpType.SET)))))) validateConfigs(admin, Map(new ConfigResource(Type.BROKER, "") -> Seq( ("log.roll.ms", "1234567"), - ("max.connections.per.ip", "6"))), true) + ("max.connections.per.ip", "600"))), true) admin.createTopics(Arrays.asList( new NewTopic("foo", 2, 3.toShort), @@ -588,10 +588,10 @@ class KRaftClusterTest { assertEquals(Seq(ApiError.NONE), incrementalAlter(admin, Seq( (new ConfigResource(Type.BROKER, "2"), Seq( - new AlterConfigOp(new ConfigEntry("max.connections.per.ip", "7"), OpType.SET)))))) + new AlterConfigOp(new ConfigEntry("max.connections.per.ip", "700"), OpType.SET)))))) validateConfigs(admin, Map(new ConfigResource(Type.BROKER, "2") -> Seq( - ("max.connections.per.ip", "7")))) + ("max.connections.per.ip", "700")))) } finally { admin.close() } From b8d6c005d1934161742d621eb514674f64869f02 Mon Sep 17 00:00:00 2001 From: "Colin P. McCabe" Date: Thu, 3 Nov 2022 02:51:47 -0700 Subject: [PATCH 09/15] JointServer refactor --- .../scala/kafka/server/BrokerServer.scala | 55 ++-- .../scala/kafka/server/ControllerServer.scala | 73 +++-- .../main/scala/kafka/server/JointServer.scala | 269 ++++++++++++++++++ .../scala/kafka/server/KafkaRaftServer.scala | 109 +------ .../kafka/testkit/KafkaClusterTestKit.java | 215 +++++--------- .../kafka/server/KRaftClusterTest.scala | 6 +- .../kafka/server/QuorumTestHarness.scala | 94 ++---- .../server/RaftClusterSnapshotTest.scala | 23 +- .../BrokerMetadataPublisherTest.scala | 11 +- .../scala/unit/kafka/utils/TestUtils.scala | 2 +- 10 files changed, 454 insertions(+), 403 deletions(-) create mode 100644 core/src/main/scala/kafka/server/JointServer.scala diff --git a/core/src/main/scala/kafka/server/BrokerServer.scala b/core/src/main/scala/kafka/server/BrokerServer.scala index 34845f749e951..dec2adff9c2be 100644 --- a/core/src/main/scala/kafka/server/BrokerServer.scala +++ b/core/src/main/scala/kafka/server/BrokerServer.scala @@ -27,28 +27,24 @@ import kafka.coordinator.group.GroupCoordinator import kafka.coordinator.transaction.{ProducerIdManager, TransactionCoordinator} import kafka.log.LogManager import kafka.network.{DataPlaneAcceptor, SocketServer} -import kafka.raft.RaftManager +import kafka.raft.KafkaRaftManager import kafka.security.CredentialProvider import kafka.server.metadata.{BrokerMetadataPublisher, ClientQuotaMetadataManager, DynamicConfigPublisher, KRaftMetadataCache} import kafka.utils.{CoreUtils, KafkaScheduler} import org.apache.kafka.common.feature.SupportedVersionRange import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.BrokerRegistrationRequestData.{Listener, ListenerCollection} -import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.security.scram.internals.ScramMechanism import org.apache.kafka.common.security.token.delegation.internals.DelegationTokenCache import org.apache.kafka.common.utils.{AppInfoParser, LogContext, Time, Utils} import org.apache.kafka.common.{ClusterResource, Endpoint} -import org.apache.kafka.image.loader.MetadataLoader import org.apache.kafka.metadata.authorizer.ClusterMetadataAuthorizer import org.apache.kafka.metadata.{BrokerState, VersionRange} -import org.apache.kafka.raft.RaftConfig.AddressSpec import org.apache.kafka.raft.RaftConfig import org.apache.kafka.server.authorizer.Authorizer import org.apache.kafka.server.common.ApiMessageAndVersion -import org.apache.kafka.server.fault.FaultHandler import org.apache.kafka.server.metrics.KafkaYammerMetrics import scala.collection.{Map, Seq} @@ -59,18 +55,14 @@ import scala.jdk.CollectionConverters._ * A Kafka broker that runs in KRaft (Kafka Raft) mode. */ class BrokerServer( - val config: KafkaConfig, - val metaProps: MetaProperties, - val raftManager: RaftManager[ApiMessageAndVersion], - val time: Time, - val metrics: Metrics, - val threadNamePrefix: Option[String], + val jointServer: JointServer, val initialOfflineDirs: Seq[String], - val controllerQuorumVotersFuture: CompletableFuture[util.Map[Integer, AddressSpec]], - val fatalFaultHandler: FaultHandler, - val metadataPublishingFaultHandler: FaultHandler, - val metadataLoader: MetadataLoader, ) extends KafkaBroker { + val threadNamePrefix = jointServer.threadNamePrefix + val config = jointServer.config + val time = jointServer.time + val metrics = jointServer.metrics + def raftManager: KafkaRaftManager[ApiMessageAndVersion] = jointServer.raftManager class Builder @@ -132,7 +124,7 @@ class BrokerServer( @volatile var brokerTopicStats: BrokerTopicStats = _ - val clusterId: String = metaProps.clusterId + val clusterId: String = jointServer.metaProps.clusterId var metadataPublisher: BrokerMetadataPublisher = _ @@ -164,9 +156,9 @@ class BrokerServer( override def startup(): Unit = { if (!maybeChangeStatus(SHUTDOWN, STARTING)) return try { - info("Starting broker") + jointServer.startForBroker() - config.dynamicConfig.initialize(zkClientOpt = None) + info("Starting broker") lifecycleManager = new BrokerLifecycleManager(config, time, @@ -195,8 +187,8 @@ class BrokerServer( tokenCache = new DelegationTokenCache(ScramMechanism.mechanismNames) credentialProvider = new CredentialProvider(ScramMechanism.mechanismNames, tokenCache) - val controllerNodes = RaftConfig.voterConnectionsToNodes(controllerQuorumVotersFuture.get()).asScala - val controllerNodeProvider = RaftControllerNodeProvider(raftManager, config, controllerNodes) + val controllerNodes = RaftConfig.voterConnectionsToNodes(jointServer.controllerQuorumVotersFuture.get()).asScala + val controllerNodeProvider = RaftControllerNodeProvider(jointServer.raftManager, config, controllerNodes) clientToControllerChannelManager = BrokerToControllerChannelManager( controllerNodeProvider, @@ -312,9 +304,9 @@ class BrokerServer( config.brokerSessionTimeoutMs.toLong ) lifecycleManager.start( - () => metadataLoader.lastAppliedOffset(), + () => jointServer.metadataLoader.lastAppliedOffset(), brokerLifecycleChannelManager, - metaProps.clusterId, + clusterId, networkListeners, featuresRemapped ) @@ -390,13 +382,14 @@ class BrokerServer( // Block until we've caught up with the latest metadata from the controller quorum. info("Waiting for broker metadata to catch up.") try { - lifecycleManager.initialCatchUpFuture.get() + lifecycleManager.initialCatchUpFuture.get(20, TimeUnit.SECONDS) } catch { case t: Throwable => throw new RuntimeException("Received a fatal error while " + "waiting for the broker to catch up with the current cluster metadata.", t) } // Apply the metadata log changes that we've accumulated. + val metadataPublishingFaultHandler = jointServer.metadataPublishingFaultHandler metadataPublisher = new BrokerMetadataPublisher(config, metadataCache, logManager, @@ -409,7 +402,7 @@ class BrokerServer( metadataPublishingFaultHandler, dynamicConfigHandlers.toMap), authorizer, - fatalFaultHandler, + jointServer.initialBrokerMetadataLoadFaultHandler, metadataPublishingFaultHandler) // Tell the metadata listener to start publishing its output, and wait for the first @@ -417,7 +410,7 @@ class BrokerServer( // replicaManager, groupCoordinator, and txnCoordinator. The log manager may perform // a potentially lengthy recovery-from-unclean-shutdown operation here, if required. try { - metadataLoader.installPublishers(List(metadataPublisher).asJava).get() + jointServer.metadataLoader.installPublishers(List(metadataPublisher).asJava).get(20, TimeUnit.SECONDS) } catch { case t: Throwable => throw new RuntimeException("Received a fatal error while " + "publishing the initial metadata update.", t) @@ -441,7 +434,7 @@ class BrokerServer( // We're now ready to unfence the broker. This also allows this broker to transition // from RECOVERY state to RUNNING state, once the controller unfences the broker. try { - lifecycleManager.setReadyToUnfence().get() + lifecycleManager.setReadyToUnfence().get(20, TimeUnit.SECONDS) } catch { case t: Throwable => throw new RuntimeException("Received a fatal error while " + "waiting for the broker to be unfenced.", t) @@ -477,8 +470,6 @@ class BrokerServer( } } - metadataLoader.beginShutdown() - lifecycleManager.beginShutdown() // Stop socket server to stop accepting any more connections and requests. @@ -493,7 +484,6 @@ class BrokerServer( if (controlPlaneRequestProcessor != null) CoreUtils.swallow(controlPlaneRequestProcessor.close(), this) CoreUtils.swallow(authorizer.foreach(_.close()), this) - CoreUtils.swallow(metadataLoader.close(), this) /** * We must shutdown the scheduler early because otherwise, the scheduler could touch other @@ -531,19 +521,18 @@ class BrokerServer( if (quotaManagers != null) CoreUtils.swallow(quotaManagers.shutdown(), this) - if (socketServer != null) + if (socketServer != null) { CoreUtils.swallow(socketServer.shutdown(), this) + } if (metrics != null) CoreUtils.swallow(metrics.close(), this) if (brokerTopicStats != null) CoreUtils.swallow(brokerTopicStats.close(), this) - // Clear all reconfigurable instances stored in DynamicBrokerConfig - config.dynamicConfig.clear() - isShuttingDown.set(false) CoreUtils.swallow(lifecycleManager.close(), this) + jointServer.stopForBroker() CoreUtils.swallow(AppInfoParser.unregisterAppInfo(MetricsPrefix, config.nodeId.toString, metrics), this) info("shut down completed") diff --git a/core/src/main/scala/kafka/server/ControllerServer.scala b/core/src/main/scala/kafka/server/ControllerServer.scala index 99007421b4b89..3a626e1ccbe27 100644 --- a/core/src/main/scala/kafka/server/ControllerServer.scala +++ b/core/src/main/scala/kafka/server/ControllerServer.scala @@ -17,38 +17,32 @@ package kafka.server -import java.util import java.util.OptionalLong import java.util.concurrent.locks.ReentrantLock import java.util.concurrent.{CompletableFuture, TimeUnit} import kafka.cluster.Broker.ServerInfo import kafka.metrics.{KafkaMetricsGroup, LinuxIoMetricsCollector} import kafka.network.{DataPlaneAcceptor, SocketServer} -import kafka.raft.RaftManager +import kafka.raft.KafkaRaftManager import kafka.security.CredentialProvider import kafka.server.KafkaConfig.{AlterConfigPolicyClassNameProp, CreateTopicPolicyClassNameProp} import kafka.server.KafkaRaftServer.BrokerRole import kafka.server.QuotaFactory.QuotaManagers import kafka.server.metadata.DynamicConfigPublisher import kafka.utils.{CoreUtils, Logging} -import org.apache.kafka.clients.ApiVersions import org.apache.kafka.common.message.ApiMessageType.ListenerType -import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.security.scram.internals.ScramMechanism import org.apache.kafka.common.security.token.delegation.internals.DelegationTokenCache -import org.apache.kafka.common.utils.{LogContext, Time} +import org.apache.kafka.common.utils.LogContext import org.apache.kafka.common.{ClusterResource, Endpoint} -import org.apache.kafka.controller.{Controller, ControllerMetrics, QuorumController, QuorumFeatures} +import org.apache.kafka.controller.{Controller, QuorumController, QuorumControllerMetrics, QuorumFeatures} import org.apache.kafka.metadata.KafkaConfigSchema import org.apache.kafka.raft.RaftConfig -import org.apache.kafka.raft.RaftConfig.AddressSpec import org.apache.kafka.server.authorizer.Authorizer -import org.apache.kafka.server.common.ApiMessageAndVersion import org.apache.kafka.common.config.ConfigException -import org.apache.kafka.image.loader.MetadataLoader import org.apache.kafka.metadata.authorizer.ClusterMetadataAuthorizer import org.apache.kafka.metadata.bootstrap.BootstrapMetadata -import org.apache.kafka.server.fault.FaultHandler +import org.apache.kafka.server.common.ApiMessageAndVersion import org.apache.kafka.server.metrics.KafkaYammerMetrics import org.apache.kafka.server.policy.{AlterConfigPolicy, CreateTopicPolicy} @@ -60,24 +54,16 @@ import scala.compat.java8.OptionConverters._ * A Kafka controller that runs in KRaft (Kafka Raft) mode. */ class ControllerServer( - val metaProperties: MetaProperties, - val config: KafkaConfig, - val raftManager: RaftManager[ApiMessageAndVersion], - val time: Time, - val metrics: Metrics, - val controllerMetrics: ControllerMetrics, - val threadNamePrefix: Option[String], - val controllerQuorumVotersFuture: CompletableFuture[util.Map[Integer, AddressSpec]], + val jointServer: JointServer, val configSchema: KafkaConfigSchema, - val raftApiVersions: ApiVersions, val bootstrapMetadata: BootstrapMetadata, - val metadataFaultHandler: FaultHandler, - val fatalFaultHandler: FaultHandler, - val metadataLoader: MetadataLoader, ) extends Logging with KafkaMetricsGroup { - import kafka.server.Server._ + val config = jointServer.config + val time = jointServer.time + val metrics = jointServer.metrics + def raftManager: KafkaRaftManager[ApiMessageAndVersion] = jointServer.raftManager - config.dynamicConfig.initialize(zkClientOpt = None) + import kafka.server.Server._ val lock = new ReentrantLock() val awaitShutdownCond = lock.newCondition() @@ -91,6 +77,7 @@ class ControllerServer( val socketServerFirstBoundPortFuture = new CompletableFuture[Integer]() var createTopicPolicy: Option[CreateTopicPolicy] = None var alterConfigPolicy: Option[AlterConfigPolicy] = None + var controllerMetrics: QuorumControllerMetrics = _ var controller: Controller = _ var quotaManagers: QuotaManagers = _ var controllerApis: ControllerApis = _ @@ -117,11 +104,13 @@ class ControllerServer( new DynamicMetricReporterState(config.nodeId, config, metrics, clusterId) } - def clusterId: String = metaProperties.clusterId + def clusterId: String = jointServer.metaProps.clusterId def startup(): Unit = { if (!maybeChangeStatus(SHUTDOWN, STARTING)) return try { + jointServer.startForController() + info("Starting controller") maybeChangeStatus(STARTING, STARTED) @@ -177,15 +166,18 @@ class ControllerServer( throw new ConfigException("No controller.listener.names defined for controller") } - val threadNamePrefixAsString = threadNamePrefix.getOrElse("") + val threadNamePrefixAsString = jointServer.threadNamePrefix.getOrElse("") createTopicPolicy = Option(config. getConfiguredInstance(CreateTopicPolicyClassNameProp, classOf[CreateTopicPolicy])) alterConfigPolicy = Option(config. getConfiguredInstance(AlterConfigPolicyClassNameProp, classOf[AlterConfigPolicy])) - val controllerNodes = RaftConfig.voterConnectionsToNodes(controllerQuorumVotersFuture.get()) - val quorumFeatures = QuorumFeatures.create(config.nodeId, raftApiVersions, QuorumFeatures.defaultFeatureMap(), controllerNodes) + val controllerNodes = RaftConfig.voterConnectionsToNodes(jointServer.controllerQuorumVotersFuture.get()) + val quorumFeatures = QuorumFeatures.create(config.nodeId, + jointServer.raftManager.apiVersions, + QuorumFeatures.defaultFeatureMap(), + controllerNodes) val controllerBuilder = { val leaderImbalanceCheckIntervalNs = if (config.autoLeaderRebalanceEnable) { @@ -196,11 +188,11 @@ class ControllerServer( val maxIdleIntervalNs = config.metadataMaxIdleIntervalNs.fold(OptionalLong.empty)(OptionalLong.of) - new QuorumController.Builder(config.nodeId, metaProperties.clusterId). + new QuorumController.Builder(config.nodeId, jointServer.metaProps.clusterId). setTime(time). setThreadNamePrefix(threadNamePrefixAsString). setConfigSchema(configSchema). - setRaftClient(raftManager.client). + setRaftClient(jointServer.raftManager.client). setQuorumFeatures(quorumFeatures). setDefaultReplicationFactor(config.defaultReplicationFactor.toShort). setDefaultNumPartitions(config.numPartitions.intValue()). @@ -208,13 +200,13 @@ class ControllerServer( TimeUnit.MILLISECONDS)). setLeaderImbalanceCheckIntervalNs(leaderImbalanceCheckIntervalNs). setMaxIdleIntervalNs(maxIdleIntervalNs). - setMetrics(controllerMetrics). + setMetrics(jointServer.controllerMetrics). setCreateTopicPolicy(createTopicPolicy.asJava). setAlterConfigPolicy(alterConfigPolicy.asJava). setConfigurationValidator(new ControllerConfigurationValidator()). setStaticConfig(config.originals). setBootstrapMetadata(bootstrapMetadata). - setFatalFaultHandler(fatalFaultHandler) + setFatalFaultHandler(jointServer.quorumControllerFaultHandler) } authorizer match { case Some(a: ClusterMetadataAuthorizer) => controllerBuilder.setAuthorizer(a) @@ -227,15 +219,15 @@ class ControllerServer( doRemoteKraftSetup() } - quotaManagers = QuotaFactory.instantiate(config, metrics, time, threadNamePrefix.getOrElse("")) + quotaManagers = QuotaFactory.instantiate(config, metrics, time, jointServer.threadNamePrefix.getOrElse("")) controllerApis = new ControllerApis(socketServer.dataPlaneRequestChannel, authorizer, quotaManagers, time, controller, - raftManager, + jointServer.raftManager, config, - metaProperties, + jointServer.metaProps, controllerNodes.asScala.toSeq, apiVersionManager) controllerApisHandlerPool = new KafkaRequestHandlerPool(config.nodeId, @@ -271,9 +263,9 @@ class ControllerServer( ConfigType.Broker -> new BrokerConfigHandler(config, quotaManagers)) dynamicConfigPublisher = new DynamicConfigPublisher( config, - fatalFaultHandler, + jointServer.metadataPublishingFaultHandler, dynamicConfigHandlers) - metadataLoader.installPublishers(List(dynamicConfigPublisher).asJava) + jointServer.metadataLoader.installPublishers(List(dynamicConfigPublisher).asJava) } } @@ -286,7 +278,10 @@ class ControllerServer( if (controller != null) { controller.beginShutdown() } - metadataLoader.beginShutdown() + if (controllerMetrics != null) { + CoreUtils.swallow(controllerApis.close(), this) + controllerMetrics = null + } if (socketServer != null) CoreUtils.swallow(socketServer.shutdown(), this) if (controllerApisHandlerPool != null) @@ -297,11 +292,11 @@ class ControllerServer( CoreUtils.swallow(quotaManagers.shutdown(), this) if (controller != null) controller.close() - CoreUtils.swallow(metadataLoader.close(), this) CoreUtils.swallow(authorizer.foreach(_.close()), this) createTopicPolicy.foreach(policy => CoreUtils.swallow(policy.close(), this)) alterConfigPolicy.foreach(policy => CoreUtils.swallow(policy.close(), this)) socketServerFirstBoundPortFuture.completeExceptionally(new RuntimeException("shutting down")) + jointServer.stopForController() } catch { case e: Throwable => fatal("Fatal error during controller shutdown.", e) diff --git a/core/src/main/scala/kafka/server/JointServer.scala b/core/src/main/scala/kafka/server/JointServer.scala new file mode 100644 index 0000000000000..949bc39f946cd --- /dev/null +++ b/core/src/main/scala/kafka/server/JointServer.scala @@ -0,0 +1,269 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.server + +import kafka.raft.KafkaRaftManager +import kafka.server.KafkaRaftServer.{BrokerRole, ControllerRole} +import kafka.server.metadata.BrokerServerMetrics +import kafka.utils.{CoreUtils, Logging} +import org.apache.kafka.common.metrics.Metrics +import org.apache.kafka.common.utils.{LogContext, Time} +import org.apache.kafka.controller.QuorumControllerMetrics +import org.apache.kafka.image.loader.MetadataLoader +import org.apache.kafka.image.publisher.{SnapshotEmitter, SnapshotGenerator} +import org.apache.kafka.metadata.MetadataRecordSerde +import org.apache.kafka.raft.RaftConfig.AddressSpec +import org.apache.kafka.server.common.ApiMessageAndVersion +import org.apache.kafka.server.fault.{FaultHandler, LoggingFaultHandler, ProcessExitingFaultHandler} +import org.apache.kafka.server.metrics.KafkaYammerMetrics + +import java.util +import java.util.concurrent.{CompletableFuture, TimeUnit} + + +/** + * Creates a fault handler. + */ +trait FaultHandlerFactory { + def build( + name: String, + fatal: Boolean, + action: Runnable + ): FaultHandler +} + +/** + * The standard FaultHandlerFactory which is used when we're not in a junit test. + */ +class StandardFaultHandlerFactory extends FaultHandlerFactory { + override def build( + name: String, + fatal: Boolean, + action: Runnable + ): FaultHandler = { + if (fatal) { + new ProcessExitingFaultHandler(action) + } else { + new LoggingFaultHandler(name, action) + } + } +} + +/** + * The JointServer manages the components which are shared between the BrokerServer and + * ControllerServer. These shared components include the Raft manager, snapshot generator, + * and metadata loader. A KRaft server running in combined mode as both a broker and a controller + * will still contain only a single JointServer instance. + * + * The JointServer will be started as soon as either the broker or the controller needs it, + * via the appropriate function (startForBroker or startForController). Similarly, it will be + * stopped as soon as neither the broker nor the controller need it, via stopForBroker or + * stopForController. One way of thinking about this is that both the broker and the controller + * could hold a "reference" to this class, and we don't truly stop it until both have dropped + * their reference. We opted to use two booleans here rather than a reference count in order to + * make debugging easier and reduce the chance of resource leaks. + */ +class JointServer( + val config: KafkaConfig, + val metaProps: MetaProperties, + val time: Time, + val metrics: Metrics, + val threadNamePrefix: Option[String], + val controllerQuorumVotersFuture: CompletableFuture[util.Map[Integer, AddressSpec]], + val faultHandlerFactory: FaultHandlerFactory +) extends Logging { + private val logContext: LogContext = new LogContext(s"[JointServer id=${config.nodeId}] ") + this.logIdent = logContext.logPrefix + + var usedByBroker: Boolean = false + var usedByController: Boolean = false + var raftManager: KafkaRaftManager[ApiMessageAndVersion] = _ + var snapshotEmitter: SnapshotEmitter = _ + var snapshotGenerator: SnapshotGenerator = _ + var brokerMetrics: BrokerServerMetrics = _ + var controllerMetrics: QuorumControllerMetrics = _ + var metadataLoader: MetadataLoader = _ + + def isUsed(): Boolean = synchronized { + usedByController || usedByBroker + } + + /** + * The start function called by the broker. + */ + def startForBroker(): Unit = synchronized { + if (!isUsed()) { + start() + } + usedByBroker = true + } + + /** + * The start function called by the controller. + */ + def startForController(): Unit = synchronized { + if (!isUsed()) { + start() + } + usedByController = true + } + + /** + * The stop function called by the broker. + */ + def stopForBroker(): Unit = synchronized { + if (usedByBroker) { + usedByBroker = false + if (!isUsed()) stop() + } + } + + /** + * The stop function called by the controller. + */ + def stopForController(): Unit = synchronized { + if (usedByController) { + usedByController = false + if (!isUsed()) stop() + } + } + + /** + * The fault handler to use when metadata loading fails. + */ + def metadataLoaderFaultHandler: FaultHandler = faultHandlerFactory.build("metadata loading", + config.processRoles.contains(ControllerRole), + () => { + if (snapshotGenerator != null) { + snapshotGenerator.disable("there was a metadata loading error") + } + if (brokerMetrics != null) brokerMetrics.metadataLoadErrorCount.getAndIncrement() + if (controllerMetrics != null) controllerMetrics.incrementMetadataErrorCount() + }) + + /** + * The fault handler to use when the initial broker metadata load fails. + */ + def initialBrokerMetadataLoadFaultHandler: FaultHandler = faultHandlerFactory.build("initial metadata loading", + true, + () => { + if (snapshotGenerator != null) { + snapshotGenerator.disable("there was an initial metadata loading error") + } + if (brokerMetrics != null) brokerMetrics.metadataApplyErrorCount.getAndIncrement() + if (controllerMetrics != null) controllerMetrics.incrementMetadataErrorCount() + }) + + /** + * The fault handler to use when the QuorumController experiences a fault. + */ + def quorumControllerFaultHandler: FaultHandler = faultHandlerFactory.build("quorum controller", + true, + () => {} + ) + + /** + * The fault handler to use when metadata cannot be published. + */ + def metadataPublishingFaultHandler: FaultHandler = faultHandlerFactory.build("metadata publishing", + false, + () => { + if (brokerMetrics != null) brokerMetrics.metadataApplyErrorCount.getAndIncrement() + if (controllerMetrics != null) controllerMetrics.incrementMetadataErrorCount() + }) + + private def start(): Unit = synchronized { + info("Starting JointServer") + try { + config.dynamicConfig.initialize(zkClientOpt = None) + + if (config.processRoles.contains(BrokerRole)) { + brokerMetrics = BrokerServerMetrics(metrics) + } + if (config.processRoles.contains(ControllerRole)) { + controllerMetrics = new QuorumControllerMetrics(KafkaYammerMetrics.defaultRegistry(), time) + } + raftManager = new KafkaRaftManager[ApiMessageAndVersion]( + metaProps, + config, + new MetadataRecordSerde, + KafkaRaftServer.MetadataPartition, + KafkaRaftServer.MetadataTopicId, + time, + metrics, + threadNamePrefix, + controllerQuorumVotersFuture) + snapshotEmitter = new SnapshotEmitter.Builder(). + setNodeId(config.nodeId). + setRaftClient(raftManager.client). + build() + snapshotGenerator = new SnapshotGenerator.Builder(snapshotEmitter). + setTime(time). + setNodeId(config.nodeId). + setFaultHandler(new LoggingFaultHandler("snapshot generation", () => { })). + setMinBytesSinceLastSnapshot(config.metadataSnapshotMaxNewRecordBytes). + setMaxTimeSinceLastSnapshotNs(TimeUnit.MILLISECONDS.toNanos(config.metadataSnapshotMaxIntervalMsProp)). + build() + val loaderBuilder = new MetadataLoader.Builder(). + setTime(time). + setNodeId(config.nodeId). + setFaultHandler(metadataLoaderFaultHandler) + if (config.processRoles.contains(BrokerRole)) { + // In broker-only mode or combined mode, we want to expose the broker metrics. + loaderBuilder.setMetadataLoaderMetrics(brokerMetrics) + } + metadataLoader = loaderBuilder.build() + raftManager.client.register(metadataLoader) + } catch { + case e: Throwable => stop() + } + } + + private def stop(): Unit = { + info("Stopping JointServer") + + // Begin shutting these down in parallel to save time. + if (metadataLoader != null) metadataLoader.beginShutdown() + if (snapshotGenerator != null) snapshotGenerator.beginShutdown() + + // Close and null out components in reverse order of construction. + if (metadataLoader != null) { + CoreUtils.swallow(metadataLoader.close(), this) + metadataLoader = null + } + if (snapshotGenerator != null) { + CoreUtils.swallow(snapshotGenerator.close(), this) + snapshotGenerator = null + } + snapshotEmitter = null + if (raftManager != null) { + CoreUtils.swallow(raftManager.shutdown(), this) + raftManager = null + } + if (controllerMetrics != null) { + CoreUtils.swallow(controllerMetrics.close(), this) + controllerMetrics = null + } + if (brokerMetrics != null) { + CoreUtils.swallow(brokerMetrics.close(), this) + brokerMetrics = null + } + // Clear all reconfigurable instances stored in DynamicBrokerConfig + config.dynamicConfig.clear() + } +} diff --git a/core/src/main/scala/kafka/server/KafkaRaftServer.scala b/core/src/main/scala/kafka/server/KafkaRaftServer.scala index 7036f7709ccf9..68f958ed9036c 100644 --- a/core/src/main/scala/kafka/server/KafkaRaftServer.scala +++ b/core/src/main/scala/kafka/server/KafkaRaftServer.scala @@ -17,26 +17,19 @@ package kafka.server import java.io.File -import java.util.concurrent.{CompletableFuture, TimeUnit} +import java.util.concurrent.CompletableFuture import kafka.common.InconsistentNodeIdException import kafka.log.{LogConfig, UnifiedLog} import kafka.metrics.KafkaMetricsReporter -import kafka.raft.KafkaRaftManager import kafka.server.KafkaRaftServer.{BrokerRole, ControllerRole} -import kafka.server.metadata.BrokerServerMetrics import kafka.utils.{CoreUtils, Logging, Mx4jLoader, VerifiableProperties} import org.apache.kafka.common.config.{ConfigDef, ConfigResource} import org.apache.kafka.common.internals.Topic import org.apache.kafka.common.utils.{AppInfoParser, Time} import org.apache.kafka.common.{KafkaException, Uuid} -import org.apache.kafka.controller.QuorumControllerMetrics -import org.apache.kafka.image.loader.MetadataLoader -import org.apache.kafka.image.publisher.{SnapshotEmitter, SnapshotGenerator} import org.apache.kafka.metadata.bootstrap.{BootstrapDirectory, BootstrapMetadata} -import org.apache.kafka.metadata.{KafkaConfigSchema, MetadataRecordSerde} +import org.apache.kafka.metadata.KafkaConfigSchema import org.apache.kafka.raft.RaftConfig -import org.apache.kafka.server.common.ApiMessageAndVersion -import org.apache.kafka.server.fault.{LoggingFaultHandler, ProcessExitingFaultHandler} import org.apache.kafka.server.metrics.KafkaYammerMetrics import java.util.Optional @@ -71,105 +64,30 @@ class KafkaRaftServer( private val controllerQuorumVotersFuture = CompletableFuture.completedFuture( RaftConfig.parseVoterConnections(config.quorumVoters)) - private val raftManager = new KafkaRaftManager[ApiMessageAndVersion]( - metaProps, + private val jointServer = new JointServer( config, - new MetadataRecordSerde, - KafkaRaftServer.MetadataPartition, - KafkaRaftServer.MetadataTopicId, + metaProps, time, metrics, threadNamePrefix, - controllerQuorumVotersFuture + controllerQuorumVotersFuture, + new StandardFaultHandlerFactory(), ) - private val snapshotEmitter: SnapshotEmitter = new SnapshotEmitter.Builder(). - setNodeId(config.nodeId). - setRaftClient(raftManager.client). - build() - - private val snapshotGenerator: SnapshotGenerator = new SnapshotGenerator.Builder(snapshotEmitter). - setTime(time). - setNodeId(config.nodeId). - setFaultHandler(new LoggingFaultHandler("snapshot generation", () => { })). - setMinBytesSinceLastSnapshot(config.metadataSnapshotMaxNewRecordBytes). - setMaxTimeSinceLastSnapshotNs(TimeUnit.MILLISECONDS.toNanos(config.metadataSnapshotMaxIntervalMsProp)). - build() - - private val brokerMetrics: Option[BrokerServerMetrics] = if (config.processRoles.contains(BrokerRole)) { - Some(BrokerServerMetrics(metrics)) - } else { - None - } - - private val metadataLoader: MetadataLoader = { - val builder = new MetadataLoader.Builder(). - setTime(time). - setNodeId(config.nodeId) - if (config.processRoles.contains(ControllerRole)) { - // Controllers and combined nodes must exit immediately if there is a metadata loading error. - // We disable snapshot generation here so that we don't create a snapshot while exiting. - builder.setFaultHandler(new ProcessExitingFaultHandler(() => { - snapshotGenerator.disable("there was a metadata loading error") - })) - } else { - // Broker-only nodes disable snapshot creation and increment a metric if there is a metadata - // loading error. - builder.setFaultHandler(new LoggingFaultHandler("metadata loading", () => { - snapshotGenerator.disable("there was a metadata loading error") - brokerMetrics.get.metadataLoadErrorCount.getAndIncrement() - })) - } - if (config.processRoles.contains(BrokerRole)) { - // In broker-only mode or combined mode, we want to expose the broker metrics. - builder.setMetadataLoaderMetrics(brokerMetrics.get) - } - builder.build() - } - - raftManager.register(metadataLoader) - private val broker: Option[BrokerServer] = if (config.processRoles.contains(BrokerRole)) { - val fatalFaultHandler = new ProcessExitingFaultHandler() - val metadataApplyingFaultHandler = new LoggingFaultHandler("metadata application", - () => brokerMetrics.get.metadataApplyErrorCount.getAndIncrement()) Some(new BrokerServer( - config, - metaProps, - raftManager, - time, - metrics, - threadNamePrefix, + jointServer, offlineDirs, - controllerQuorumVotersFuture, - fatalFaultHandler, - metadataApplyingFaultHandler, - metadataLoader, )) } else { None } private val controller: Option[ControllerServer] = if (config.processRoles.contains(ControllerRole)) { - val controllerMetrics = new QuorumControllerMetrics(KafkaYammerMetrics.defaultRegistry(), time) - val metadataFaultHandler = new LoggingFaultHandler("controller metadata", - () => controllerMetrics.incrementMetadataErrorCount()) - val fatalFaultHandler = new ProcessExitingFaultHandler() Some(new ControllerServer( - metaProps, - config, - raftManager, - time, - metrics, - controllerMetrics, - threadNamePrefix, - controllerQuorumVotersFuture, + jointServer, KafkaRaftServer.configSchema, - raftManager.apiVersions, bootstrapMetadata, - metadataFaultHandler, - fatalFaultHandler, - metadataLoader, )) } else { None @@ -177,9 +95,6 @@ class KafkaRaftServer( override def startup(): Unit = { Mx4jLoader.maybeLoad() - // Note that we startup `RaftManager` first so that the controller and broker - // can register listeners during initialization. - raftManager.startup() controller.foreach(_.startup()) broker.foreach(_.startup()) AppInfoParser.registerAppInfo(Server.MetricsPrefix, config.brokerId.toString, metrics, time.milliseconds()) @@ -187,22 +102,16 @@ class KafkaRaftServer( } override def shutdown(): Unit = { + // The last component to be shut down will stop JointServer. broker.foreach(_.shutdown()) - // The order of shutdown for `RaftManager` and `ControllerServer` is backwards - // compared to `startup()`. This is because the `SocketServer` implementation that - // we rely on to receive requests is owned by `ControllerServer`, so we need it - // to stick around until graceful shutdown of `RaftManager` can be completed. - raftManager.shutdown() controller.foreach(_.shutdown()) CoreUtils.swallow(AppInfoParser.unregisterAppInfo(Server.MetricsPrefix, config.brokerId.toString, metrics), this) - } override def awaitShutdown(): Unit = { broker.foreach(_.awaitShutdown()) controller.foreach(_.awaitShutdown()) } - } object KafkaRaftServer { diff --git a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java index 3d65f377c6e65..971b832b88541 100644 --- a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java +++ b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java @@ -17,9 +17,10 @@ package kafka.testkit; -import kafka.raft.KafkaRaftManager; import kafka.server.BrokerServer; import kafka.server.ControllerServer; +import kafka.server.FaultHandlerFactory; +import kafka.server.JointServer; import kafka.server.KafkaConfig; import kafka.server.KafkaConfig$; import kafka.server.KafkaRaftServer; @@ -35,16 +36,10 @@ import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.controller.Controller; -import org.apache.kafka.controller.MockControllerMetrics; -import org.apache.kafka.image.loader.MetadataLoader; -import org.apache.kafka.image.publisher.SnapshotEmitter; -import org.apache.kafka.image.publisher.SnapshotGenerator; -import org.apache.kafka.metadata.MetadataRecordSerde; import org.apache.kafka.metadata.bootstrap.BootstrapMetadata; -import org.apache.kafka.raft.KafkaRaftClient; import org.apache.kafka.raft.RaftConfig; -import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.server.common.MetadataVersion; +import org.apache.kafka.server.fault.FaultHandler; import org.apache.kafka.server.fault.MockFaultHandler; import org.apache.kafka.test.TestUtils; import org.slf4j.Logger; @@ -61,14 +56,12 @@ import java.nio.file.Paths; import java.util.AbstractMap.SimpleImmutableEntry; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Optional; import java.util.Properties; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; @@ -120,11 +113,33 @@ public void close() { } } + static class SimpleFaultHandlerFactory implements FaultHandlerFactory { + private final MockFaultHandler fatalFaultHandler = new MockFaultHandler("fatalFaultHandler"); + private final MockFaultHandler nonFatalFaultHandler = new MockFaultHandler("nonFatalFaultHandler"); + + MockFaultHandler fatalFaultHandler() { + return fatalFaultHandler; + } + + MockFaultHandler nonFatalFaultHandler() { + return nonFatalFaultHandler; + } + + @Override + public FaultHandler build(String name, boolean fatal, Runnable action) { + if (fatal) { + return fatalFaultHandler; + } else { + return nonFatalFaultHandler; + + } + } + } + public static class Builder { private TestKitNodes nodes; private Map configProps = new HashMap<>(); - private MockFaultHandler metadataFaultHandler = new MockFaultHandler("metadataFaultHandler"); - private MockFaultHandler fatalFaultHandler = new MockFaultHandler("fatalFaultHandler"); + private SimpleFaultHandlerFactory faultHandlerFactory = new SimpleFaultHandlerFactory(); public Builder(TestKitNodes nodes) { this.nodes = nodes; @@ -135,15 +150,10 @@ public Builder setConfigProp(String key, String value) { return this; } - public Builder setMetadataFaultHandler(MockFaultHandler metadataFaultHandler) { - this.metadataFaultHandler = metadataFaultHandler; - return this; - } - public KafkaClusterTestKit build() throws Exception { Map controllers = new HashMap<>(); Map brokers = new HashMap<>(); - Map> raftManagers = new HashMap<>(); + Map jointServers = new HashMap<>(); String uninitializedQuorumVotersString = nodes.controllerNodes().keySet().stream(). map(controllerNode -> String.format("%d@0.0.0.0:0", controllerNode)). collect(Collectors.joining(",")); @@ -193,43 +203,21 @@ public KafkaClusterTestKit build() throws Exception { TopicPartition metadataPartition = new TopicPartition(KafkaRaftServer.MetadataTopic(), 0); BootstrapMetadata bootstrapMetadata = BootstrapMetadata. fromVersion(nodes.bootstrapMetadataVersion(), "testkit"); - KafkaRaftManager raftManager = new KafkaRaftManager<>( - metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftServer.MetadataTopicId(), - Time.SYSTEM, new Metrics(), Option.apply(threadNamePrefix), connectFutureManager.future); - MetadataLoader metadataLoader = null; + JointServer jointServer = new JointServer(config, + metaProperties, + Time.SYSTEM, + new Metrics(), + Option.apply(String.format("controller%d_", node.id())), + connectFutureManager.future, + faultHandlerFactory); ControllerServer controller = null; - SnapshotGenerator snapshotGenerator = null; try { - metadataLoader = new MetadataLoader.Builder(). - setNodeId(node.id()). - setTime(Time.SYSTEM). - setThreadNamePrefix(threadNamePrefix). - setFaultHandler(fatalFaultHandler). - build(); - raftManager.register(metadataLoader); - snapshotGenerator = maybeCreateSnapshotGenerator(node.id(), raftManager.client()); - if (snapshotGenerator != null) { - metadataLoader.installPublishers(Arrays.asList(snapshotGenerator)); - } controller = new ControllerServer( - nodes.controllerProperties(node.id()), - config, - raftManager, - Time.SYSTEM, - new Metrics(), - new MockControllerMetrics(), - Option.apply(threadNamePrefix), - connectFutureManager.future, + jointServer, KafkaRaftServer.configSchema(), - raftManager.apiVersions(), - bootstrapMetadata, - metadataFaultHandler, - fatalFaultHandler, - metadataLoader); + bootstrapMetadata); } catch (Throwable e) { log.error("Error creating controller {}", node.id(), e); - Utils.closeQuietly(metadataLoader, "metadataLoader"); - Utils.closeQuietly(snapshotGenerator, "snapshotGenerator"); if (controller != null) controller.shutdown(); throw e; } @@ -241,7 +229,7 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS connectFutureManager.registerPort(node.id(), port); } }); - raftManagers.put(node.id(), raftManager); + jointServers.put(node.id(), jointServer); } for (BrokerNode node : nodes.brokerNodes().values()) { Map props = new HashMap<>(configProps); @@ -270,49 +258,26 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS KafkaConfig config = new KafkaConfig(props, false, Option.empty()); String threadNamePrefix = String.format("broker%d_", node.id()); - MetaProperties metaProperties = MetaProperties.apply(nodes.clusterId().toString(), node.id()); - TopicPartition metadataPartition = new TopicPartition(KafkaRaftServer.MetadataTopic(), 0); - KafkaRaftManager raftManager; - if (raftManagers.containsKey(node.id())) { - raftManager = raftManagers.get(node.id()); + JointServer jointServer; + if (jointServers.containsKey(node.id())) { + jointServer = jointServers.get(node.id()); } else { - raftManager = new KafkaRaftManager<>( - metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftServer.MetadataTopicId(), - Time.SYSTEM, new Metrics(), Option.apply(threadNamePrefix), connectFutureManager.future); - raftManagers.put(node.id(), raftManager); + MetaProperties metaProperties = MetaProperties.apply(nodes.clusterId().toString(), node.id()); + jointServer = new JointServer(config, + metaProperties, + Time.SYSTEM, + new Metrics(), + Option.apply(String.format("controller%d_", node.id())), + connectFutureManager.future, + faultHandlerFactory); } - Metrics metrics = new Metrics(); - MetadataLoader metadataLoader = null; BrokerServer broker = null; - SnapshotGenerator snapshotGenerator = null; try { - metadataLoader = new MetadataLoader.Builder(). - setNodeId(node.id()). - setTime(Time.SYSTEM). - setThreadNamePrefix(threadNamePrefix). - setFaultHandler(fatalFaultHandler). - build(); - raftManager.register(metadataLoader); - snapshotGenerator = maybeCreateSnapshotGenerator(node.id(), raftManager.client()); - if (snapshotGenerator != null) { - metadataLoader.installPublishers(Arrays.asList(snapshotGenerator)); - } broker = new BrokerServer( - config, - nodes.brokerProperties(node.id()), - raftManager, - Time.SYSTEM, - metrics, - Option.apply(threadNamePrefix), - JavaConverters.asScalaBuffer(Collections.emptyList()).toSeq(), - connectFutureManager.future, - fatalFaultHandler, - metadataFaultHandler, - metadataLoader); + jointServer, + JavaConverters.asScalaBuffer(Collections.emptyList()).toSeq()); } catch (Throwable e) { log.error("Error creating broker {}", node.id(), e); - Utils.closeQuietly(metadataLoader, "metadataLoader"); - Utils.closeQuietly(snapshotGenerator, "snapshotGenerator"); if (broker != null) broker.shutdown(); throw e; } @@ -326,9 +291,6 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS for (BrokerServer brokerServer : brokers.values()) { brokerServer.shutdown(); } - for (KafkaRaftManager raftManager : raftManagers.values()) { - raftManager.shutdown(); - } for (ControllerServer controller : controllers.values()) { controller.shutdown(); } @@ -338,34 +300,13 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS } throw e; } - return new KafkaClusterTestKit(executorService, nodes, controllers, - brokers, raftManagers, connectFutureManager, baseDirectory, - metadataFaultHandler, fatalFaultHandler); - } - - private SnapshotGenerator maybeCreateSnapshotGenerator( - int nodeId, - KafkaRaftClient raftClient - ) { - String minBytes = configProps.get(KafkaConfig$.MODULE$.MetadataSnapshotMaxNewRecordBytesProp()); - String maxIntervalMs = configProps.get(KafkaConfig$.MODULE$.MetadataSnapshotMaxIntervalMsProp()); - if (minBytes == null && maxIntervalMs == null) { - return null; - } - SnapshotEmitter emitter = new SnapshotEmitter.Builder(). - setNodeId(nodeId). - setRaftClient(raftClient). - build(); - SnapshotGenerator.Builder builder = new SnapshotGenerator.Builder(emitter). - setTime(Time.SYSTEM). - setFaultHandler(fatalFaultHandler); - if (minBytes != null) { - builder.setMinBytesSinceLastSnapshot(Long.parseLong(minBytes)); - } - if (maxIntervalMs != null) { - builder.setMaxTimeSinceLastSnapshotNs(Long.parseLong(maxIntervalMs)); - } - return builder.build(); + return new KafkaClusterTestKit(executorService, + nodes, + controllers, + brokers, + connectFutureManager, + baseDirectory, + faultHandlerFactory); } private String listeners(int node) { @@ -403,32 +344,26 @@ static private void setupNodeDirectories(File baseDirectory, private final TestKitNodes nodes; private final Map controllers; private final Map brokers; - private final Map> raftManagers; private final ControllerQuorumVotersFutureManager controllerQuorumVotersFutureManager; private final File baseDirectory; - private final MockFaultHandler metadataFaultHandler; - private final MockFaultHandler fatalFaultHandler; + private final SimpleFaultHandlerFactory faultHandlerFactory; private KafkaClusterTestKit( ExecutorService executorService, TestKitNodes nodes, Map controllers, Map brokers, - Map> raftManagers, ControllerQuorumVotersFutureManager controllerQuorumVotersFutureManager, File baseDirectory, - MockFaultHandler metadataFaultHandler, - MockFaultHandler fatalFaultHandler + SimpleFaultHandlerFactory faultHandlerFactory ) { this.executorService = executorService; this.nodes = nodes; this.controllers = controllers; this.brokers = brokers; - this.raftManagers = raftManagers; this.controllerQuorumVotersFutureManager = controllerQuorumVotersFutureManager; this.baseDirectory = baseDirectory; - this.metadataFaultHandler = metadataFaultHandler; - this.fatalFaultHandler = fatalFaultHandler; + this.faultHandlerFactory = faultHandlerFactory; } public void format() throws Exception { @@ -483,10 +418,6 @@ public void startup() throws ExecutionException, InterruptedException { try { // Note the startup order here is chosen to be consistent with // `KafkaRaftServer`. See comments in that class for an explanation. - - for (KafkaRaftManager raftManager : raftManagers.values()) { - futures.add(controllerQuorumVotersFutureManager.future.thenRunAsync(raftManager::startup)); - } for (ControllerServer controller : controllers.values()) { futures.add(executorService.submit(controller::startup)); } @@ -576,14 +507,18 @@ public Map brokers() { return brokers; } - public Map> raftManagers() { - return raftManagers; - } - public TestKitNodes nodes() { return nodes; } + public MockFaultHandler fatalFaultHandler() { + return faultHandlerFactory.fatalFaultHandler(); + } + + public MockFaultHandler nonFatalFaultHandler() { + return faultHandlerFactory.nonFatalFaultHandler(); + } + @Override public void close() throws Exception { List>> futureEntries = new ArrayList<>(); @@ -601,14 +536,6 @@ public void close() throws Exception { } waitForAllFutures(futureEntries); futureEntries.clear(); - for (Entry> entry : raftManagers.entrySet()) { - int raftManagerId = entry.getKey(); - KafkaRaftManager raftManager = entry.getValue(); - futureEntries.add(new SimpleImmutableEntry<>("raftManager" + raftManagerId, - executorService.submit(raftManager::shutdown))); - } - waitForAllFutures(futureEntries); - futureEntries.clear(); for (Entry entry : controllers.entrySet()) { int controllerId = entry.getKey(); ControllerServer controller = entry.getValue(); @@ -627,8 +554,8 @@ public void close() throws Exception { executorService.shutdownNow(); executorService.awaitTermination(5, TimeUnit.MINUTES); } - metadataFaultHandler.maybeRethrowFirstException(); - fatalFaultHandler.maybeRethrowFirstException(); + faultHandlerFactory.fatalFaultHandler().maybeRethrowFirstException(); + faultHandlerFactory.nonFatalFaultHandler().maybeRethrowFirstException(); } private void waitForAllFutures(List>> futureEntries) diff --git a/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala b/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala index 36996314df32e..bacdd6e666bd4 100644 --- a/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala +++ b/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala @@ -78,7 +78,7 @@ class KRaftClusterTest { cluster.startup() TestUtils.waitUntilTrue(() => cluster.brokers().get(0).brokerState == BrokerState.RUNNING, "Broker never made it to RUNNING state.") - TestUtils.waitUntilTrue(() => cluster.raftManagers().get(0).client.leaderAndEpoch().leaderId.isPresent, + TestUtils.waitUntilTrue(() => cluster.brokers().get(0).jointServer.raftManager.client.leaderAndEpoch().leaderId.isPresent, "RaftManager was not initialized.") val admin = Admin.create(cluster.clientProperties()) try { @@ -104,7 +104,7 @@ class KRaftClusterTest { cluster.waitForReadyBrokers() TestUtils.waitUntilTrue(() => cluster.brokers().get(0).brokerState == BrokerState.RUNNING, "Broker never made it to RUNNING state.") - TestUtils.waitUntilTrue(() => cluster.raftManagers().get(0).client.leaderAndEpoch().leaderId.isPresent, + TestUtils.waitUntilTrue(() => cluster.brokers().get(0).jointServer.raftManager.client.leaderAndEpoch().leaderId.isPresent, "RaftManager was not initialized.") val admin = Admin.create(cluster.clientProperties()) @@ -141,7 +141,7 @@ class KRaftClusterTest { cluster.waitForReadyBrokers() TestUtils.waitUntilTrue(() => cluster.brokers().get(0).brokerState == BrokerState.RUNNING, "Broker never made it to RUNNING state.") - TestUtils.waitUntilTrue(() => cluster.raftManagers().get(0).client.leaderAndEpoch().leaderId.isPresent, + TestUtils.waitUntilTrue(() => cluster.brokers().get(0).jointServer.raftManager.client.leaderAndEpoch().leaderId.isPresent, "RaftManager was not initialized.") val admin = Admin.create(cluster.clientProperties()) try { diff --git a/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala b/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala index d7bb80c468037..632ad36054a4f 100755 --- a/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala +++ b/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala @@ -23,23 +23,18 @@ import java.util import java.util.{Collections, Properties} import java.util.concurrent.CompletableFuture import javax.security.auth.login.Configuration -import kafka.raft.KafkaRaftManager import kafka.tools.StorageTool import kafka.utils.{CoreUtils, Logging, TestInfoUtils, TestUtils} import kafka.zk.{AdminZkClient, EmbeddedZookeeper, KafkaZkClient} import org.apache.kafka.common.metrics.Metrics -import org.apache.kafka.common.{TopicPartition, Uuid} +import org.apache.kafka.common.Uuid import org.apache.kafka.common.security.JaasUtils import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.utils.{Exit, Time} -import org.apache.kafka.controller.QuorumControllerMetrics -import org.apache.kafka.image.loader.MetadataLoader -import org.apache.kafka.metadata.MetadataRecordSerde import org.apache.kafka.metadata.bootstrap.BootstrapMetadata import org.apache.kafka.raft.RaftConfig.{AddressSpec, InetAddressSpec} import org.apache.kafka.server.common.{ApiMessageAndVersion, MetadataVersion} import org.apache.kafka.server.fault.{FaultHandler, MockFaultHandler} -import org.apache.kafka.server.metrics.KafkaYammerMetrics import org.apache.zookeeper.client.ZKClientConfig import org.apache.zookeeper.{WatchedEvent, Watcher, ZooKeeper} import org.junit.jupiter.api.Assertions._ @@ -85,8 +80,8 @@ class ZooKeeperQuorumImplementation( } class KRaftQuorumImplementation( - val raftManager: KafkaRaftManager[ApiMessageAndVersion], val controllerServer: ControllerServer, + val faultHandlerFactory: FaultHandlerFactory, val metadataDir: File, val controllerQuorumVotersFuture: CompletableFuture[util.Map[Integer, AddressSpec]], val clusterId: String, @@ -99,32 +94,21 @@ class KRaftQuorumImplementation( startup: Boolean, threadNamePrefix: Option[String], ): KafkaBroker = { - val threadNamePrefix = "Broker%02d_".format(config.nodeId) - var metadataLoader: MetadataLoader = null + val jointServer = new JointServer(config, + new MetaProperties(clusterId, config.nodeId), + Time.SYSTEM, + new Metrics(), + Option("Broker%02d_".format(config.nodeId)), + controllerQuorumVotersFuture, + faultHandlerFactory) var broker: BrokerServer = null try { - metadataLoader = new MetadataLoader.Builder(). - setNodeId(config.nodeId). - setTime(time). - setThreadNamePrefix(threadNamePrefix). - setFaultHandler(faultHandler). - build() - broker = new BrokerServer(config = config, - metaProps = new MetaProperties(clusterId, config.nodeId), - raftManager = raftManager, - time = time, - metrics = new Metrics(), - threadNamePrefix = Some(threadNamePrefix), - initialOfflineDirs = Seq(), - controllerQuorumVotersFuture = controllerQuorumVotersFuture, - fatalFaultHandler = faultHandler, - metadataPublishingFaultHandler = faultHandler, - metadataLoader = metadataLoader) + broker = new BrokerServer(jointServer, + initialOfflineDirs = Seq()) if (startup) broker.startup() broker } catch { case e: Throwable => { - if (metadataLoader != null) CoreUtils.swallow(metadataLoader.close(), log) if (broker != null) CoreUtils.swallow(broker.shutdown(), log) throw e } @@ -132,7 +116,6 @@ class KRaftQuorumImplementation( } override def shutdown(): Unit = { - CoreUtils.swallow(raftManager.shutdown(), log) CoreUtils.swallow(controllerServer.shutdown(), log) } } @@ -216,6 +199,10 @@ abstract class QuorumTestHarness extends Logging { val faultHandler = new MockFaultHandler("quorumTestHarnessFaultHandler") + val faultHandlerFactory = new FaultHandlerFactory { + override def build(name: String, fatal: Boolean, action: Runnable): FaultHandler = faultHandler + } + // Note: according to the junit documentation: "JUnit Jupiter does not guarantee the execution // order of multiple @BeforeEach methods that are declared within a single test class or test // interface." Therefore, if you have things you would like to do before each test case runs, it @@ -303,7 +290,6 @@ abstract class QuorumTestHarness extends Logging { val metadataDir = TestUtils.tempDir() val metaProperties = new MetaProperties(Uuid.randomUuid().toString, nodeId) formatDirectories(immutable.Seq(metadataDir.getAbsolutePath), metaProperties) - val controllerMetrics = new Metrics() props.setProperty(KafkaConfig.MetadataLogDirProp, metadataDir.getAbsolutePath) val proto = controllerListenerSecurityProtocol.toString props.setProperty(KafkaConfig.ListenerSecurityProtocolMapProp, s"CONTROLLER:${proto}") @@ -311,43 +297,20 @@ abstract class QuorumTestHarness extends Logging { props.setProperty(KafkaConfig.ControllerListenerNamesProp, "CONTROLLER") props.setProperty(KafkaConfig.QuorumVotersProp, s"${nodeId}@localhost:0") val config = new KafkaConfig(props) - val threadNamePrefix = "Controller_" + testInfo.getDisplayName val controllerQuorumVotersFuture = new CompletableFuture[util.Map[Integer, AddressSpec]] - val raftManager = new KafkaRaftManager( - metaProperties = metaProperties, - config = config, - recordSerde = MetadataRecordSerde.INSTANCE, - topicPartition = new TopicPartition(KafkaRaftServer.MetadataTopic, 0), - topicId = KafkaRaftServer.MetadataTopicId, - time = Time.SYSTEM, - metrics = controllerMetrics, - threadNamePrefixOpt = Option(threadNamePrefix), - controllerQuorumVotersFuture = controllerQuorumVotersFuture) - val time: Time = Time.SYSTEM - var metadataLoader: MetadataLoader = null + val jointServer = new JointServer(config, + metaProperties, + Time.SYSTEM, + new Metrics(), + Option("Controller_" + testInfo.getDisplayName), + controllerQuorumVotersFuture, + faultHandlerFactory) var controllerServer: ControllerServer = null try { - metadataLoader = new MetadataLoader.Builder(). - setNodeId(config.nodeId). - setTime(time). - setThreadNamePrefix(threadNamePrefix). - setFaultHandler(faultHandler). - build() controllerServer = new ControllerServer( - metaProperties = metaProperties, - config = config, - raftManager = raftManager, - time = time, - metrics = controllerMetrics, - controllerMetrics = new QuorumControllerMetrics(KafkaYammerMetrics.defaultRegistry(), Time.SYSTEM), - threadNamePrefix = Option(threadNamePrefix), - controllerQuorumVotersFuture = controllerQuorumVotersFuture, - configSchema = KafkaRaftServer.configSchema, - raftApiVersions = raftManager.apiVersions, - bootstrapMetadata = BootstrapMetadata.fromVersion(metadataVersion, "test harness"), - metadataFaultHandler = faultHandler, - fatalFaultHandler = faultHandler, - metadataLoader = metadataLoader, + jointServer, + KafkaRaftServer.configSchema, + BootstrapMetadata.fromVersion(metadataVersion, "test harness") ) controllerServer.socketServerFirstBoundPortFuture.whenComplete((port, e) => { if (e != null) { @@ -359,16 +322,13 @@ abstract class QuorumTestHarness extends Logging { } }) controllerServer.startup() - raftManager.startup() } catch { case e: Throwable => if (controllerServer != null) CoreUtils.swallow(controllerServer.shutdown(), this) - if (metadataLoader != null) CoreUtils.swallow(metadataLoader.close(), this) - CoreUtils.swallow(raftManager.shutdown(), this) throw e } - new KRaftQuorumImplementation(raftManager, - controllerServer, + new KRaftQuorumImplementation(controllerServer, + faultHandlerFactory, metadataDir, controllerQuorumVotersFuture, metaProperties.clusterId, diff --git a/core/src/test/scala/integration/kafka/server/RaftClusterSnapshotTest.scala b/core/src/test/scala/integration/kafka/server/RaftClusterSnapshotTest.scala index f1c47ff1bea97..4dddbf5823fb3 100644 --- a/core/src/test/scala/integration/kafka/server/RaftClusterSnapshotTest.scala +++ b/core/src/test/scala/integration/kafka/server/RaftClusterSnapshotTest.scala @@ -17,6 +17,8 @@ package kafka.server +import kafka.raft.KafkaRaftManager + import java.util.Collections import kafka.testkit.KafkaClusterTestKit import kafka.testkit.TestKitNodes @@ -24,16 +26,21 @@ import kafka.utils.TestUtils import kafka.server.KafkaConfig.{MetadataMaxIdleIntervalMsProp, MetadataSnapshotMaxNewRecordBytesProp} import org.apache.kafka.common.utils.BufferSupplier import org.apache.kafka.metadata.MetadataRecordSerde +import org.apache.kafka.server.common.ApiMessageAndVersion import org.apache.kafka.snapshot.RecordsSnapshotReader -import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.junit.jupiter.api.Timeout + import scala.jdk.CollectionConverters._ @Timeout(120) class RaftClusterSnapshotTest { + private def allRaftManagers(cluster: KafkaClusterTestKit): Iterable[KafkaRaftManager[ApiMessageAndVersion]] = { + (cluster.brokers().values().asScala.map(_.jointServer.raftManager) ++ + cluster.controllers().values().asScala.map(_.jointServer.raftManager)) + } @Test def testSnapshotsGenerated(): Unit = { @@ -58,21 +65,17 @@ class RaftClusterSnapshotTest { // Check that every controller and broker has a snapshot TestUtils.waitUntilTrue( () => { - cluster.raftManagers().asScala.forall { case (_, raftManager) => - raftManager.replicatedLog.latestSnapshotId.isPresent + allRaftManagers(cluster).forall { + case raftManager => raftManager.replicatedLog.latestSnapshotId().isPresent } }, s"Expected for every controller and broker to generate a snapshot: ${ - cluster.raftManagers().asScala.map { case (id, raftManager) => - (id, raftManager.replicatedLog.latestSnapshotId) - } + allRaftManagers(cluster).map(_.replicatedLog.latestSnapshotId()) }" ) - assertEquals(numberOfControllers + numberOfBrokers, cluster.raftManagers.size()) - // For every controller and broker perform some sanity checks against the lastest snapshot - for ((_, raftManager) <- cluster.raftManagers().asScala) { + allRaftManagers(cluster).foreach(raftManager => { TestUtils.resource( RecordsSnapshotReader.of( raftManager.replicatedLog.latestSnapshot.get(), @@ -92,7 +95,7 @@ class RaftClusterSnapshotTest { assertNotEquals(Collections.emptyList(), batch.records()) } } - } + }) } } } diff --git a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala b/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala index 603b38ca76499..d5c1ae6ff63cb 100644 --- a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala +++ b/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala @@ -231,12 +231,11 @@ class BrokerMetadataPublisherTest { @Test def testExceptionInUpdateCoordinator(): Unit = { - val errorHandler = new MockFaultHandler("publisher") val cluster = new KafkaClusterTestKit.Builder( new TestKitNodes.Builder(). setNumBrokerNodes(1). setNumControllerNodes(1).build()). - setMetadataFaultHandler(errorHandler).build() + build() try { cluster.format() cluster.startup() @@ -247,8 +246,8 @@ class BrokerMetadataPublisherTest { } val publisher = Mockito.spy(broker.metadataPublisher) doThrow(new RuntimeException("injected failure")).when(publisher).updateCoordinator(any(), any(), any(), any(), any()) - broker.metadataLoader.removeAndClosePublisher(broker.metadataPublisher).get() - broker.metadataLoader.installPublishers(Collections.singletonList(publisher)).get() + broker.jointServer.metadataLoader.removeAndClosePublisher(broker.metadataPublisher).get() + broker.jointServer.metadataLoader.installPublishers(Collections.singletonList(publisher)).get() val admin = Admin.create(cluster.clientProperties()) try { admin.createTopics(singletonList(new NewTopic("foo", 1, 1.toShort))).all().get() @@ -256,11 +255,11 @@ class BrokerMetadataPublisherTest { admin.close() } TestUtils.retry(60000) { - assertTrue(Option(errorHandler.firstException()). + assertTrue(Option(cluster.nonFatalFaultHandler().firstException()). flatMap(e => Option(e.getMessage())).getOrElse("(none)").contains("injected failure")) } } finally { - errorHandler.setIgnore(true) + cluster.nonFatalFaultHandler().setIgnore(true) cluster.close() } } diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala index a606a84f59363..2d3b9ee8cb092 100755 --- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala +++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala @@ -1224,7 +1224,7 @@ object TestUtils extends Logging { TestUtils.waitUntilTrue( () => { brokers.forall { broker => - val metadataOffset = broker.asInstanceOf[BrokerServer].metadataLoader.lastAppliedOffset() + val metadataOffset = broker.asInstanceOf[BrokerServer].jointServer.metadataLoader.lastAppliedOffset() metadataOffset >= controllerOffset } }, msg) From da781d7990d6c6c5a02e9f713d29251cc3452d4f Mon Sep 17 00:00:00 2001 From: "Colin P. McCabe" Date: Thu, 3 Nov 2022 09:17:44 -0700 Subject: [PATCH 10/15] SnapshotGenerator.java: add more log4j --- .../kafka/image/publisher/SnapshotGenerator.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java b/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java index 61fb31025680f..bd340f2615110 100644 --- a/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java +++ b/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java @@ -176,6 +176,7 @@ private SnapshotGenerator( this.disabledReason = null; this.eventQueue = new KafkaEventQueue(time, logContext, "SnapshotGenerator" + nodeId); resetSnapshotCounters(); + log.debug("Starting SnapshotGenerator."); } @Override @@ -194,7 +195,8 @@ public void publishSnapshot( MetadataImage newImage, SnapshotManifest manifest ) { - // Reset the snapshot counters because we just read a snapshot. + log.debug("Resetting the snapshot counters because we just read a snapshot at offset {}.", + newImage.provenance().offset()); resetSnapshotCounters(); } @@ -204,6 +206,8 @@ public void publishPreVersionChangeImage( MetadataImage preVersionChangeImage, PreVersionChangeManifest manifest ) { + log.info("Scheduling snapshot on metadata version change from {} to {}.", + manifest.change().oldVersion(), manifest.change().newVersion()); scheduleEmit("the metadata version changed", preVersionChangeImage); } @@ -218,13 +222,19 @@ public void publishLogDelta( if (eventQueue.isEmpty()) { scheduleEmit("we have replayed at least " + minBytesSinceLastSnapshot + " bytes", newImage); + } else if (log.isTraceEnabled()) { + log.trace("Not scheduling bytes-based snapshot because event queue is not empty yet."); } } else if (maxTimeSinceLastSnapshotNs != 0 && (time.nanoseconds() - lastSnapshotTimeNs >= maxTimeSinceLastSnapshotNs)) { if (eventQueue.isEmpty()) { scheduleEmit("we have waited at least " + TimeUnit.NANOSECONDS.toMinutes(maxTimeSinceLastSnapshotNs) + " minute(s)", newImage); + } else if (log.isTraceEnabled()) { + log.trace("Not scheduling time-based snapshot because event queue is not empty yet."); } + } else if (log.isTraceEnabled()) { + log.trace("Neither time-based nor bytes-based criteria are met; not scheduling snapshot."); } } @@ -252,6 +262,7 @@ void scheduleEmit( } public void beginShutdown() { + log.debug("Beginning shutdown of SnapshotGenerator."); this.disabledReason = "we are shutting down"; eventQueue.beginShutdown("beginShutdown"); } @@ -264,6 +275,7 @@ public void disable(String disabledReason) { @Override public void close() throws InterruptedException { eventQueue.beginShutdown("close"); + log.debug("Closing SnapshotGenerator."); eventQueue.close(); } } From 9673a6a2c04ff6b03fcf62203fef2fa4bfebc862 Mon Sep 17 00:00:00 2001 From: "Colin P. McCabe" Date: Thu, 3 Nov 2022 09:18:01 -0700 Subject: [PATCH 11/15] Some simplifications and be sure to register SnapshotGenerator in JointServer --- .../main/scala/kafka/server/ControllerServer.scala | 6 ++---- core/src/main/scala/kafka/server/JointServer.scala | 8 +++++++- .../java/kafka/testkit/KafkaClusterTestKit.java | 13 ++++--------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/core/src/main/scala/kafka/server/ControllerServer.scala b/core/src/main/scala/kafka/server/ControllerServer.scala index 3a626e1ccbe27..792445419e0c9 100644 --- a/core/src/main/scala/kafka/server/ControllerServer.scala +++ b/core/src/main/scala/kafka/server/ControllerServer.scala @@ -109,8 +109,6 @@ class ControllerServer( def startup(): Unit = { if (!maybeChangeStatus(SHUTDOWN, STARTING)) return try { - jointServer.startForController() - info("Starting controller") maybeChangeStatus(STARTING, STARTED) @@ -166,7 +164,7 @@ class ControllerServer( throw new ConfigException("No controller.listener.names defined for controller") } - val threadNamePrefixAsString = jointServer.threadNamePrefix.getOrElse("") + jointServer.startForController() createTopicPolicy = Option(config. getConfiguredInstance(CreateTopicPolicyClassNameProp, classOf[CreateTopicPolicy])) @@ -190,7 +188,7 @@ class ControllerServer( new QuorumController.Builder(config.nodeId, jointServer.metaProps.clusterId). setTime(time). - setThreadNamePrefix(threadNamePrefixAsString). + setThreadNamePrefix(jointServer.threadNamePrefix.getOrElse("")). setConfigSchema(configSchema). setRaftClient(jointServer.raftManager.client). setQuorumFeatures(quorumFeatures). diff --git a/core/src/main/scala/kafka/server/JointServer.scala b/core/src/main/scala/kafka/server/JointServer.scala index 949bc39f946cd..dc2e10c6c3b6d 100644 --- a/core/src/main/scala/kafka/server/JointServer.scala +++ b/core/src/main/scala/kafka/server/JointServer.scala @@ -208,6 +208,7 @@ class JointServer( metrics, threadNamePrefix, controllerQuorumVotersFuture) + raftManager.startup() snapshotEmitter = new SnapshotEmitter.Builder(). setNodeId(config.nodeId). setRaftClient(raftManager.client). @@ -228,9 +229,14 @@ class JointServer( loaderBuilder.setMetadataLoaderMetrics(brokerMetrics) } metadataLoader = loaderBuilder.build() + metadataLoader.installPublishers(util.Arrays.asList(snapshotGenerator)) raftManager.client.register(metadataLoader) + debug("Completed JointServer startup.") } catch { - case e: Throwable => stop() + case e: Throwable => { + error("Got exception while starting JointServer", e) + stop() + } } } diff --git a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java index 971b832b88541..543017fbf3443 100644 --- a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java +++ b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java @@ -29,7 +29,6 @@ import kafka.utils.Logging; import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.common.Node; -import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.utils.ThreadUtils; @@ -198,9 +197,7 @@ public KafkaClusterTestKit build() throws Exception { setupNodeDirectories(baseDirectory, node.metadataDirectory(), Collections.emptyList()); KafkaConfig config = new KafkaConfig(props, false, Option.empty()); - String threadNamePrefix = String.format("controller%d", node.id()); MetaProperties metaProperties = MetaProperties.apply(nodes.clusterId().toString(), node.id()); - TopicPartition metadataPartition = new TopicPartition(KafkaRaftServer.MetadataTopic(), 0); BootstrapMetadata bootstrapMetadata = BootstrapMetadata. fromVersion(nodes.bootstrapMetadataVersion(), "testkit"); JointServer jointServer = new JointServer(config, @@ -257,19 +254,17 @@ public KafkaClusterTestKit build() throws Exception { props.putAll(node.propertyOverrides()); KafkaConfig config = new KafkaConfig(props, false, Option.empty()); - String threadNamePrefix = String.format("broker%d_", node.id()); - JointServer jointServer; - if (jointServers.containsKey(node.id())) { - jointServer = jointServers.get(node.id()); - } else { + JointServer jointServer = jointServers.get(node.id()); + if (jointServer == null) { MetaProperties metaProperties = MetaProperties.apply(nodes.clusterId().toString(), node.id()); jointServer = new JointServer(config, metaProperties, Time.SYSTEM, new Metrics(), - Option.apply(String.format("controller%d_", node.id())), + Option.apply(String.format("broker%d_", node.id())), connectFutureManager.future, faultHandlerFactory); + jointServers.put(node.id(), jointServer); } BrokerServer broker = null; try { From c5c8290dd27e87992f554be5a2df0a1585a6089e Mon Sep 17 00:00:00 2001 From: "Colin P. McCabe" Date: Thu, 3 Nov 2022 09:34:13 -0700 Subject: [PATCH 12/15] KafkaClusterTestKit: fix colocated mode --- .../kafka/testkit/KafkaClusterTestKit.java | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java index 543017fbf3443..cc51d8d45771d 100644 --- a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java +++ b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java @@ -200,11 +200,14 @@ public KafkaClusterTestKit build() throws Exception { MetaProperties metaProperties = MetaProperties.apply(nodes.clusterId().toString(), node.id()); BootstrapMetadata bootstrapMetadata = BootstrapMetadata. fromVersion(nodes.bootstrapMetadataVersion(), "testkit"); + String threadNamePrefix = (nodes.brokerNodes().containsKey(node.id())) ? + String.format("colocated%d", node.id()) : + String.format("controller%d", node.id()); JointServer jointServer = new JointServer(config, metaProperties, Time.SYSTEM, new Metrics(), - Option.apply(String.format("controller%d_", node.id())), + Option.apply(threadNamePrefix), connectFutureManager.future, faultHandlerFactory); ControllerServer controller = null; @@ -233,8 +236,15 @@ public KafkaClusterTestKit build() throws Exception { props.put(KafkaConfig$.MODULE$.ProcessRolesProp(), roles(node.id())); props.put(KafkaConfig$.MODULE$.BrokerIdProp(), Integer.toString(node.id())); - props.put(KafkaConfig$.MODULE$.MetadataLogDirProp(), - node.metadataDirectory()); + if (nodes.controllerNodes().containsKey(node.id())) { + // Combined mode: use the metadata log dir of the associated controller node. + props.put(KafkaConfig$.MODULE$.MetadataLogDirProp(), + nodes.controllerNodes().get(node.id()).metadataDirectory()); + } else { + // Separate mode: use the broker node's log dir. + props.put(KafkaConfig$.MODULE$.MetadataLogDirProp(), + node.metadataDirectory()); + } props.put(KafkaConfig$.MODULE$.LogDirsProp(), String.join(",", node.logDataDirectories())); props.put(KafkaConfig$.MODULE$.ListenerSecurityProtocolMapProp(), @@ -371,10 +381,12 @@ public void format() throws Exception { controller, futures::add); } for (Entry entry : brokers.entrySet()) { - int nodeId = entry.getKey(); - BrokerServer broker = entry.getValue(); - formatNodeAndLog(nodes.brokerProperties(nodeId), broker.config().metadataLogDir(), - broker, futures::add); + if (!controllers.containsKey(entry.getKey())) { + int nodeId = entry.getKey(); + BrokerServer broker = entry.getValue(); + formatNodeAndLog(nodes.brokerProperties(nodeId), broker.config().metadataLogDir(), + broker, futures::add); + } } for (Future future: futures) { future.get(); From 50a8ef3d0b94e33f83f51f0d217a87f9c4ef835d Mon Sep 17 00:00:00 2001 From: "Colin P. McCabe" Date: Thu, 3 Nov 2022 09:51:58 -0700 Subject: [PATCH 13/15] Add EventQueue#size and make EventQueue#isEmpty take into account currently running events --- .../org/apache/kafka/queue/EventQueue.java | 11 ++++++-- .../apache/kafka/queue/KafkaEventQueue.java | 25 +++++++++++++------ .../kafka/queue/KafkaEventQueueTest.java | 5 +++- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/server-common/src/main/java/org/apache/kafka/queue/EventQueue.java b/server-common/src/main/java/org/apache/kafka/queue/EventQueue.java index 10165bb89f6ab..99a08c3e1c2cc 100644 --- a/server-common/src/main/java/org/apache/kafka/queue/EventQueue.java +++ b/server-common/src/main/java/org/apache/kafka/queue/EventQueue.java @@ -249,9 +249,16 @@ default void beginShutdown(String source, Event cleanupEvent) { void beginShutdown(String source, Event cleanupEvent, long timeSpan, TimeUnit timeUnit); /** - * @return True only if there are no events scheduled in the queue. + * @return The number of pending events in the queue. If an event is running, it is counted. */ - boolean isEmpty(); + int size(); + + /** + * @return True if there are no pending events in the queue. If an event is running, it is counted. + */ + default boolean isEmpty() { + return size() == 0; + } /** * This method is used during unit tests where MockTime is in use. diff --git a/server-common/src/main/java/org/apache/kafka/queue/KafkaEventQueue.java b/server-common/src/main/java/org/apache/kafka/queue/KafkaEventQueue.java index 39ae8bbe350bf..b53339edfe337 100644 --- a/server-common/src/main/java/org/apache/kafka/queue/KafkaEventQueue.java +++ b/server-common/src/main/java/org/apache/kafka/queue/KafkaEventQueue.java @@ -147,6 +147,8 @@ void completeWithException(Throwable t) { } private class EventHandler implements Runnable { + private int size = 0; + /** * Event contexts indexed by tag. Events without a tag are not included here. */ @@ -195,13 +197,19 @@ private void handleEvents() throws InterruptedException { while (true) { if (toTimeout != null) { toTimeout.completeWithTimeout(); - toTimeout = null; } else if (toRun != null) { toRun.run(log); - toRun = null; } lock.lock(); try { + if (toTimeout != null) { + size--; + toTimeout = null; + } + if (toRun != null) { + size--; + toRun = null; + } long awaitNs = Long.MAX_VALUE; Map.Entry entry = deadlineMap.firstEntry(); if (entry != null) { @@ -272,6 +280,7 @@ Exception enqueue(EventContext eventContext, if (toRemove != null) { existingDeadlineNs = toRemove.deadlineNs; remove(toRemove); + size--; } } OptionalLong deadlineNs = deadlineNsCalculator.apply(existingDeadlineNs); @@ -311,6 +320,7 @@ Exception enqueue(EventContext eventContext, shouldSignal = true; } } + size++; if (shouldSignal) { cond.signal(); } @@ -326,6 +336,7 @@ void cancelDeferred(String tag) { EventContext eventContext = tagToEventContext.get(tag); if (eventContext != null) { remove(eventContext); + size--; } } finally { lock.unlock(); @@ -341,12 +352,10 @@ void wakeUp() { } } - boolean isEmpty() { + int size() { lock.lock(); try { - if (head != head.next) return false; - if (!deadlineMap.isEmpty()) return false; - return true; + return size; } finally { lock.unlock(); } @@ -424,8 +433,8 @@ public void beginShutdown(String source, Event newCleanupEvent, } @Override - public boolean isEmpty() { - return eventHandler.isEmpty(); + public int size() { + return eventHandler.size(); } @Override diff --git a/server-common/src/test/java/org/apache/kafka/queue/KafkaEventQueueTest.java b/server-common/src/test/java/org/apache/kafka/queue/KafkaEventQueueTest.java index 8dcc02191633d..09910200a1f87 100644 --- a/server-common/src/test/java/org/apache/kafka/queue/KafkaEventQueueTest.java +++ b/server-common/src/test/java/org/apache/kafka/queue/KafkaEventQueueTest.java @@ -244,13 +244,16 @@ public void handleException(Throwable e) { } @Test - public void testEmpty() throws Exception { + public void testSize() throws Exception { KafkaEventQueue queue = new KafkaEventQueue(Time.SYSTEM, new LogContext(), "testEmpty"); assertTrue(queue.isEmpty()); CompletableFuture future = new CompletableFuture<>(); queue.append(() -> future.get()); assertFalse(queue.isEmpty()); + assertEquals(1, queue.size()); + queue.append(() -> future.get()); + assertEquals(2, queue.size()); future.complete(null); TestUtils.waitForCondition(() -> queue.isEmpty(), "Failed to see the queue become empty."); queue.scheduleDeferred("later", From 669f2aab9866e39abac3f4fb7ee50f185d0efcc7 Mon Sep 17 00:00:00 2001 From: "Colin P. McCabe" Date: Thu, 3 Nov 2022 10:42:17 -0700 Subject: [PATCH 14/15] Re-fix KafkaClusterTestKit in combined mode --- .../kafka/testkit/KafkaClusterTestKit.java | 108 ++++++++---------- 1 file changed, 48 insertions(+), 60 deletions(-) diff --git a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java index cc51d8d45771d..d3d2bd71ad20f 100644 --- a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java +++ b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java @@ -149,13 +149,56 @@ public Builder setConfigProp(String key, String value) { return this; } + private KafkaConfig createNodeConfig(TestKitNode node) { + BrokerNode brokerNode = nodes.brokerNodes().get(node.id()); + ControllerNode controllerNode = nodes.controllerNodes().get(node.id()); + + Map props = new HashMap<>(configProps); + props.put(KafkaConfig$.MODULE$.ProcessRolesProp(), roles(node.id())); + props.put(KafkaConfig$.MODULE$.NodeIdProp(), + Integer.toString(node.id())); + // In combined mode, always prefer the metadata log directory of the controller node. + if (controllerNode != null) { + props.put(KafkaConfig$.MODULE$.MetadataLogDirProp(), + controllerNode.metadataDirectory()); + } else { + props.put(KafkaConfig$.MODULE$.MetadataLogDirProp(), + node.metadataDirectory()); + } + // Set the log.dirs according to the broker node setting (if there is a broker node) + if (brokerNode != null) { + props.put(KafkaConfig$.MODULE$.LogDirsProp(), + String.join(",", brokerNode.logDataDirectories())); + } + props.put(KafkaConfig$.MODULE$.ListenerSecurityProtocolMapProp(), + "EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT"); + props.put(KafkaConfig$.MODULE$.ListenersProp(), listeners(node.id())); + props.put(KafkaConfig$.MODULE$.InterBrokerListenerNameProp(), + nodes.interBrokerListenerName().value()); + props.put(KafkaConfig$.MODULE$.ControllerListenerNamesProp(), + "CONTROLLER"); + // Note: we can't accurately set controller.quorum.voters yet, since we don't + // yet know what ports each controller will pick. Set it to a dummy string \ + // for now as a placeholder. + String uninitializedQuorumVotersString = nodes.controllerNodes().keySet().stream(). + map(n -> String.format("%d@0.0.0.0:0", n)). + collect(Collectors.joining(",")); + props.put(RaftConfig.QUORUM_VOTERS_CONFIG, uninitializedQuorumVotersString); + + // reduce log cleaner offset map memory usage + props.put(KafkaConfig$.MODULE$.LogCleanerDedupeBufferSizeProp(), "2097152"); + + // Add associated broker node property overrides + if (brokerNode != null) { + props.putAll(brokerNode.propertyOverrides()); + } + return new KafkaConfig(props, false, Option.empty()); + } + public KafkaClusterTestKit build() throws Exception { Map controllers = new HashMap<>(); Map brokers = new HashMap<>(); Map jointServers = new HashMap<>(); - String uninitializedQuorumVotersString = nodes.controllerNodes().keySet().stream(). - map(controllerNode -> String.format("%d@0.0.0.0:0", controllerNode)). - collect(Collectors.joining(",")); /* Number of threads = Total number of brokers + Total number of controllers + Total number of Raft Managers = Total number of brokers + Total number of controllers * 2 @@ -173,37 +216,14 @@ public KafkaClusterTestKit build() throws Exception { executorService = Executors.newFixedThreadPool(numOfExecutorThreads, ThreadUtils.createThreadFactory("KafkaClusterTestKit%d", false)); for (ControllerNode node : nodes.controllerNodes().values()) { - Map props = new HashMap<>(configProps); - props.put(KafkaConfig$.MODULE$.ProcessRolesProp(), roles(node.id())); - props.put(KafkaConfig$.MODULE$.NodeIdProp(), - Integer.toString(node.id())); - props.put(KafkaConfig$.MODULE$.MetadataLogDirProp(), - node.metadataDirectory()); - props.put(KafkaConfig$.MODULE$.ListenerSecurityProtocolMapProp(), - "EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT"); - props.put(KafkaConfig$.MODULE$.ListenersProp(), listeners(node.id())); - props.put(KafkaConfig$.MODULE$.InterBrokerListenerNameProp(), - nodes.interBrokerListenerName().value()); - props.put(KafkaConfig$.MODULE$.ControllerListenerNamesProp(), - "CONTROLLER"); - // Note: we can't accurately set controller.quorum.voters yet, since we don't - // yet know what ports each controller will pick. Set it to a dummy string \ - // for now as a placeholder. - props.put(RaftConfig.QUORUM_VOTERS_CONFIG, uninitializedQuorumVotersString); - - // reduce log cleaner offset map memory usage - props.put(KafkaConfig$.MODULE$.LogCleanerDedupeBufferSizeProp(), "2097152"); - setupNodeDirectories(baseDirectory, node.metadataDirectory(), Collections.emptyList()); - KafkaConfig config = new KafkaConfig(props, false, Option.empty()); - MetaProperties metaProperties = MetaProperties.apply(nodes.clusterId().toString(), node.id()); BootstrapMetadata bootstrapMetadata = BootstrapMetadata. fromVersion(nodes.bootstrapMetadataVersion(), "testkit"); String threadNamePrefix = (nodes.brokerNodes().containsKey(node.id())) ? String.format("colocated%d", node.id()) : String.format("controller%d", node.id()); - JointServer jointServer = new JointServer(config, + JointServer jointServer = new JointServer(createNodeConfig(node), metaProperties, Time.SYSTEM, new Metrics(), @@ -232,42 +252,10 @@ public KafkaClusterTestKit build() throws Exception { jointServers.put(node.id(), jointServer); } for (BrokerNode node : nodes.brokerNodes().values()) { - Map props = new HashMap<>(configProps); - props.put(KafkaConfig$.MODULE$.ProcessRolesProp(), roles(node.id())); - props.put(KafkaConfig$.MODULE$.BrokerIdProp(), - Integer.toString(node.id())); - if (nodes.controllerNodes().containsKey(node.id())) { - // Combined mode: use the metadata log dir of the associated controller node. - props.put(KafkaConfig$.MODULE$.MetadataLogDirProp(), - nodes.controllerNodes().get(node.id()).metadataDirectory()); - } else { - // Separate mode: use the broker node's log dir. - props.put(KafkaConfig$.MODULE$.MetadataLogDirProp(), - node.metadataDirectory()); - } - props.put(KafkaConfig$.MODULE$.LogDirsProp(), - String.join(",", node.logDataDirectories())); - props.put(KafkaConfig$.MODULE$.ListenerSecurityProtocolMapProp(), - "EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT"); - props.put(KafkaConfig$.MODULE$.ListenersProp(), listeners(node.id())); - props.put(KafkaConfig$.MODULE$.InterBrokerListenerNameProp(), - nodes.interBrokerListenerName().value()); - props.put(KafkaConfig$.MODULE$.ControllerListenerNamesProp(), - "CONTROLLER"); - - setupNodeDirectories(baseDirectory, node.metadataDirectory(), - node.logDataDirectories()); - - // Just like above, we set a placeholder voter list here until we - // find out what ports the controllers picked. - props.put(RaftConfig.QUORUM_VOTERS_CONFIG, uninitializedQuorumVotersString); - props.putAll(node.propertyOverrides()); - KafkaConfig config = new KafkaConfig(props, false, Option.empty()); - JointServer jointServer = jointServers.get(node.id()); if (jointServer == null) { MetaProperties metaProperties = MetaProperties.apply(nodes.clusterId().toString(), node.id()); - jointServer = new JointServer(config, + jointServer = new JointServer(createNodeConfig(node), metaProperties, Time.SYSTEM, new Metrics(), From 6039578480d7ab9b4e519ed2b98ab03c29ef8e95 Mon Sep 17 00:00:00 2001 From: "Colin P. McCabe" Date: Mon, 7 Nov 2022 12:02:40 -0800 Subject: [PATCH 15/15] Fix PlaintextAdminIntegrationTest.testCreateTopicsReturnsConfigs --- .../kafka/api/BaseAdminIntegrationTest.scala | 5 ---- .../api/PlaintextAdminIntegrationTest.scala | 24 ++++++++++++++----- .../controller/FeatureControlManager.java | 4 ++++ .../kafka/controller/QuorumController.java | 9 +++---- .../kafka/controller/QuorumFeatures.java | 4 ++++ 5 files changed, 31 insertions(+), 15 deletions(-) diff --git a/core/src/test/scala/integration/kafka/api/BaseAdminIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/BaseAdminIntegrationTest.scala index 293b198c0ecdb..6fa8bc9b2bc55 100644 --- a/core/src/test/scala/integration/kafka/api/BaseAdminIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/BaseAdminIntegrationTest.scala @@ -215,11 +215,6 @@ abstract class BaseAdminIntegrationTest extends IntegrationTestHarness with Logg override def kraftControllerConfigs(): Seq[Properties] = { val controllerConfig = new Properties() - if (testInfo.getTestMethod.toString.contains("testCreateTopicsReturnsConfigs")) { - // For testCreateTopicsReturnsConfigs, set the controller's ID to 1 so that the dynamic - // config we set for node 1 will apply to it. - controllerConfig.setProperty(KafkaConfig.NodeIdProp, "1") - } val controllerConfigs = Seq(controllerConfig) modifyConfigs(controllerConfigs) controllerConfigs diff --git a/core/src/test/scala/integration/kafka/api/PlaintextAdminIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextAdminIntegrationTest.scala index 1656af08bc15e..c3854ffa29368 100644 --- a/core/src/test/scala/integration/kafka/api/PlaintextAdminIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/PlaintextAdminIntegrationTest.scala @@ -44,12 +44,14 @@ import org.apache.kafka.common.requests.{DeleteRecordsRequest, MetadataResponse} import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourceType} import org.apache.kafka.common.utils.{Time, Utils} import org.apache.kafka.common.{ConsumerGroupState, ElectionType, TopicCollection, TopicPartition, TopicPartitionInfo, TopicPartitionReplica, Uuid} +import org.apache.kafka.controller.ControllerRequestContextUtil.ANONYMOUS_CONTEXT import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Disabled, TestInfo} import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.ValueSource import org.slf4j.LoggerFactory +import java.util.AbstractMap.SimpleImmutableEntry import scala.annotation.nowarn import scala.collection.Seq import scala.compat.java8.OptionConverters._ @@ -2500,22 +2502,32 @@ class PlaintextAdminIntegrationTest extends BaseAdminIntegrationTest { * Test that createTopics returns the dynamic configurations of the topics that were created. * * Note: this test requires some custom static broker and controller configurations, which are set up in - * BaseAdminIntegrationTest.modifyConfigs and BaseAdminIntegrationTest.kraftControllerConfigs. + * BaseAdminIntegrationTest.modifyConfigs. */ @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumName) @ValueSource(strings = Array("zk", "kraft")) def testCreateTopicsReturnsConfigs(quorum: String): Unit = { client = Admin.create(super.createConfig) - val newLogRetentionProperties = new Properties newLogRetentionProperties.put(KafkaConfig.LogRetentionTimeMillisProp, "10800000") TestUtils.incrementalAlterConfigs(null, client, newLogRetentionProperties, perBrokerConfig = false) .all().get(15, TimeUnit.SECONDS) - val newLogCleanerDeleteRetention = new Properties - newLogCleanerDeleteRetention.put(KafkaConfig.LogCleanerDeleteRetentionMsProp, "34") - TestUtils.incrementalAlterConfigs(brokers, client, newLogCleanerDeleteRetention, perBrokerConfig = true) - .all().get(15, TimeUnit.SECONDS) + if (isKRaftTest()) { + // In KRaft mode, we don't yet support altering configs on controller nodes, except by setting + // default node configs. Therefore, we have to set the dynamic config directly to test this. + val controllerNodeResource = new ConfigResource(ConfigResource.Type.BROKER, + controllerServer.config.nodeId.toString) + controllerServer.controller.incrementalAlterConfigs(ANONYMOUS_CONTEXT, + Collections.singletonMap(controllerNodeResource, + Collections.singletonMap(KafkaConfig.LogCleanerDeleteRetentionMsProp, + new SimpleImmutableEntry(AlterConfigOp.OpType.SET, "34"))), false).get() + } else { + val newLogCleanerDeleteRetention = new Properties + newLogCleanerDeleteRetention.put(KafkaConfig.LogCleanerDeleteRetentionMsProp, "34") + TestUtils.incrementalAlterConfigs(brokers, client, newLogCleanerDeleteRetention, perBrokerConfig = true) + .all().get(15, TimeUnit.SECONDS) + } if (isKRaftTest()) { ensureConsistentKRaftMetadata() diff --git a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java index c777d9828cdcc..774dc1e7fb307 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java @@ -305,4 +305,8 @@ public void replay(FeatureLevelRecord record) { } } } + + boolean isQuorumNodeId(int nodeId) { + return quorumFeatures.isQuorumNodeId(nodeId); + } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index c0141ac71b577..fae44019b639f 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -364,16 +364,17 @@ public void accept(ConfigResource configResource) { if (configResource.name().isEmpty()) break; // Otherwise, check that the broker ID is valid. - int brokerId; + int nodeId; try { - brokerId = Integer.parseInt(configResource.name()); + nodeId = Integer.parseInt(configResource.name()); } catch (NumberFormatException e) { throw new InvalidRequestException("Invalid broker name " + configResource.name()); } - if (!clusterControl.brokerRegistrations().containsKey(brokerId)) { + if (!(clusterControl.brokerRegistrations().containsKey(nodeId) || + featureControl.isQuorumNodeId(nodeId))) { throw new BrokerIdNotRegisteredException("No broker with id " + - brokerId + " found."); + nodeId + " found."); } break; case TOPIC: diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumFeatures.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumFeatures.java index 36725c251857c..85a121a045840 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumFeatures.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumFeatures.java @@ -124,4 +124,8 @@ public Optional reasonNotSupported(String featureName, short level) { VersionRange localSupportedFeature(String featureName) { return localSupportedFeatures.getOrDefault(featureName, DISABLED); } + + boolean isQuorumNodeId(int nodeId) { + return quorumNodeIds.contains(nodeId); + } }