diff --git a/checkstyle/import-control-core.xml b/checkstyle/import-control-core.xml index 4042cba402fdd..82802ff4c68ad 100644 --- a/checkstyle/import-control-core.xml +++ b/checkstyle/import-control-core.xml @@ -49,6 +49,7 @@ + diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 63137cf6c1703..32809c4b79a55 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -225,6 +225,7 @@ + @@ -252,10 +253,13 @@ + + + @@ -478,6 +482,10 @@ + + + + diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 1d60fde34d20b..debc77ccd5f5d 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -38,9 +38,7 @@ - - + Some(m.state)).getOrElse(BrokerState.NOT_RUNNING) @@ -144,11 +124,7 @@ class BrokerServer( @volatile var brokerTopicStats: BrokerTopicStats = _ - val clusterId: String = metaProps.clusterId - - var metadataSnapshotter: Option[BrokerMetadataSnapshotter] = None - - var metadataListener: BrokerMetadataListener = _ + val clusterId: String = jointServer.metaProps.clusterId var metadataPublisher: BrokerMetadataPublisher = _ @@ -180,9 +156,9 @@ class BrokerServer( override def startup(): Unit = { if (!maybeChangeStatus(SHUTDOWN, STARTING)) return try { - info("Starting broker") + jointServer.startForBroker() - config.dynamicConfig.initialize(zkClientOpt = None) + info("Starting broker") lifecycleManager = new BrokerLifecycleManager(config, time, @@ -211,8 +187,8 @@ class BrokerServer( tokenCache = new DelegationTokenCache(ScramMechanism.mechanismNames) credentialProvider = new CredentialProvider(ScramMechanism.mechanismNames, tokenCache) - val controllerNodes = RaftConfig.voterConnectionsToNodes(controllerQuorumVotersFuture.get()).asScala - val controllerNodeProvider = RaftControllerNodeProvider(raftManager, config, controllerNodes) + val controllerNodes = RaftConfig.voterConnectionsToNodes(jointServer.controllerQuorumVotersFuture.get()).asScala + val controllerNodeProvider = RaftControllerNodeProvider(jointServer.raftManager, config, controllerNodes) clientToControllerChannelManager = BrokerToControllerChannelManager( controllerNodeProvider, @@ -304,25 +280,6 @@ class BrokerServer( ConfigType.Topic -> new TopicConfigHandler(logManager, config, quotaManagers, None), ConfigType.Broker -> new BrokerConfigHandler(config, quotaManagers)) - if (!config.processRoles.contains(ControllerRole)) { - // If no controller is defined, we rely on the broker to generate snapshots. - metadataSnapshotter = Some(new BrokerMetadataSnapshotter( - config.nodeId, - time, - threadNamePrefix, - new BrokerSnapshotWriterBuilder(raftManager.client) - )) - } - - metadataListener = new BrokerMetadataListener( - config.nodeId, - time, - threadNamePrefix, - config.metadataSnapshotMaxNewRecordBytes, - metadataSnapshotter, - brokerMetrics, - metadataLoadingFaultHandler) - val networkListeners = new ListenerCollection() config.effectiveAdvertisedListeners.foreach { ep => networkListeners.add(new Listener(). @@ -347,16 +304,13 @@ class BrokerServer( config.brokerSessionTimeoutMs.toLong ) lifecycleManager.start( - () => metadataListener.highestMetadataOffset, + () => jointServer.metadataLoader.lastAppliedOffset(), brokerLifecycleChannelManager, - metaProps.clusterId, + clusterId, networkListeners, featuresRemapped ) - // Register a listener with the Raft layer to receive metadata event notifications - raftManager.register(metadataListener) - val endpoints = new util.ArrayList[Endpoint](networkListeners.size()) var interBrokerListener: Endpoint = null networkListeners.iterator().forEachRemaining(listener => { @@ -426,9 +380,16 @@ class BrokerServer( DataPlaneAcceptor.ThreadPrefix) // Block until we've caught up with the latest metadata from the controller quorum. - lifecycleManager.initialCatchUpFuture.get() + info("Waiting for broker metadata to catch up.") + try { + lifecycleManager.initialCatchUpFuture.get(20, TimeUnit.SECONDS) + } catch { + case t: Throwable => throw new RuntimeException("Received a fatal error while " + + "waiting for the broker to catch up with the current cluster metadata.", t) + } // Apply the metadata log changes that we've accumulated. + val metadataPublishingFaultHandler = jointServer.metadataPublishingFaultHandler metadataPublisher = new BrokerMetadataPublisher(config, metadataCache, logManager, @@ -436,9 +397,12 @@ class BrokerServer( groupCoordinator, transactionCoordinator, clientQuotaMetadataManager, - dynamicConfigHandlers.toMap, + new DynamicConfigPublisher( + config, + metadataPublishingFaultHandler, + dynamicConfigHandlers.toMap), authorizer, - fatalFaultHandler, + jointServer.initialBrokerMetadataLoadFaultHandler, metadataPublishingFaultHandler) // Tell the metadata listener to start publishing its output, and wait for the first @@ -446,10 +410,10 @@ class BrokerServer( // replicaManager, groupCoordinator, and txnCoordinator. The log manager may perform // a potentially lengthy recovery-from-unclean-shutdown operation here, if required. try { - metadataListener.startPublishing(metadataPublisher).get() + jointServer.metadataLoader.installPublishers(List(metadataPublisher).asJava).get(20, TimeUnit.SECONDS) } catch { case t: Throwable => throw new RuntimeException("Received a fatal error while " + - "waiting for the broker to catch up with the current cluster metadata.", t) + "publishing the initial metadata update.", t) } // Log static broker configurations. @@ -470,7 +434,7 @@ class BrokerServer( // We're now ready to unfence the broker. This also allows this broker to transition // from RECOVERY state to RUNNING state, once the controller unfences the broker. try { - lifecycleManager.setReadyToUnfence().get() + lifecycleManager.setReadyToUnfence().get(20, TimeUnit.SECONDS) } catch { case t: Throwable => throw new RuntimeException("Received a fatal error while " + "waiting for the broker to be unfenced.", t) @@ -506,9 +470,6 @@ class BrokerServer( } } - if (metadataListener != null) - metadataListener.beginShutdown() - lifecycleManager.beginShutdown() // Stop socket server to stop accepting any more connections and requests. @@ -523,10 +484,6 @@ class BrokerServer( if (controlPlaneRequestProcessor != null) CoreUtils.swallow(controlPlaneRequestProcessor.close(), this) CoreUtils.swallow(authorizer.foreach(_.close()), this) - if (metadataListener != null) { - CoreUtils.swallow(metadataListener.close(), this) - } - metadataSnapshotter.foreach(snapshotter => CoreUtils.swallow(snapshotter.close(), this)) /** * We must shutdown the scheduler early because otherwise, the scheduler could touch other @@ -564,19 +521,18 @@ class BrokerServer( if (quotaManagers != null) CoreUtils.swallow(quotaManagers.shutdown(), this) - if (socketServer != null) + if (socketServer != null) { CoreUtils.swallow(socketServer.shutdown(), this) + } if (metrics != null) CoreUtils.swallow(metrics.close(), this) if (brokerTopicStats != null) CoreUtils.swallow(brokerTopicStats.close(), this) - // Clear all reconfigurable instances stored in DynamicBrokerConfig - config.dynamicConfig.clear() - isShuttingDown.set(false) CoreUtils.swallow(lifecycleManager.close(), this) + jointServer.stopForBroker() CoreUtils.swallow(AppInfoParser.unregisterAppInfo(MetricsPrefix, config.nodeId.toString, metrics), this) info("shut down completed") diff --git a/core/src/main/scala/kafka/server/ControllerServer.scala b/core/src/main/scala/kafka/server/ControllerServer.scala index 6e4c09b7a3780..792445419e0c9 100644 --- a/core/src/main/scala/kafka/server/ControllerServer.scala +++ b/core/src/main/scala/kafka/server/ControllerServer.scala @@ -17,39 +17,36 @@ package kafka.server -import java.util import java.util.OptionalLong import java.util.concurrent.locks.ReentrantLock import java.util.concurrent.{CompletableFuture, TimeUnit} import kafka.cluster.Broker.ServerInfo import kafka.metrics.{KafkaMetricsGroup, LinuxIoMetricsCollector} import kafka.network.{DataPlaneAcceptor, SocketServer} -import kafka.raft.RaftManager +import kafka.raft.KafkaRaftManager import kafka.security.CredentialProvider import kafka.server.KafkaConfig.{AlterConfigPolicyClassNameProp, CreateTopicPolicyClassNameProp} import kafka.server.KafkaRaftServer.BrokerRole import kafka.server.QuotaFactory.QuotaManagers +import kafka.server.metadata.DynamicConfigPublisher import kafka.utils.{CoreUtils, Logging} -import org.apache.kafka.clients.ApiVersions import org.apache.kafka.common.message.ApiMessageType.ListenerType -import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.security.scram.internals.ScramMechanism import org.apache.kafka.common.security.token.delegation.internals.DelegationTokenCache -import org.apache.kafka.common.utils.{LogContext, Time} +import org.apache.kafka.common.utils.LogContext import org.apache.kafka.common.{ClusterResource, Endpoint} -import org.apache.kafka.controller.{Controller, ControllerMetrics, QuorumController, QuorumFeatures} +import org.apache.kafka.controller.{Controller, QuorumController, QuorumControllerMetrics, QuorumFeatures} import org.apache.kafka.metadata.KafkaConfigSchema import org.apache.kafka.raft.RaftConfig -import org.apache.kafka.raft.RaftConfig.AddressSpec import org.apache.kafka.server.authorizer.Authorizer -import org.apache.kafka.server.common.ApiMessageAndVersion import org.apache.kafka.common.config.ConfigException import org.apache.kafka.metadata.authorizer.ClusterMetadataAuthorizer import org.apache.kafka.metadata.bootstrap.BootstrapMetadata -import org.apache.kafka.server.fault.FaultHandler +import org.apache.kafka.server.common.ApiMessageAndVersion import org.apache.kafka.server.metrics.KafkaYammerMetrics import org.apache.kafka.server.policy.{AlterConfigPolicy, CreateTopicPolicy} +import scala.collection.{Map, immutable} import scala.jdk.CollectionConverters._ import scala.compat.java8.OptionConverters._ @@ -57,23 +54,16 @@ import scala.compat.java8.OptionConverters._ * A Kafka controller that runs in KRaft (Kafka Raft) mode. */ class ControllerServer( - val metaProperties: MetaProperties, - val config: KafkaConfig, - val raftManager: RaftManager[ApiMessageAndVersion], - val time: Time, - val metrics: Metrics, - val controllerMetrics: ControllerMetrics, - val threadNamePrefix: Option[String], - val controllerQuorumVotersFuture: CompletableFuture[util.Map[Integer, AddressSpec]], + val jointServer: JointServer, val configSchema: KafkaConfigSchema, - val raftApiVersions: ApiVersions, val bootstrapMetadata: BootstrapMetadata, - val metadataFaultHandler: FaultHandler, - val fatalFaultHandler: FaultHandler, ) extends Logging with KafkaMetricsGroup { - import kafka.server.Server._ + val config = jointServer.config + val time = jointServer.time + val metrics = jointServer.metrics + def raftManager: KafkaRaftManager[ApiMessageAndVersion] = jointServer.raftManager - config.dynamicConfig.initialize(zkClientOpt = None) + import kafka.server.Server._ val lock = new ReentrantLock() val awaitShutdownCond = lock.newCondition() @@ -87,10 +77,13 @@ class ControllerServer( val socketServerFirstBoundPortFuture = new CompletableFuture[Integer]() var createTopicPolicy: Option[CreateTopicPolicy] = None var alterConfigPolicy: Option[AlterConfigPolicy] = None + var controllerMetrics: QuorumControllerMetrics = _ var controller: Controller = _ var quotaManagers: QuotaManagers = _ var controllerApis: ControllerApis = _ var controllerApisHandlerPool: KafkaRequestHandlerPool = _ + var dynamicConfigPublisher: DynamicConfigPublisher = _ + def kafkaYammerMetrics: KafkaYammerMetrics = KafkaYammerMetrics.INSTANCE private def maybeChangeStatus(from: ProcessStatus, to: ProcessStatus): Boolean = { lock.lock() @@ -111,7 +104,7 @@ class ControllerServer( new DynamicMetricReporterState(config.nodeId, config, metrics, clusterId) } - def clusterId: String = metaProperties.clusterId + def clusterId: String = jointServer.metaProps.clusterId def startup(): Unit = { if (!maybeChangeStatus(SHUTDOWN, STARTING)) return @@ -171,15 +164,18 @@ class ControllerServer( throw new ConfigException("No controller.listener.names defined for controller") } - val threadNamePrefixAsString = threadNamePrefix.getOrElse("") + jointServer.startForController() createTopicPolicy = Option(config. getConfiguredInstance(CreateTopicPolicyClassNameProp, classOf[CreateTopicPolicy])) alterConfigPolicy = Option(config. getConfiguredInstance(AlterConfigPolicyClassNameProp, classOf[AlterConfigPolicy])) - val controllerNodes = RaftConfig.voterConnectionsToNodes(controllerQuorumVotersFuture.get()) - val quorumFeatures = QuorumFeatures.create(config.nodeId, raftApiVersions, QuorumFeatures.defaultFeatureMap(), controllerNodes) + val controllerNodes = RaftConfig.voterConnectionsToNodes(jointServer.controllerQuorumVotersFuture.get()) + val quorumFeatures = QuorumFeatures.create(config.nodeId, + jointServer.raftManager.apiVersions, + QuorumFeatures.defaultFeatureMap(), + controllerNodes) val controllerBuilder = { val leaderImbalanceCheckIntervalNs = if (config.autoLeaderRebalanceEnable) { @@ -190,26 +186,25 @@ class ControllerServer( val maxIdleIntervalNs = config.metadataMaxIdleIntervalNs.fold(OptionalLong.empty)(OptionalLong.of) - new QuorumController.Builder(config.nodeId, metaProperties.clusterId). + new QuorumController.Builder(config.nodeId, jointServer.metaProps.clusterId). setTime(time). - setThreadNamePrefix(threadNamePrefixAsString). + setThreadNamePrefix(jointServer.threadNamePrefix.getOrElse("")). setConfigSchema(configSchema). - setRaftClient(raftManager.client). + setRaftClient(jointServer.raftManager.client). setQuorumFeatures(quorumFeatures). setDefaultReplicationFactor(config.defaultReplicationFactor.toShort). setDefaultNumPartitions(config.numPartitions.intValue()). setSessionTimeoutNs(TimeUnit.NANOSECONDS.convert(config.brokerSessionTimeoutMs.longValue(), TimeUnit.MILLISECONDS)). - setSnapshotMaxNewRecordBytes(config.metadataSnapshotMaxNewRecordBytes). setLeaderImbalanceCheckIntervalNs(leaderImbalanceCheckIntervalNs). setMaxIdleIntervalNs(maxIdleIntervalNs). - setMetrics(controllerMetrics). + setMetrics(jointServer.controllerMetrics). setCreateTopicPolicy(createTopicPolicy.asJava). setAlterConfigPolicy(alterConfigPolicy.asJava). setConfigurationValidator(new ControllerConfigurationValidator()). setStaticConfig(config.originals). setBootstrapMetadata(bootstrapMetadata). - setFatalFaultHandler(fatalFaultHandler) + setFatalFaultHandler(jointServer.quorumControllerFaultHandler) } authorizer match { case Some(a: ClusterMetadataAuthorizer) => controllerBuilder.setAuthorizer(a) @@ -222,15 +217,15 @@ class ControllerServer( doRemoteKraftSetup() } - quotaManagers = QuotaFactory.instantiate(config, metrics, time, threadNamePrefix.getOrElse("")) + quotaManagers = QuotaFactory.instantiate(config, metrics, time, jointServer.threadNamePrefix.getOrElse("")) controllerApis = new ControllerApis(socketServer.dataPlaneRequestChannel, authorizer, quotaManagers, time, controller, - raftManager, + jointServer.raftManager, config, - metaProperties, + jointServer.metaProps, controllerNodes.asScala.toSeq, apiVersionManager) controllerApisHandlerPool = new KafkaRequestHandlerPool(config.nodeId, @@ -258,6 +253,18 @@ class ControllerServer( shutdown() throw e } + config.dynamicConfig.addReconfigurables(this) + if (!config.processRoles.contains(BrokerRole)) { + // In standalone mode, install a DynamicConfigPublisher. + // In combined mode, we can rely on the broker's DynamicConfigPublisher. + val dynamicConfigHandlers = immutable.Map[String, ConfigHandler]( + ConfigType.Broker -> new BrokerConfigHandler(config, quotaManagers)) + dynamicConfigPublisher = new DynamicConfigPublisher( + config, + jointServer.metadataPublishingFaultHandler, + dynamicConfigHandlers) + jointServer.metadataLoader.installPublishers(List(dynamicConfigPublisher).asJava) + } } def shutdown(): Unit = { @@ -266,8 +273,13 @@ class ControllerServer( info("shutting down") if (socketServer != null) CoreUtils.swallow(socketServer.stopProcessingRequests(), this) - if (controller != null) + if (controller != null) { controller.beginShutdown() + } + if (controllerMetrics != null) { + CoreUtils.swallow(controllerApis.close(), this) + controllerMetrics = null + } if (socketServer != null) CoreUtils.swallow(socketServer.shutdown(), this) if (controllerApisHandlerPool != null) @@ -282,6 +294,7 @@ class ControllerServer( createTopicPolicy.foreach(policy => CoreUtils.swallow(policy.close(), this)) alterConfigPolicy.foreach(policy => CoreUtils.swallow(policy.close(), this)) socketServerFirstBoundPortFuture.completeExceptionally(new RuntimeException("shutting down")) + jointServer.stopForController() } catch { case e: Throwable => fatal("Fatal error during controller shutdown.", e) diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala index 38f7722d34378..bfba7fc1e756b 100755 --- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala +++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala @@ -25,6 +25,7 @@ import kafka.cluster.EndPoint import kafka.log.{LogCleaner, LogConfig, LogManager, ProducerStateManagerConfig} import kafka.network.{DataPlaneAcceptor, SocketServer} import kafka.server.DynamicBrokerConfig._ +import kafka.server.KafkaRaftServer.BrokerRole import kafka.utils.{CoreUtils, Logging, PasswordEncoder} import kafka.utils.Implicits._ import kafka.zk.{AdminZkClient, KafkaZkClient} @@ -269,6 +270,31 @@ class DynamicBrokerConfig(private val kafkaConfig: KafkaConfig) extends Logging addBrokerReconfigurable(kafkaServer.logManager.producerStateManagerConfig) } + /** + * Add reconfigurables to be notified when a dynamic controller config is updated. + */ + def addReconfigurables(controller: ControllerServer): Unit = { + controller.authorizer match { + case Some(authz: Reconfigurable) => addReconfigurable(authz) + case _ => + } + if (!kafkaConfig.processRoles.contains(BrokerRole)) { + // In standalone mode, add these reconfigurables. + // In combined mode, we can rely on the broker to add them. + addReconfigurable(controller.kafkaYammerMetrics) + addReconfigurable(new DynamicMetricsReporters( + kafkaConfig.brokerId, controller.config, controller.metrics, controller.clusterId)) + } + addBrokerReconfigurable(controller.socketServer) + + // KAFKA-14349: add dynamic thread pool resizing here + + // KAFKA-14350: add dynamic listener reconfiguration here + + // When we implement controller mutation quotas in KRaft, as discussed in KAFKA-14351, + // we'll need to make them reconfigurable here (probably via DynamicClientQuotaCallback). + } + def addReconfigurable(reconfigurable: Reconfigurable): Unit = CoreUtils.inWriteLock(lock) { verifyReconfigurableConfigs(reconfigurable.reconfigurableConfigs.asScala) reconfigurables.add(reconfigurable) diff --git a/core/src/main/scala/kafka/server/JointServer.scala b/core/src/main/scala/kafka/server/JointServer.scala new file mode 100644 index 0000000000000..dc2e10c6c3b6d --- /dev/null +++ b/core/src/main/scala/kafka/server/JointServer.scala @@ -0,0 +1,275 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.server + +import kafka.raft.KafkaRaftManager +import kafka.server.KafkaRaftServer.{BrokerRole, ControllerRole} +import kafka.server.metadata.BrokerServerMetrics +import kafka.utils.{CoreUtils, Logging} +import org.apache.kafka.common.metrics.Metrics +import org.apache.kafka.common.utils.{LogContext, Time} +import org.apache.kafka.controller.QuorumControllerMetrics +import org.apache.kafka.image.loader.MetadataLoader +import org.apache.kafka.image.publisher.{SnapshotEmitter, SnapshotGenerator} +import org.apache.kafka.metadata.MetadataRecordSerde +import org.apache.kafka.raft.RaftConfig.AddressSpec +import org.apache.kafka.server.common.ApiMessageAndVersion +import org.apache.kafka.server.fault.{FaultHandler, LoggingFaultHandler, ProcessExitingFaultHandler} +import org.apache.kafka.server.metrics.KafkaYammerMetrics + +import java.util +import java.util.concurrent.{CompletableFuture, TimeUnit} + + +/** + * Creates a fault handler. + */ +trait FaultHandlerFactory { + def build( + name: String, + fatal: Boolean, + action: Runnable + ): FaultHandler +} + +/** + * The standard FaultHandlerFactory which is used when we're not in a junit test. + */ +class StandardFaultHandlerFactory extends FaultHandlerFactory { + override def build( + name: String, + fatal: Boolean, + action: Runnable + ): FaultHandler = { + if (fatal) { + new ProcessExitingFaultHandler(action) + } else { + new LoggingFaultHandler(name, action) + } + } +} + +/** + * The JointServer manages the components which are shared between the BrokerServer and + * ControllerServer. These shared components include the Raft manager, snapshot generator, + * and metadata loader. A KRaft server running in combined mode as both a broker and a controller + * will still contain only a single JointServer instance. + * + * The JointServer will be started as soon as either the broker or the controller needs it, + * via the appropriate function (startForBroker or startForController). Similarly, it will be + * stopped as soon as neither the broker nor the controller need it, via stopForBroker or + * stopForController. One way of thinking about this is that both the broker and the controller + * could hold a "reference" to this class, and we don't truly stop it until both have dropped + * their reference. We opted to use two booleans here rather than a reference count in order to + * make debugging easier and reduce the chance of resource leaks. + */ +class JointServer( + val config: KafkaConfig, + val metaProps: MetaProperties, + val time: Time, + val metrics: Metrics, + val threadNamePrefix: Option[String], + val controllerQuorumVotersFuture: CompletableFuture[util.Map[Integer, AddressSpec]], + val faultHandlerFactory: FaultHandlerFactory +) extends Logging { + private val logContext: LogContext = new LogContext(s"[JointServer id=${config.nodeId}] ") + this.logIdent = logContext.logPrefix + + var usedByBroker: Boolean = false + var usedByController: Boolean = false + var raftManager: KafkaRaftManager[ApiMessageAndVersion] = _ + var snapshotEmitter: SnapshotEmitter = _ + var snapshotGenerator: SnapshotGenerator = _ + var brokerMetrics: BrokerServerMetrics = _ + var controllerMetrics: QuorumControllerMetrics = _ + var metadataLoader: MetadataLoader = _ + + def isUsed(): Boolean = synchronized { + usedByController || usedByBroker + } + + /** + * The start function called by the broker. + */ + def startForBroker(): Unit = synchronized { + if (!isUsed()) { + start() + } + usedByBroker = true + } + + /** + * The start function called by the controller. + */ + def startForController(): Unit = synchronized { + if (!isUsed()) { + start() + } + usedByController = true + } + + /** + * The stop function called by the broker. + */ + def stopForBroker(): Unit = synchronized { + if (usedByBroker) { + usedByBroker = false + if (!isUsed()) stop() + } + } + + /** + * The stop function called by the controller. + */ + def stopForController(): Unit = synchronized { + if (usedByController) { + usedByController = false + if (!isUsed()) stop() + } + } + + /** + * The fault handler to use when metadata loading fails. + */ + def metadataLoaderFaultHandler: FaultHandler = faultHandlerFactory.build("metadata loading", + config.processRoles.contains(ControllerRole), + () => { + if (snapshotGenerator != null) { + snapshotGenerator.disable("there was a metadata loading error") + } + if (brokerMetrics != null) brokerMetrics.metadataLoadErrorCount.getAndIncrement() + if (controllerMetrics != null) controllerMetrics.incrementMetadataErrorCount() + }) + + /** + * The fault handler to use when the initial broker metadata load fails. + */ + def initialBrokerMetadataLoadFaultHandler: FaultHandler = faultHandlerFactory.build("initial metadata loading", + true, + () => { + if (snapshotGenerator != null) { + snapshotGenerator.disable("there was an initial metadata loading error") + } + if (brokerMetrics != null) brokerMetrics.metadataApplyErrorCount.getAndIncrement() + if (controllerMetrics != null) controllerMetrics.incrementMetadataErrorCount() + }) + + /** + * The fault handler to use when the QuorumController experiences a fault. + */ + def quorumControllerFaultHandler: FaultHandler = faultHandlerFactory.build("quorum controller", + true, + () => {} + ) + + /** + * The fault handler to use when metadata cannot be published. + */ + def metadataPublishingFaultHandler: FaultHandler = faultHandlerFactory.build("metadata publishing", + false, + () => { + if (brokerMetrics != null) brokerMetrics.metadataApplyErrorCount.getAndIncrement() + if (controllerMetrics != null) controllerMetrics.incrementMetadataErrorCount() + }) + + private def start(): Unit = synchronized { + info("Starting JointServer") + try { + config.dynamicConfig.initialize(zkClientOpt = None) + + if (config.processRoles.contains(BrokerRole)) { + brokerMetrics = BrokerServerMetrics(metrics) + } + if (config.processRoles.contains(ControllerRole)) { + controllerMetrics = new QuorumControllerMetrics(KafkaYammerMetrics.defaultRegistry(), time) + } + raftManager = new KafkaRaftManager[ApiMessageAndVersion]( + metaProps, + config, + new MetadataRecordSerde, + KafkaRaftServer.MetadataPartition, + KafkaRaftServer.MetadataTopicId, + time, + metrics, + threadNamePrefix, + controllerQuorumVotersFuture) + raftManager.startup() + snapshotEmitter = new SnapshotEmitter.Builder(). + setNodeId(config.nodeId). + setRaftClient(raftManager.client). + build() + snapshotGenerator = new SnapshotGenerator.Builder(snapshotEmitter). + setTime(time). + setNodeId(config.nodeId). + setFaultHandler(new LoggingFaultHandler("snapshot generation", () => { })). + setMinBytesSinceLastSnapshot(config.metadataSnapshotMaxNewRecordBytes). + setMaxTimeSinceLastSnapshotNs(TimeUnit.MILLISECONDS.toNanos(config.metadataSnapshotMaxIntervalMsProp)). + build() + val loaderBuilder = new MetadataLoader.Builder(). + setTime(time). + setNodeId(config.nodeId). + setFaultHandler(metadataLoaderFaultHandler) + if (config.processRoles.contains(BrokerRole)) { + // In broker-only mode or combined mode, we want to expose the broker metrics. + loaderBuilder.setMetadataLoaderMetrics(brokerMetrics) + } + metadataLoader = loaderBuilder.build() + metadataLoader.installPublishers(util.Arrays.asList(snapshotGenerator)) + raftManager.client.register(metadataLoader) + debug("Completed JointServer startup.") + } catch { + case e: Throwable => { + error("Got exception while starting JointServer", e) + stop() + } + } + } + + private def stop(): Unit = { + info("Stopping JointServer") + + // Begin shutting these down in parallel to save time. + if (metadataLoader != null) metadataLoader.beginShutdown() + if (snapshotGenerator != null) snapshotGenerator.beginShutdown() + + // Close and null out components in reverse order of construction. + if (metadataLoader != null) { + CoreUtils.swallow(metadataLoader.close(), this) + metadataLoader = null + } + if (snapshotGenerator != null) { + CoreUtils.swallow(snapshotGenerator.close(), this) + snapshotGenerator = null + } + snapshotEmitter = null + if (raftManager != null) { + CoreUtils.swallow(raftManager.shutdown(), this) + raftManager = null + } + if (controllerMetrics != null) { + CoreUtils.swallow(controllerMetrics.close(), this) + controllerMetrics = null + } + if (brokerMetrics != null) { + CoreUtils.swallow(brokerMetrics.close(), this) + brokerMetrics = null + } + // Clear all reconfigurable instances stored in DynamicBrokerConfig + config.dynamicConfig.clear() + } +} diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala index 6fcc7ba40b132..20ded50980c5e 100755 --- a/core/src/main/scala/kafka/server/KafkaConfig.scala +++ b/core/src/main/scala/kafka/server/KafkaConfig.scala @@ -81,6 +81,7 @@ object Defaults { val BrokerHeartbeatIntervalMs = 2000 val BrokerSessionTimeoutMs = 9000 val MetadataSnapshotMaxNewRecordBytes = 20 * 1024 * 1024 + val MetadataSnapshotMaxIntervalMs = TimeUnit.HOURS.toMillis(1) val MetadataMaxIdleIntervalMs = 500 /** KRaft mode configs */ @@ -400,6 +401,7 @@ object KafkaConfig { val NodeIdProp = "node.id" val MetadataLogDirProp = "metadata.log.dir" val MetadataSnapshotMaxNewRecordBytesProp = "metadata.log.max.record.bytes.between.snapshots" + val MetadataSnapshotMaxIntervalMsProp = "metadata.log.max.snapshot.interval.ms" val ControllerListenerNamesProp = "controller.listener.names" val SaslMechanismControllerProtocolProp = "sasl.mechanism.controller.protocol" val MetadataLogSegmentMinBytesProp = "metadata.log.segment.min.bytes" @@ -726,6 +728,10 @@ object KafkaConfig { val MetadataLogDirDoc = "This configuration determines where we put the metadata log for clusters in KRaft mode. " + "If it is not set, the metadata log is placed in the first log directory from log.dirs." val MetadataSnapshotMaxNewRecordBytesDoc = "This is the maximum number of bytes in the log between the latest snapshot and the high-watermark needed before generating a new snapshot." + val MetadataSnapshotMaxIntervalMsDoc = "This is the maximum number of milliseconds that can elapse between the previous metadata snapshot and the next one. " + + "In other words, metadata snapshots will be generated at least as frequently as specified by this configuration. Note that if " + + s"${MetadataSnapshotMaxNewRecordBytesProp} bytes of new data is reached before the full specified interval completes, then " + + "the snapshot will still be generated." val MetadataMaxIdleIntervalMsDoc = "This configuration controls how often the active " + "controller should write no-op records to the metadata partition. If the value is 0, no-op records " + s"are not appended to the metadata partition. The default value is ${Defaults.MetadataMaxIdleIntervalMs}"; @@ -1157,10 +1163,7 @@ object KafkaConfig { * KRaft mode configs. */ .define(MetadataSnapshotMaxNewRecordBytesProp, LONG, Defaults.MetadataSnapshotMaxNewRecordBytes, atLeast(1), HIGH, MetadataSnapshotMaxNewRecordBytesDoc) - - /* - * KRaft mode private configs. Note that these configs are defined as internal. We will make them public in the 3.0.0 release. - */ + .define(MetadataSnapshotMaxIntervalMsProp, LONG, Defaults.MetadataSnapshotMaxIntervalMs, atLeast(0), HIGH, MetadataSnapshotMaxIntervalMsDoc) .define(ProcessRolesProp, LIST, Collections.emptyList(), ValidList.in("broker", "controller"), HIGH, ProcessRolesDoc) .define(NodeIdProp, INT, Defaults.EmptyNodeId, null, HIGH, NodeIdDoc) .define(InitialBrokerRegistrationTimeoutMsProp, INT, Defaults.InitialBrokerRegistrationTimeoutMs, null, MEDIUM, InitialBrokerRegistrationTimeoutMsDoc) @@ -1697,6 +1700,7 @@ class KafkaConfig private(doLog: Boolean, val props: java.util.Map[_, _], dynami /************* Metadata Configuration ***********/ val metadataSnapshotMaxNewRecordBytes = getLong(KafkaConfig.MetadataSnapshotMaxNewRecordBytesProp) + val metadataSnapshotMaxIntervalMsProp = getLong(KafkaConfig.MetadataSnapshotMaxIntervalMsProp) val metadataMaxIdleIntervalNs: Option[Long] = { val value = TimeUnit.NANOSECONDS.convert(getInt(KafkaConfig.MetadataMaxIdleIntervalMsProp).toLong, TimeUnit.MILLISECONDS) if (value > 0) Some(value) else None diff --git a/core/src/main/scala/kafka/server/KafkaRaftServer.scala b/core/src/main/scala/kafka/server/KafkaRaftServer.scala index 84ea10da57fa1..68f958ed9036c 100644 --- a/core/src/main/scala/kafka/server/KafkaRaftServer.scala +++ b/core/src/main/scala/kafka/server/KafkaRaftServer.scala @@ -21,20 +21,15 @@ import java.util.concurrent.CompletableFuture import kafka.common.InconsistentNodeIdException import kafka.log.{LogConfig, UnifiedLog} import kafka.metrics.KafkaMetricsReporter -import kafka.raft.KafkaRaftManager import kafka.server.KafkaRaftServer.{BrokerRole, ControllerRole} -import kafka.server.metadata.BrokerServerMetrics import kafka.utils.{CoreUtils, Logging, Mx4jLoader, VerifiableProperties} import org.apache.kafka.common.config.{ConfigDef, ConfigResource} import org.apache.kafka.common.internals.Topic import org.apache.kafka.common.utils.{AppInfoParser, Time} import org.apache.kafka.common.{KafkaException, Uuid} -import org.apache.kafka.controller.QuorumControllerMetrics import org.apache.kafka.metadata.bootstrap.{BootstrapDirectory, BootstrapMetadata} -import org.apache.kafka.metadata.{KafkaConfigSchema, MetadataRecordSerde} +import org.apache.kafka.metadata.KafkaConfigSchema import org.apache.kafka.raft.RaftConfig -import org.apache.kafka.server.common.ApiMessageAndVersion -import org.apache.kafka.server.fault.{LoggingFaultHandler, ProcessExitingFaultHandler} import org.apache.kafka.server.metrics.KafkaYammerMetrics import java.util.Optional @@ -69,62 +64,30 @@ class KafkaRaftServer( private val controllerQuorumVotersFuture = CompletableFuture.completedFuture( RaftConfig.parseVoterConnections(config.quorumVoters)) - private val raftManager = new KafkaRaftManager[ApiMessageAndVersion]( - metaProps, + private val jointServer = new JointServer( config, - new MetadataRecordSerde, - KafkaRaftServer.MetadataPartition, - KafkaRaftServer.MetadataTopicId, + metaProps, time, metrics, threadNamePrefix, - controllerQuorumVotersFuture + controllerQuorumVotersFuture, + new StandardFaultHandlerFactory(), ) private val broker: Option[BrokerServer] = if (config.processRoles.contains(BrokerRole)) { - val brokerMetrics = BrokerServerMetrics(metrics) - val fatalFaultHandler = new ProcessExitingFaultHandler() - val metadataLoadingFaultHandler = new LoggingFaultHandler("metadata loading", - () => brokerMetrics.metadataLoadErrorCount.getAndIncrement()) - val metadataApplyingFaultHandler = new LoggingFaultHandler("metadata application", - () => brokerMetrics.metadataApplyErrorCount.getAndIncrement()) Some(new BrokerServer( - config, - metaProps, - raftManager, - time, - metrics, - brokerMetrics, - threadNamePrefix, + jointServer, offlineDirs, - controllerQuorumVotersFuture, - fatalFaultHandler, - metadataLoadingFaultHandler, - metadataApplyingFaultHandler )) } else { None } private val controller: Option[ControllerServer] = if (config.processRoles.contains(ControllerRole)) { - val controllerMetrics = new QuorumControllerMetrics(KafkaYammerMetrics.defaultRegistry(), time) - val metadataFaultHandler = new LoggingFaultHandler("controller metadata", - () => controllerMetrics.incrementMetadataErrorCount()) - val fatalFaultHandler = new ProcessExitingFaultHandler() Some(new ControllerServer( - metaProps, - config, - raftManager, - time, - metrics, - controllerMetrics, - threadNamePrefix, - controllerQuorumVotersFuture, + jointServer, KafkaRaftServer.configSchema, - raftManager.apiVersions, bootstrapMetadata, - metadataFaultHandler, - fatalFaultHandler )) } else { None @@ -132,9 +95,6 @@ class KafkaRaftServer( override def startup(): Unit = { Mx4jLoader.maybeLoad() - // Note that we startup `RaftManager` first so that the controller and broker - // can register listeners during initialization. - raftManager.startup() controller.foreach(_.startup()) broker.foreach(_.startup()) AppInfoParser.registerAppInfo(Server.MetricsPrefix, config.brokerId.toString, metrics, time.milliseconds()) @@ -142,22 +102,16 @@ class KafkaRaftServer( } override def shutdown(): Unit = { + // The last component to be shut down will stop JointServer. broker.foreach(_.shutdown()) - // The order of shutdown for `RaftManager` and `ControllerServer` is backwards - // compared to `startup()`. This is because the `SocketServer` implementation that - // we rely on to receive requests is owned by `ControllerServer`, so we need it - // to stick around until graceful shutdown of `RaftManager` can be completed. - raftManager.shutdown() controller.foreach(_.shutdown()) CoreUtils.swallow(AppInfoParser.unregisterAppInfo(Server.MetricsPrefix, config.brokerId.toString, metrics), this) - } override def awaitShutdown(): Unit = { broker.foreach(_.awaitShutdown()) controller.foreach(_.awaitShutdown()) } - } object KafkaRaftServer { diff --git a/core/src/main/scala/kafka/server/metadata/BrokerMetadataListener.scala b/core/src/main/scala/kafka/server/metadata/BrokerMetadataListener.scala deleted file mode 100644 index 179d934ddad8b..0000000000000 --- a/core/src/main/scala/kafka/server/metadata/BrokerMetadataListener.scala +++ /dev/null @@ -1,407 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package kafka.server.metadata - -import java.util -import java.util.concurrent.{CompletableFuture, TimeUnit} -import kafka.metrics.KafkaMetricsGroup -import org.apache.kafka.image.{MetadataDelta, MetadataImage} -import org.apache.kafka.common.utils.{LogContext, Time} -import org.apache.kafka.image.writer.{ImageWriterOptions, RecordListWriter} -import org.apache.kafka.metadata.util.SnapshotReason -import org.apache.kafka.queue.{EventQueue, KafkaEventQueue} -import org.apache.kafka.raft.{Batch, BatchReader, LeaderAndEpoch, RaftClient} -import org.apache.kafka.server.common.ApiMessageAndVersion -import org.apache.kafka.server.fault.FaultHandler -import org.apache.kafka.snapshot.SnapshotReader - -import java.util.concurrent.atomic.AtomicBoolean - - -object BrokerMetadataListener { - val MetadataBatchProcessingTimeUs = "MetadataBatchProcessingTimeUs" - val MetadataBatchSizes = "MetadataBatchSizes" -} - -class BrokerMetadataListener( - val brokerId: Int, - time: Time, - threadNamePrefix: Option[String], - val maxBytesBetweenSnapshots: Long, - val snapshotter: Option[MetadataSnapshotter], - brokerMetrics: BrokerServerMetrics, - _metadataLoadingFaultHandler: FaultHandler -) extends RaftClient.Listener[ApiMessageAndVersion] with KafkaMetricsGroup { - - private val metadataFaultOccurred = new AtomicBoolean(false) - private val metadataLoadingFaultHandler: FaultHandler = new FaultHandler() { - override def handleFault(failureMessage: String, cause: Throwable): RuntimeException = { - // If the broker has any kind of error handling metadata records or publishing a new image - // we will disable taking new snapshots in order to preserve the local metadata log. Once we - // encounter a metadata processing error, the broker will be in an undetermined state. - if (metadataFaultOccurred.compareAndSet(false, true)) { - error("Disabling metadata snapshots until this broker is restarted.") - } - _metadataLoadingFaultHandler.handleFault(failureMessage, cause) - } - } - - private val logContext = new LogContext(s"[BrokerMetadataListener id=$brokerId] ") - private val log = logContext.logger(classOf[BrokerMetadataListener]) - logIdent = logContext.logPrefix() - - /** - * A histogram tracking the time in microseconds it took to process batches of events. - */ - private val batchProcessingTimeHist = newHistogram(BrokerMetadataListener.MetadataBatchProcessingTimeUs) - - /** - * A histogram tracking the sizes of batches that we have processed. - */ - private val metadataBatchSizeHist = newHistogram(BrokerMetadataListener.MetadataBatchSizes) - - /** - * The highest metadata offset that we've seen. Written only from the event queue thread. - */ - @volatile var _highestOffset = -1L - - /** - * The highest metadata log time that we've seen. Written only from the event queue thread. - */ - private var _highestTimestamp = -1L - - /** - * The current broker metadata image. Accessed only from the event queue thread. - */ - private var _image = MetadataImage.EMPTY - - /** - * The current metadata delta. Accessed only from the event queue thread. - */ - private var _delta = new MetadataDelta(_image) - - /** - * The object to use to publish new metadata changes, or None if this listener has not - * been activated yet. Accessed only from the event queue thread. - */ - private var _publisher: Option[MetadataPublisher] = None - - /** - * The number of bytes of records that we have read since the last snapshot we took. - * This does not include records we read from a snapshot. - * Accessed only from the event queue thread. - */ - private var _bytesSinceLastSnapshot: Long = 0L - - /** - * The event queue which runs this listener. - */ - val eventQueue = new KafkaEventQueue(time, logContext, threadNamePrefix.getOrElse("")) - - /** - * Returns the highest metadata-offset. Thread-safe. - */ - def highestMetadataOffset: Long = _highestOffset - - /** - * Handle new metadata records. - */ - override def handleCommit(reader: BatchReader[ApiMessageAndVersion]): Unit = - eventQueue.append(new HandleCommitsEvent(reader)) - - class HandleCommitsEvent(reader: BatchReader[ApiMessageAndVersion]) - extends EventQueue.FailureLoggingEvent(log) { - override def run(): Unit = { - val results = try { - val loadResults = loadBatches(_delta, reader, None, None, None, None) - if (isDebugEnabled) { - debug(s"Loaded new commits: $loadResults") - } - loadResults - } catch { - case e: Throwable => - metadataLoadingFaultHandler.handleFault(s"Unable to load metadata commits " + - s"from the BatchReader starting at base offset ${reader.baseOffset()}", e) - return - } finally { - reader.close() - } - - _bytesSinceLastSnapshot = _bytesSinceLastSnapshot + results.numBytes - - val shouldTakeSnapshot: Set[SnapshotReason] = shouldSnapshot() - if (shouldTakeSnapshot.nonEmpty) { - maybeStartSnapshot(shouldTakeSnapshot) - } - - _publisher.foreach(publish) - } - } - - private def shouldSnapshot(): Set[SnapshotReason] = { - val metadataVersionHasChanged = metadataVersionChanged() - val maxBytesHaveExceeded = (_bytesSinceLastSnapshot >= maxBytesBetweenSnapshots) - - if (maxBytesHaveExceeded && metadataVersionHasChanged) { - Set(SnapshotReason.MetadataVersionChanged, SnapshotReason.MaxBytesExceeded) - } else if (maxBytesHaveExceeded) { - Set(SnapshotReason.MaxBytesExceeded) - } else if (metadataVersionHasChanged) { - Set(SnapshotReason.MetadataVersionChanged) - } else { - Set() - } - } - - private def metadataVersionChanged(): Boolean = { - // The _publisher is empty before starting publishing, and we won't compute feature delta - // until we starting publishing - _publisher.nonEmpty && Option(_delta.featuresDelta()).exists { featuresDelta => - featuresDelta.metadataVersionChange().isPresent - } - } - - private def maybeStartSnapshot(reason: Set[SnapshotReason]): Unit = { - snapshotter.foreach { snapshotter => - if (metadataFaultOccurred.get()) { - trace("Not starting metadata snapshot since we previously had an error") - } else if (snapshotter.maybeStartSnapshot(_highestTimestamp, _delta.apply(), reason)) { - _bytesSinceLastSnapshot = 0L - } - } - } - - /** - * Handle metadata snapshots - */ - override def handleSnapshot(reader: SnapshotReader[ApiMessageAndVersion]): Unit = - eventQueue.append(new HandleSnapshotEvent(reader)) - - class HandleSnapshotEvent(reader: SnapshotReader[ApiMessageAndVersion]) - extends EventQueue.FailureLoggingEvent(log) { - override def run(): Unit = { - val snapshotName = s"${reader.snapshotId().offset}-${reader.snapshotId().epoch}" - try { - info(s"Loading snapshot ${snapshotName}") - _delta = new MetadataDelta(_image) // Discard any previous deltas. - val loadResults = loadBatches(_delta, - reader, - Some(reader.lastContainedLogTimestamp), - Some(reader.lastContainedLogOffset), - Some(reader.lastContainedLogEpoch), - Some(snapshotName)) - try { - _delta.finishSnapshot() - } catch { - case e: Throwable => metadataLoadingFaultHandler.handleFault( - s"Error finishing snapshot ${snapshotName}", e) - } - info(s"Loaded snapshot ${snapshotName}: ${loadResults}") - } catch { - case t: Throwable => metadataLoadingFaultHandler.handleFault("Uncaught exception while " + - s"loading broker metadata from Metadata snapshot ${snapshotName}", t) - } finally { - reader.close() - } - _publisher.foreach(publish) - } - } - - case class BatchLoadResults(numBatches: Int, numRecords: Int, elapsedUs: Long, numBytes: Long) { - override def toString: String = { - s"$numBatches batch(es) with $numRecords record(s) in $numBytes bytes " + - s"ending at offset $highestMetadataOffset in $elapsedUs microseconds" - } - } - - /** - * Load and replay the batches to the metadata delta. - * - * When loading and replay a snapshot the appendTimestamp and snapshotId parameter should be provided. - * In a snapshot the append timestamp, offset and epoch reported by the batch is independent of the ones - * reported by the metadata log. - * - * @param delta metadata delta on which to replay the records - * @param iterator sequence of metadata record bacthes to replay - * @param lastAppendTimestamp optional append timestamp to use instead of the batches timestamp - * @param lastCommittedOffset optional offset to use instead of the batches offset - * @param lastCommittedEpoch optional epoch to use instead of the batches epoch - */ - private def loadBatches( - delta: MetadataDelta, - iterator: util.Iterator[Batch[ApiMessageAndVersion]], - lastAppendTimestamp: Option[Long], - lastCommittedOffset: Option[Long], - lastCommittedEpoch: Option[Int], - snapshotName: Option[String] - ): BatchLoadResults = { - val startTimeNs = time.nanoseconds() - var numBatches = 0 - var numRecords = 0 - var numBytes = 0L - - while (iterator.hasNext) { - val batch = iterator.next() - - val epoch = lastCommittedEpoch.getOrElse(batch.epoch()) - _highestTimestamp = lastAppendTimestamp.getOrElse(batch.appendTimestamp()) - - var index = 0 - batch.records().forEach { messageAndVersion => - if (isTraceEnabled) { - trace(s"Metadata batch ${batch.lastOffset}: processing [${index + 1}/${batch.records.size}]:" + - s" ${messageAndVersion.message}") - } - _highestOffset = lastCommittedOffset.getOrElse(batch.baseOffset() + index) - try { - delta.replay(highestMetadataOffset, epoch, messageAndVersion.message()) - } catch { - case e: Throwable => snapshotName match { - case None => metadataLoadingFaultHandler.handleFault( - s"Error replaying metadata log record at offset ${_highestOffset}", e) - case Some(name) => metadataLoadingFaultHandler.handleFault( - s"Error replaying record ${index} from snapshot ${name} at offset ${_highestOffset}", e) - } - } finally { - numRecords += 1 - index += 1 - } - } - numBytes = numBytes + batch.sizeInBytes() - metadataBatchSizeHist.update(batch.records().size()) - numBatches = numBatches + 1 - } - - val endTimeNs = time.nanoseconds() - val elapsedUs = TimeUnit.MICROSECONDS.convert(endTimeNs - startTimeNs, TimeUnit.NANOSECONDS) - batchProcessingTimeHist.update(elapsedUs) - BatchLoadResults(numBatches, numRecords, elapsedUs, numBytes) - } - - def startPublishing(publisher: MetadataPublisher): CompletableFuture[Void] = { - val event = new StartPublishingEvent(publisher) - eventQueue.append(event) - event.future - } - - class StartPublishingEvent(publisher: MetadataPublisher) - extends EventQueue.FailureLoggingEvent(log) { - val future = new CompletableFuture[Void]() - - override def run(): Unit = { - _publisher = Some(publisher) - log.info(s"Starting to publish metadata events at offset $highestMetadataOffset.") - try { - if (metadataVersionChanged()) { - maybeStartSnapshot(Set(SnapshotReason.MetadataVersionChanged)) - } - publish(publisher) - future.complete(null) - } catch { - case e: Throwable => - future.completeExceptionally(e) - throw e - } - } - } - - // This is used in tests to alter the publisher that is in use by the broker. - def alterPublisher(publisher: MetadataPublisher): CompletableFuture[Void] = { - val event = new AlterPublisherEvent(publisher) - eventQueue.append(event) - event.future - } - - class AlterPublisherEvent(publisher: MetadataPublisher) - extends EventQueue.FailureLoggingEvent(log) { - val future = new CompletableFuture[Void]() - - override def run(): Unit = { - _publisher = Some(publisher) - log.info(s"Set publisher to ${publisher}") - future.complete(null) - } - } - - private def publish(publisher: MetadataPublisher): Unit = { - val delta = _delta - try { - _image = _delta.apply() - } catch { - case t: Throwable => - // If we cannot apply the delta, this publish event will throw and we will not publish a new image. - // The broker will continue applying metadata records and attempt to publish again. - throw metadataLoadingFaultHandler.handleFault(s"Error applying metadata delta $delta", t) - } - - _delta = new MetadataDelta(_image) - if (isDebugEnabled) { - debug(s"Publishing new metadata delta $delta at offset ${_image.highestOffsetAndEpoch().offset}.") - } - - // This publish call is done with its own try-catch and fault handler - publisher.publish(delta, _image) - - // Update the metrics since the publisher handled the lastest image - brokerMetrics.lastAppliedRecordOffset.set(_highestOffset) - brokerMetrics.lastAppliedRecordTimestamp.set(_highestTimestamp) - } - - override def handleLeaderChange(leaderAndEpoch: LeaderAndEpoch): Unit = { - // Nothing to do. - } - - override def beginShutdown(): Unit = { - eventQueue.beginShutdown("beginShutdown", new ShutdownEvent()) - } - - class ShutdownEvent extends EventQueue.FailureLoggingEvent(log) { - override def run(): Unit = { - brokerMetrics.close() - removeMetric(BrokerMetadataListener.MetadataBatchProcessingTimeUs) - removeMetric(BrokerMetadataListener.MetadataBatchSizes) - } - } - - def close(): Unit = { - beginShutdown() - eventQueue.close() - } - - // VisibleForTesting - private[kafka] def getImageRecords(): CompletableFuture[util.List[ApiMessageAndVersion]] = { - val future = new CompletableFuture[util.List[ApiMessageAndVersion]]() - eventQueue.append(new GetImageRecordsEvent(future)) - future - } - - class GetImageRecordsEvent(future: CompletableFuture[util.List[ApiMessageAndVersion]]) - extends EventQueue.FailureLoggingEvent(log) { - override def run(): Unit = { - val writer = new RecordListWriter() - val options = new ImageWriterOptions.Builder(). - setMetadataVersion(_image.features().metadataVersion()). - build() - try { - _image.write(writer, options) - } finally { - writer.close() - } - future.complete(writer.records()) - } - } -} diff --git a/core/src/main/scala/kafka/server/metadata/BrokerMetadataPublisher.scala b/core/src/main/scala/kafka/server/metadata/BrokerMetadataPublisher.scala index 0192bb4afcfd7..c98e007f01d4a 100644 --- a/core/src/main/scala/kafka/server/metadata/BrokerMetadataPublisher.scala +++ b/core/src/main/scala/kafka/server/metadata/BrokerMetadataPublisher.scala @@ -18,16 +18,15 @@ package kafka.server.metadata import java.util.Properties -import java.util.concurrent.atomic.AtomicLong import kafka.coordinator.group.GroupCoordinator import kafka.coordinator.transaction.TransactionCoordinator import kafka.log.{LogManager, UnifiedLog} -import kafka.server.ConfigAdminManager.toLoggableProps -import kafka.server.{ConfigEntityName, ConfigHandler, ConfigType, KafkaConfig, ReplicaManager, RequestLocal} +import kafka.server.{KafkaConfig, ReplicaManager, RequestLocal} import kafka.utils.Logging import org.apache.kafka.common.TopicPartition -import org.apache.kafka.common.config.ConfigResource.Type.{BROKER, TOPIC} import org.apache.kafka.common.internals.Topic +import org.apache.kafka.image.loader.{LogDeltaManifest, SnapshotManifest} +import org.apache.kafka.image.publisher.MetadataPublisher import org.apache.kafka.image.{MetadataDelta, MetadataImage, TopicDelta, TopicsImage} import org.apache.kafka.metadata.authorizer.ClusterMetadataAuthorizer import org.apache.kafka.server.authorizer.Authorizer @@ -103,7 +102,7 @@ class BrokerMetadataPublisher( groupCoordinator: GroupCoordinator, txnCoordinator: TransactionCoordinator, clientQuotaMetadataManager: ClientQuotaMetadataManager, - dynamicConfigHandlers: Map[String, ConfigHandler], + var dynamicConfigPublisher: DynamicConfigPublisher, private val _authorizer: Option[Authorizer], fatalFaultHandler: FaultHandler, metadataPublishingFaultHandler: FaultHandler @@ -122,12 +121,25 @@ class BrokerMetadataPublisher( */ var _firstPublish = true - /** - * This is updated after all components (e.g. LogManager) has finished publishing the new metadata delta - */ - val publishedOffsetAtomic = new AtomicLong(-1) + override def name(): String = "BrokerMetadataPublisher" + + override def publishSnapshot( + delta: MetadataDelta, + newImage: MetadataImage, + manifest: SnapshotManifest + ): Unit = { + publish(delta, newImage) + } + + override def publishLogDelta( + delta: MetadataDelta, + newImage: MetadataImage, + manifest: LogDeltaManifest + ): Unit = { + publish(delta, newImage) + } - override def publish(delta: MetadataDelta, newImage: MetadataImage): Unit = { + def publish(delta: MetadataDelta, newImage: MetadataImage): Unit = { val highestOffsetAndEpoch = newImage.highestOffsetAndEpoch() val deltaName = if (_firstPublish) { @@ -156,8 +168,9 @@ class BrokerMetadataPublisher( } Option(delta.featuresDelta()).foreach { featuresDelta => - featuresDelta.metadataVersionChange().ifPresent{ metadataVersion => - info(s"Updating metadata.version to ${metadataVersion.featureLevel()} at offset $highestOffsetAndEpoch.") + featuresDelta.metadataVersionChange().ifPresent{ change => + info(s"Changing metadata.version from ${change.oldVersion()} to ${change.newVersion()} in " + + s"delta ending at offset ${highestOffsetAndEpoch}.") } } @@ -211,61 +224,10 @@ class BrokerMetadataPublisher( } // Apply configuration deltas. - Option(delta.configsDelta()).foreach { configsDelta => - configsDelta.changes().keySet().forEach { resource => - val props = newImage.configs().configProperties(resource) - resource.`type`() match { - case TOPIC => - try { - // Apply changes to a topic's dynamic configuration. - info(s"Updating topic ${resource.name()} with new configuration : " + - toLoggableProps(resource, props).mkString(",")) - dynamicConfigHandlers(ConfigType.Topic). - processConfigChanges(resource.name(), props) - } catch { - case t: Throwable => metadataPublishingFaultHandler.handleFault("Error updating topic " + - s"${resource.name()} with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + - s"in ${deltaName}", t) - } - case BROKER => - if (resource.name().isEmpty) { - try { - // Apply changes to "cluster configs" (also known as default BROKER configs). - // These are stored in KRaft with an empty name field. - info("Updating cluster configuration : " + - toLoggableProps(resource, props).mkString(",")) - dynamicConfigHandlers(ConfigType.Broker). - processConfigChanges(ConfigEntityName.Default, props) - } catch { - case t: Throwable => metadataPublishingFaultHandler.handleFault("Error updating " + - s"cluster with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + - s"in ${deltaName}", t) - } - } else if (resource.name() == brokerId.toString) { - try { - // Apply changes to this broker's dynamic configuration. - info(s"Updating broker $brokerId with new configuration : " + - toLoggableProps(resource, props).mkString(",")) - dynamicConfigHandlers(ConfigType.Broker). - processConfigChanges(resource.name(), props) - // When applying a per broker config (not a cluster config), we also - // reload any associated file. For example, if the ssl.keystore is still - // set to /tmp/foo, we still want to reload /tmp/foo in case its contents - // have changed. This doesn't apply to topic configs or cluster configs. - reloadUpdatedFilesWithoutConfigChange(props) - } catch { - case t: Throwable => metadataPublishingFaultHandler.handleFault("Error updating " + - s"broker with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + - s"in ${deltaName}", t) - } - } - case _ => // nothing to do - } - } - } + dynamicConfigPublisher.publish(delta, newImage) + // Apply client quotas delta. try { - // Apply client quotas delta. Option(delta.clientQuotasDelta()).foreach { clientQuotasDelta => clientQuotaMetadataManager.update(clientQuotasDelta) } @@ -314,7 +276,6 @@ class BrokerMetadataPublisher( if (_firstPublish) { finishInitializingReplicaManager(newImage) } - publishedOffsetAtomic.set(newImage.highestOffsetAndEpoch().offset) } catch { case t: Throwable => metadataPublishingFaultHandler.handleFault("Uncaught exception while " + s"publishing broker metadata from ${deltaName}", t) @@ -323,8 +284,6 @@ class BrokerMetadataPublisher( } } - override def publishedOffset: Long = publishedOffsetAtomic.get() - def reloadUpdatedFilesWithoutConfigChange(props: Properties): Unit = { conf.dynamicConfig.reloadUpdatedFilesWithoutConfigChange(props) } @@ -430,5 +389,9 @@ class BrokerMetadataPublisher( case t: Throwable => metadataPublishingFaultHandler.handleFault("Error starting high " + "watermark checkpoint thread during startup", t) } -} + } + + override def close(): Unit = { + dynamicConfigPublisher.close() + } } diff --git a/core/src/main/scala/kafka/server/metadata/BrokerMetadataSnapshotter.scala b/core/src/main/scala/kafka/server/metadata/BrokerMetadataSnapshotter.scala deleted file mode 100644 index 6c3d55be92bcb..0000000000000 --- a/core/src/main/scala/kafka/server/metadata/BrokerMetadataSnapshotter.scala +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package kafka.server.metadata - -import java.util.concurrent.RejectedExecutionException -import kafka.utils.Logging -import org.apache.kafka.image.MetadataImage -import org.apache.kafka.common.utils.{LogContext, Time} -import org.apache.kafka.image.writer.{ImageWriterOptions, RaftSnapshotWriter} -import org.apache.kafka.metadata.util.SnapshotReason -import org.apache.kafka.queue.{EventQueue, KafkaEventQueue} -import org.apache.kafka.server.common.ApiMessageAndVersion -import org.apache.kafka.snapshot.SnapshotWriter - -trait SnapshotWriterBuilder { - def build(committedOffset: Long, - committedEpoch: Int, - lastContainedLogTime: Long): Option[SnapshotWriter[ApiMessageAndVersion]] -} - -class BrokerMetadataSnapshotter( - brokerId: Int, - val time: Time, - threadNamePrefix: Option[String], - writerBuilder: SnapshotWriterBuilder -) extends Logging with MetadataSnapshotter { - /** - * The maximum number of records we will put in each batch. - * - * From the perspective of the Raft layer, the limit on batch size is specified in terms of - * bytes, not number of records. @See {@link KafkaRaftClient#MAX_BATCH_SIZE_BYTES} for details. - * However, it's more convenient to limit the batch size here in terms of number of records. - * So we chose a low number that will not cause problems. - */ - private val maxRecordsInBatch = 1024 - - private val logContext = new LogContext(s"[BrokerMetadataSnapshotter id=$brokerId] ") - logIdent = logContext.logPrefix() - - /** - * The offset of the snapshot in progress, or -1 if there isn't one. Accessed only under - * the object lock. - */ - private var _currentSnapshotOffset = -1L - - /** - * The event queue which runs this listener. - */ - val eventQueue = new KafkaEventQueue(time, logContext, threadNamePrefix.getOrElse("")) - - override def maybeStartSnapshot(lastContainedLogTime: Long, image: MetadataImage, snapshotReasons: Set[SnapshotReason]): Boolean = synchronized { - if (_currentSnapshotOffset != -1) { - info(s"Declining to create a new snapshot at ${image.highestOffsetAndEpoch()} because " + - s"there is already a snapshot in progress at offset ${_currentSnapshotOffset}") - false - } else { - val writer = writerBuilder.build( - image.highestOffsetAndEpoch().offset, - image.highestOffsetAndEpoch().epoch, - lastContainedLogTime - ) - if (writer.nonEmpty) { - _currentSnapshotOffset = image.highestOffsetAndEpoch().offset - - info(s"Creating a new snapshot at offset ${_currentSnapshotOffset} because, ${snapshotReasons.mkString(" and ")}") - eventQueue.append(new CreateSnapshotEvent(image, writer.get)) - true - } else { - info(s"Declining to create a new snapshot at ${image.highestOffsetAndEpoch()} because " + - s"there is already a snapshot at offset ${image.highestOffsetAndEpoch().offset}") - false - } - } - } - - class CreateSnapshotEvent( - image: MetadataImage, - snapshotWriter: SnapshotWriter[ApiMessageAndVersion] - ) extends EventQueue.Event { - - override def run(): Unit = { - val writer = new RaftSnapshotWriter(snapshotWriter, maxRecordsInBatch) - val options = new ImageWriterOptions.Builder(). - setMetadataVersion(image.features().metadataVersion()). - build() - try { - image.write(writer, options) - } finally { - try { - writer.close() - } finally { - BrokerMetadataSnapshotter.this.synchronized { - _currentSnapshotOffset = -1L - } - } - } - } - - override def handleException(e: Throwable): Unit = { - e match { - case _: RejectedExecutionException => - info("Not processing CreateSnapshotEvent because the event queue is closed.") - case _ => error("Unexpected error handling CreateSnapshotEvent", e) - } - } - } - - def beginShutdown(): Unit = { - eventQueue.beginShutdown("beginShutdown", new ShutdownEvent()) - } - - class ShutdownEvent() extends EventQueue.Event { - override def run(): Unit = { - } - } - - def close(): Unit = { - beginShutdown() - eventQueue.close() - } -} diff --git a/core/src/main/scala/kafka/server/metadata/BrokerServerMetrics.scala b/core/src/main/scala/kafka/server/metadata/BrokerServerMetrics.scala index 3e68ae85f9232..c52d6254a06be 100644 --- a/core/src/main/scala/kafka/server/metadata/BrokerServerMetrics.scala +++ b/core/src/main/scala/kafka/server/metadata/BrokerServerMetrics.scala @@ -17,17 +17,48 @@ package kafka.server.metadata -import java.util.concurrent.atomic.AtomicLong +import kafka.metrics.KafkaMetricsGroup + +import java.util.concurrent.atomic.{AtomicLong, AtomicReference} import org.apache.kafka.common.MetricName import org.apache.kafka.common.metrics.Gauge import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.metrics.MetricConfig +import org.apache.kafka.image.MetadataProvenance +import org.apache.kafka.image.loader.MetadataLoaderMetrics +import org.apache.kafka.server.metrics.KafkaYammerMetrics + +import java.util.concurrent.TimeUnit.{MICROSECONDS, NANOSECONDS} -final class BrokerServerMetrics private (metrics: Metrics) extends AutoCloseable { +final class BrokerServerMetrics private ( + metrics: Metrics +) extends MetadataLoaderMetrics with KafkaMetricsGroup { import BrokerServerMetrics._ - val lastAppliedRecordOffset: AtomicLong = new AtomicLong(0) - val lastAppliedRecordTimestamp: AtomicLong = new AtomicLong(0) + private val batchProcessingTimeHistName = explicitMetricName("kafka.server", + "BrokerMetadataListener", + "MetadataBatchProcessingTimeUs", + Map.empty) + + /** + * A histogram tracking the time in microseconds it took to process batches of events. + */ + private val batchProcessingTimeHist = + KafkaYammerMetrics.defaultRegistry().newHistogram(batchProcessingTimeHistName, true) + + private val batchSizeHistName = explicitMetricName("kafka.server", + "BrokerMetadataListener", + "MetadataBatchSizes", + Map.empty) + + /** + * A histogram tracking the sizes of batches that we have processed. + */ + private val batchSizeHist = + KafkaYammerMetrics.defaultRegistry().newHistogram(batchSizeHistName, true) + + val lastAppliedImageProvenance: AtomicReference[MetadataProvenance] = + new AtomicReference[MetadataProvenance](MetadataProvenance.EMPTY) val metadataLoadErrorCount: AtomicLong = new AtomicLong(0) val metadataApplyErrorCount: AtomicLong = new AtomicLong(0) @@ -62,15 +93,15 @@ final class BrokerServerMetrics private (metrics: Metrics) extends AutoCloseable ) addMetric(metrics, lastAppliedRecordOffsetName) { _ => - lastAppliedRecordOffset.get + lastAppliedImageProvenance.get.offset() } addMetric(metrics, lastAppliedRecordTimestampName) { _ => - lastAppliedRecordTimestamp.get + lastAppliedImageProvenance.get.lastContainedLogTimeMs() } addMetric(metrics, lastAppliedRecordLagMsName) { now => - now - lastAppliedRecordTimestamp.get + now - lastAppliedImageProvenance.get.lastContainedLogTimeMs() } addMetric(metrics, metadataLoadErrorCountName) { _ => @@ -82,6 +113,8 @@ final class BrokerServerMetrics private (metrics: Metrics) extends AutoCloseable } override def close(): Unit = { + KafkaYammerMetrics.defaultRegistry().removeMetric(batchProcessingTimeHistName) + KafkaYammerMetrics.defaultRegistry().removeMetric(batchSizeHistName) List( lastAppliedRecordOffsetName, lastAppliedRecordTimestampName, @@ -90,6 +123,16 @@ final class BrokerServerMetrics private (metrics: Metrics) extends AutoCloseable metadataApplyErrorCountName ).foreach(metrics.removeMetric) } + + override def updateBatchProcessingTime(elapsedNs: Long): Unit = + batchProcessingTimeHist.update(MICROSECONDS.convert(elapsedNs, NANOSECONDS)) + + override def updateBatchSize(size: Int): Unit = batchSizeHist.update(size) + + override def updateLastAppliedImageProvenance(provenance: MetadataProvenance): Unit = + lastAppliedImageProvenance.set(provenance) + + override def lastAppliedOffset(): Long = lastAppliedImageProvenance.get().offset() } diff --git a/core/src/main/scala/kafka/server/metadata/DynamicConfigPublisher.scala b/core/src/main/scala/kafka/server/metadata/DynamicConfigPublisher.scala new file mode 100644 index 0000000000000..44ca513c6a6a4 --- /dev/null +++ b/core/src/main/scala/kafka/server/metadata/DynamicConfigPublisher.scala @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.server.metadata + +import java.util.Properties +import kafka.server.ConfigAdminManager.toLoggableProps +import kafka.server.{ConfigEntityName, ConfigHandler, ConfigType, KafkaConfig} +import kafka.utils.Logging +import org.apache.kafka.common.config.ConfigResource.Type.{BROKER, TOPIC} +import org.apache.kafka.image.loader.{LogDeltaManifest, SnapshotManifest} +import org.apache.kafka.image.publisher.MetadataPublisher +import org.apache.kafka.image.{MetadataDelta, MetadataImage} +import org.apache.kafka.server.fault.FaultHandler + + +class DynamicConfigPublisher( + conf: KafkaConfig, + faultHandler: FaultHandler, + dynamicConfigHandlers: Map[String, ConfigHandler], +) extends MetadataPublisher with Logging { + logIdent = s"[DynamicConfigPublisher id=${conf.nodeId}] " + + override def name(): String = "DynamicConfigPublisher" + + override def publishSnapshot( + delta: MetadataDelta, + newImage: MetadataImage, + manifest: SnapshotManifest + ): Unit = { + publish(delta, newImage) + } + + override def publishLogDelta( + delta: MetadataDelta, + newImage: MetadataImage, + manifest: LogDeltaManifest + ): Unit = { + publish(delta, newImage) + } + + def publish(delta: MetadataDelta, newImage: MetadataImage): Unit = { + val deltaName = s"MetadataDelta up to ${newImage.highestOffsetAndEpoch().offset}" + try { + // Apply configuration deltas. + Option(delta.configsDelta()).foreach { configsDelta => + configsDelta.changes().keySet().forEach { resource => + val props = newImage.configs().configProperties(resource) + resource.`type`() match { + case TOPIC => + dynamicConfigHandlers.get(ConfigType.Topic).foreach(topicConfigHandler => + try { + // Apply changes to a topic's dynamic configuration. + info(s"Updating topic ${resource.name()} with new configuration : " + + toLoggableProps(resource, props).mkString(",")) + topicConfigHandler.processConfigChanges(resource.name(), props) + } catch { + case t: Throwable => faultHandler.handleFault("Error updating topic " + + s"${resource.name()} with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + + s"in ${deltaName}", t) + } + ) + case BROKER => + dynamicConfigHandlers.get(ConfigType.Broker).foreach(nodeConfigHandler => + if (resource.name().isEmpty) { + try { + // Apply changes to "cluster configs" (also known as default BROKER configs). + // These are stored in KRaft with an empty name field. + info("Updating cluster configuration : " + + toLoggableProps(resource, props).mkString(",")) + nodeConfigHandler.processConfigChanges(ConfigEntityName.Default, props) + } catch { + case t: Throwable => faultHandler.handleFault("Error updating " + + s"cluster with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + + s"in ${deltaName}", t) + } + } else if (resource.name() == conf.nodeId.toString) { + try { + // Apply changes to this broker's dynamic configuration. + info(s"Updating node ${conf.nodeId} with new configuration : " + + toLoggableProps(resource, props).mkString(",")) + nodeConfigHandler.processConfigChanges(resource.name(), props) + // When applying a per broker config (not a cluster config), we also + // reload any associated file. For example, if the ssl.keystore is still + // set to /tmp/foo, we still want to reload /tmp/foo in case its contents + // have changed. This doesn't apply to topic configs or cluster configs. + reloadUpdatedFilesWithoutConfigChange(props) + } catch { + case t: Throwable => faultHandler.handleFault("Error updating " + + s"node with new configuration: ${toLoggableProps(resource, props).mkString(",")} " + + s"in ${deltaName}", t) + } + } + ) + case _ => // nothing to do + } + } + } + } catch { + case t: Throwable => faultHandler.handleFault("Uncaught exception while " + + s"publishing dynamic configuration changes from ${deltaName}", t) + } + } + + def reloadUpdatedFilesWithoutConfigChange(props: Properties): Unit = { + conf.dynamicConfig.reloadUpdatedFilesWithoutConfigChange(props) + } + + override def close(): Unit = { + // nothing to do + } +} diff --git a/core/src/main/scala/kafka/server/metadata/MetadataPublisher.scala b/core/src/main/scala/kafka/server/metadata/MetadataPublisher.scala deleted file mode 100644 index b63a2c056c05f..0000000000000 --- a/core/src/main/scala/kafka/server/metadata/MetadataPublisher.scala +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package kafka.server.metadata - -import org.apache.kafka.image.{MetadataDelta, MetadataImage} - -/** - * An object which publishes a new metadata image. - */ -trait MetadataPublisher { - /** - * Publish a new metadata image. - * - * @param delta The delta between the old image and the new one. - * @param newImage The new image, which is the result of applying the - * delta to the previous image. - */ - def publish(delta: MetadataDelta, newImage: MetadataImage): Unit - - /** - * The highest offset of metadata topic which has been published - */ - def publishedOffset: Long -} diff --git a/core/src/main/scala/kafka/server/metadata/MetadataSnapshotter.scala b/core/src/main/scala/kafka/server/metadata/MetadataSnapshotter.scala deleted file mode 100644 index b175bf78e4964..0000000000000 --- a/core/src/main/scala/kafka/server/metadata/MetadataSnapshotter.scala +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package kafka.server.metadata - -import org.apache.kafka.image.MetadataImage -import org.apache.kafka.metadata.util.SnapshotReason - - -/** - * Handles creating snapshots. - */ -trait MetadataSnapshotter { - /** - * If there is no other snapshot being written out, start writing out a snapshot. - * - * @param lastContainedLogTime The highest time contained in the snapshot. - * @param image The metadata image to write out. - * @param reason Set of reasons due to which a snapshot is being taken. - * - * @return True if we will write out a new snapshot; false otherwise. - */ - def maybeStartSnapshot(lastContainedLogTime: Long, image: MetadataImage, reason: Set[SnapshotReason]): Boolean -} diff --git a/core/src/test/java/kafka/test/MockController.java b/core/src/test/java/kafka/test/MockController.java index ff1154d2119cd..061e19213e397 100644 --- a/core/src/test/java/kafka/test/MockController.java +++ b/core/src/test/java/kafka/test/MockController.java @@ -436,11 +436,6 @@ synchronized public CompletableFuture> createP return CompletableFuture.completedFuture(results); } - @Override - public CompletableFuture beginWritingSnapshot() { - throw new UnsupportedOperationException(); - } - @Override public void beginShutdown() { this.active = false; diff --git a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java index 417c083457f49..d3d2bd71ad20f 100644 --- a/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java +++ b/core/src/test/java/kafka/testkit/KafkaClusterTestKit.java @@ -17,31 +17,28 @@ package kafka.testkit; -import kafka.raft.KafkaRaftManager; import kafka.server.BrokerServer; import kafka.server.ControllerServer; +import kafka.server.FaultHandlerFactory; +import kafka.server.JointServer; import kafka.server.KafkaConfig; import kafka.server.KafkaConfig$; import kafka.server.KafkaRaftServer; import kafka.server.MetaProperties; -import kafka.server.metadata.BrokerServerMetrics$; import kafka.tools.StorageTool; import kafka.utils.Logging; import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.common.Node; -import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.utils.ThreadUtils; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.controller.Controller; -import org.apache.kafka.controller.MockControllerMetrics; -import org.apache.kafka.metadata.MetadataRecordSerde; import org.apache.kafka.metadata.bootstrap.BootstrapMetadata; import org.apache.kafka.raft.RaftConfig; -import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.server.common.MetadataVersion; +import org.apache.kafka.server.fault.FaultHandler; import org.apache.kafka.server.fault.MockFaultHandler; import org.apache.kafka.test.TestUtils; import org.slf4j.Logger; @@ -115,11 +112,33 @@ public void close() { } } + static class SimpleFaultHandlerFactory implements FaultHandlerFactory { + private final MockFaultHandler fatalFaultHandler = new MockFaultHandler("fatalFaultHandler"); + private final MockFaultHandler nonFatalFaultHandler = new MockFaultHandler("nonFatalFaultHandler"); + + MockFaultHandler fatalFaultHandler() { + return fatalFaultHandler; + } + + MockFaultHandler nonFatalFaultHandler() { + return nonFatalFaultHandler; + } + + @Override + public FaultHandler build(String name, boolean fatal, Runnable action) { + if (fatal) { + return fatalFaultHandler; + } else { + return nonFatalFaultHandler; + + } + } + } + public static class Builder { private TestKitNodes nodes; private Map configProps = new HashMap<>(); - private MockFaultHandler metadataFaultHandler = new MockFaultHandler("metadataFaultHandler"); - private MockFaultHandler fatalFaultHandler = new MockFaultHandler("fatalFaultHandler"); + private SimpleFaultHandlerFactory faultHandlerFactory = new SimpleFaultHandlerFactory(); public Builder(TestKitNodes nodes) { this.nodes = nodes; @@ -130,18 +149,56 @@ public Builder setConfigProp(String key, String value) { return this; } - public Builder setMetadataFaultHandler(MockFaultHandler metadataFaultHandler) { - this.metadataFaultHandler = metadataFaultHandler; - return this; + private KafkaConfig createNodeConfig(TestKitNode node) { + BrokerNode brokerNode = nodes.brokerNodes().get(node.id()); + ControllerNode controllerNode = nodes.controllerNodes().get(node.id()); + + Map props = new HashMap<>(configProps); + props.put(KafkaConfig$.MODULE$.ProcessRolesProp(), roles(node.id())); + props.put(KafkaConfig$.MODULE$.NodeIdProp(), + Integer.toString(node.id())); + // In combined mode, always prefer the metadata log directory of the controller node. + if (controllerNode != null) { + props.put(KafkaConfig$.MODULE$.MetadataLogDirProp(), + controllerNode.metadataDirectory()); + } else { + props.put(KafkaConfig$.MODULE$.MetadataLogDirProp(), + node.metadataDirectory()); + } + // Set the log.dirs according to the broker node setting (if there is a broker node) + if (brokerNode != null) { + props.put(KafkaConfig$.MODULE$.LogDirsProp(), + String.join(",", brokerNode.logDataDirectories())); + } + props.put(KafkaConfig$.MODULE$.ListenerSecurityProtocolMapProp(), + "EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT"); + props.put(KafkaConfig$.MODULE$.ListenersProp(), listeners(node.id())); + props.put(KafkaConfig$.MODULE$.InterBrokerListenerNameProp(), + nodes.interBrokerListenerName().value()); + props.put(KafkaConfig$.MODULE$.ControllerListenerNamesProp(), + "CONTROLLER"); + // Note: we can't accurately set controller.quorum.voters yet, since we don't + // yet know what ports each controller will pick. Set it to a dummy string \ + // for now as a placeholder. + String uninitializedQuorumVotersString = nodes.controllerNodes().keySet().stream(). + map(n -> String.format("%d@0.0.0.0:0", n)). + collect(Collectors.joining(",")); + props.put(RaftConfig.QUORUM_VOTERS_CONFIG, uninitializedQuorumVotersString); + + // reduce log cleaner offset map memory usage + props.put(KafkaConfig$.MODULE$.LogCleanerDedupeBufferSizeProp(), "2097152"); + + // Add associated broker node property overrides + if (brokerNode != null) { + props.putAll(brokerNode.propertyOverrides()); + } + return new KafkaConfig(props, false, Option.empty()); } public KafkaClusterTestKit build() throws Exception { Map controllers = new HashMap<>(); Map brokers = new HashMap<>(); - Map> raftManagers = new HashMap<>(); - String uninitializedQuorumVotersString = nodes.controllerNodes().keySet().stream(). - map(controllerNode -> String.format("%d@0.0.0.0:0", controllerNode)). - collect(Collectors.joining(",")); + Map jointServers = new HashMap<>(); /* Number of threads = Total number of brokers + Total number of controllers + Total number of Raft Managers = Total number of brokers + Total number of controllers * 2 @@ -159,53 +216,31 @@ public KafkaClusterTestKit build() throws Exception { executorService = Executors.newFixedThreadPool(numOfExecutorThreads, ThreadUtils.createThreadFactory("KafkaClusterTestKit%d", false)); for (ControllerNode node : nodes.controllerNodes().values()) { - Map props = new HashMap<>(configProps); - props.put(KafkaConfig$.MODULE$.ProcessRolesProp(), roles(node.id())); - props.put(KafkaConfig$.MODULE$.NodeIdProp(), - Integer.toString(node.id())); - props.put(KafkaConfig$.MODULE$.MetadataLogDirProp(), - node.metadataDirectory()); - props.put(KafkaConfig$.MODULE$.ListenerSecurityProtocolMapProp(), - "EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT"); - props.put(KafkaConfig$.MODULE$.ListenersProp(), listeners(node.id())); - props.put(KafkaConfig$.MODULE$.InterBrokerListenerNameProp(), - nodes.interBrokerListenerName().value()); - props.put(KafkaConfig$.MODULE$.ControllerListenerNamesProp(), - "CONTROLLER"); - // Note: we can't accurately set controller.quorum.voters yet, since we don't - // yet know what ports each controller will pick. Set it to a dummy string \ - // for now as a placeholder. - props.put(RaftConfig.QUORUM_VOTERS_CONFIG, uninitializedQuorumVotersString); - - // reduce log cleaner offset map memory usage - props.put(KafkaConfig$.MODULE$.LogCleanerDedupeBufferSizeProp(), "2097152"); - setupNodeDirectories(baseDirectory, node.metadataDirectory(), Collections.emptyList()); - KafkaConfig config = new KafkaConfig(props, false, Option.empty()); - - String threadNamePrefix = String.format("controller%d_", node.id()); MetaProperties metaProperties = MetaProperties.apply(nodes.clusterId().toString(), node.id()); - TopicPartition metadataPartition = new TopicPartition(KafkaRaftServer.MetadataTopic(), 0); BootstrapMetadata bootstrapMetadata = BootstrapMetadata. fromVersion(nodes.bootstrapMetadataVersion(), "testkit"); - KafkaRaftManager raftManager = new KafkaRaftManager<>( - metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftServer.MetadataTopicId(), - Time.SYSTEM, new Metrics(), Option.apply(threadNamePrefix), connectFutureManager.future); - ControllerServer controller = new ControllerServer( - nodes.controllerProperties(node.id()), - config, - raftManager, - Time.SYSTEM, - new Metrics(), - new MockControllerMetrics(), - Option.apply(threadNamePrefix), - connectFutureManager.future, - KafkaRaftServer.configSchema(), - raftManager.apiVersions(), - bootstrapMetadata, - metadataFaultHandler, - fatalFaultHandler - ); + String threadNamePrefix = (nodes.brokerNodes().containsKey(node.id())) ? + String.format("colocated%d", node.id()) : + String.format("controller%d", node.id()); + JointServer jointServer = new JointServer(createNodeConfig(node), + metaProperties, + Time.SYSTEM, + new Metrics(), + Option.apply(threadNamePrefix), + connectFutureManager.future, + faultHandlerFactory); + ControllerServer controller = null; + try { + controller = new ControllerServer( + jointServer, + KafkaRaftServer.configSchema(), + bootstrapMetadata); + } catch (Throwable e) { + log.error("Error creating controller {}", node.id(), e); + if (controller != null) controller.shutdown(); + throw e; + } controllers.put(node.id(), controller); controller.socketServerFirstBoundPortFuture().whenComplete((port, e) -> { if (e != null) { @@ -214,61 +249,31 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS connectFutureManager.registerPort(node.id(), port); } }); - raftManagers.put(node.id(), raftManager); + jointServers.put(node.id(), jointServer); } for (BrokerNode node : nodes.brokerNodes().values()) { - Map props = new HashMap<>(configProps); - props.put(KafkaConfig$.MODULE$.ProcessRolesProp(), roles(node.id())); - props.put(KafkaConfig$.MODULE$.BrokerIdProp(), - Integer.toString(node.id())); - props.put(KafkaConfig$.MODULE$.MetadataLogDirProp(), - node.metadataDirectory()); - props.put(KafkaConfig$.MODULE$.LogDirsProp(), - String.join(",", node.logDataDirectories())); - props.put(KafkaConfig$.MODULE$.ListenerSecurityProtocolMapProp(), - "EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT"); - props.put(KafkaConfig$.MODULE$.ListenersProp(), listeners(node.id())); - props.put(KafkaConfig$.MODULE$.InterBrokerListenerNameProp(), - nodes.interBrokerListenerName().value()); - props.put(KafkaConfig$.MODULE$.ControllerListenerNamesProp(), - "CONTROLLER"); - - setupNodeDirectories(baseDirectory, node.metadataDirectory(), - node.logDataDirectories()); - - // Just like above, we set a placeholder voter list here until we - // find out what ports the controllers picked. - props.put(RaftConfig.QUORUM_VOTERS_CONFIG, uninitializedQuorumVotersString); - props.putAll(node.propertyOverrides()); - KafkaConfig config = new KafkaConfig(props, false, Option.empty()); - - String threadNamePrefix = String.format("broker%d_", node.id()); - MetaProperties metaProperties = MetaProperties.apply(nodes.clusterId().toString(), node.id()); - TopicPartition metadataPartition = new TopicPartition(KafkaRaftServer.MetadataTopic(), 0); - KafkaRaftManager raftManager; - if (raftManagers.containsKey(node.id())) { - raftManager = raftManagers.get(node.id()); - } else { - raftManager = new KafkaRaftManager<>( - metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftServer.MetadataTopicId(), - Time.SYSTEM, new Metrics(), Option.apply(threadNamePrefix), connectFutureManager.future); - raftManagers.put(node.id(), raftManager); + JointServer jointServer = jointServers.get(node.id()); + if (jointServer == null) { + MetaProperties metaProperties = MetaProperties.apply(nodes.clusterId().toString(), node.id()); + jointServer = new JointServer(createNodeConfig(node), + metaProperties, + Time.SYSTEM, + new Metrics(), + Option.apply(String.format("broker%d_", node.id())), + connectFutureManager.future, + faultHandlerFactory); + jointServers.put(node.id(), jointServer); + } + BrokerServer broker = null; + try { + broker = new BrokerServer( + jointServer, + JavaConverters.asScalaBuffer(Collections.emptyList()).toSeq()); + } catch (Throwable e) { + log.error("Error creating broker {}", node.id(), e); + if (broker != null) broker.shutdown(); + throw e; } - Metrics metrics = new Metrics(); - BrokerServer broker = new BrokerServer( - config, - nodes.brokerProperties(node.id()), - raftManager, - Time.SYSTEM, - metrics, - BrokerServerMetrics$.MODULE$.apply(metrics), - Option.apply(threadNamePrefix), - JavaConverters.asScalaBuffer(Collections.emptyList()).toSeq(), - connectFutureManager.future, - fatalFaultHandler, - metadataFaultHandler, - metadataFaultHandler - ); brokers.put(node.id(), broker); } } catch (Exception e) { @@ -279,9 +284,6 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS for (BrokerServer brokerServer : brokers.values()) { brokerServer.shutdown(); } - for (KafkaRaftManager raftManager : raftManagers.values()) { - raftManager.shutdown(); - } for (ControllerServer controller : controllers.values()) { controller.shutdown(); } @@ -291,9 +293,13 @@ metaProperties, config, new MetadataRecordSerde(), metadataPartition, KafkaRaftS } throw e; } - return new KafkaClusterTestKit(executorService, nodes, controllers, - brokers, raftManagers, connectFutureManager, baseDirectory, - metadataFaultHandler, fatalFaultHandler); + return new KafkaClusterTestKit(executorService, + nodes, + controllers, + brokers, + connectFutureManager, + baseDirectory, + faultHandlerFactory); } private String listeners(int node) { @@ -331,32 +337,26 @@ static private void setupNodeDirectories(File baseDirectory, private final TestKitNodes nodes; private final Map controllers; private final Map brokers; - private final Map> raftManagers; private final ControllerQuorumVotersFutureManager controllerQuorumVotersFutureManager; private final File baseDirectory; - private final MockFaultHandler metadataFaultHandler; - private final MockFaultHandler fatalFaultHandler; + private final SimpleFaultHandlerFactory faultHandlerFactory; private KafkaClusterTestKit( ExecutorService executorService, TestKitNodes nodes, Map controllers, Map brokers, - Map> raftManagers, ControllerQuorumVotersFutureManager controllerQuorumVotersFutureManager, File baseDirectory, - MockFaultHandler metadataFaultHandler, - MockFaultHandler fatalFaultHandler + SimpleFaultHandlerFactory faultHandlerFactory ) { this.executorService = executorService; this.nodes = nodes; this.controllers = controllers; this.brokers = brokers; - this.raftManagers = raftManagers; this.controllerQuorumVotersFutureManager = controllerQuorumVotersFutureManager; this.baseDirectory = baseDirectory; - this.metadataFaultHandler = metadataFaultHandler; - this.fatalFaultHandler = fatalFaultHandler; + this.faultHandlerFactory = faultHandlerFactory; } public void format() throws Exception { @@ -369,10 +369,12 @@ public void format() throws Exception { controller, futures::add); } for (Entry entry : brokers.entrySet()) { - int nodeId = entry.getKey(); - BrokerServer broker = entry.getValue(); - formatNodeAndLog(nodes.brokerProperties(nodeId), broker.config().metadataLogDir(), - broker, futures::add); + if (!controllers.containsKey(entry.getKey())) { + int nodeId = entry.getKey(); + BrokerServer broker = entry.getValue(); + formatNodeAndLog(nodes.brokerProperties(nodeId), broker.config().metadataLogDir(), + broker, futures::add); + } } for (Future future: futures) { future.get(); @@ -411,10 +413,6 @@ public void startup() throws ExecutionException, InterruptedException { try { // Note the startup order here is chosen to be consistent with // `KafkaRaftServer`. See comments in that class for an explanation. - - for (KafkaRaftManager raftManager : raftManagers.values()) { - futures.add(controllerQuorumVotersFutureManager.future.thenRunAsync(raftManager::startup)); - } for (ControllerServer controller : controllers.values()) { futures.add(executorService.submit(controller::startup)); } @@ -504,14 +502,18 @@ public Map brokers() { return brokers; } - public Map> raftManagers() { - return raftManagers; - } - public TestKitNodes nodes() { return nodes; } + public MockFaultHandler fatalFaultHandler() { + return faultHandlerFactory.fatalFaultHandler(); + } + + public MockFaultHandler nonFatalFaultHandler() { + return faultHandlerFactory.nonFatalFaultHandler(); + } + @Override public void close() throws Exception { List>> futureEntries = new ArrayList<>(); @@ -529,14 +531,6 @@ public void close() throws Exception { } waitForAllFutures(futureEntries); futureEntries.clear(); - for (Entry> entry : raftManagers.entrySet()) { - int raftManagerId = entry.getKey(); - KafkaRaftManager raftManager = entry.getValue(); - futureEntries.add(new SimpleImmutableEntry<>("raftManager" + raftManagerId, - executorService.submit(raftManager::shutdown))); - } - waitForAllFutures(futureEntries); - futureEntries.clear(); for (Entry entry : controllers.entrySet()) { int controllerId = entry.getKey(); ControllerServer controller = entry.getValue(); @@ -555,8 +549,8 @@ public void close() throws Exception { executorService.shutdownNow(); executorService.awaitTermination(5, TimeUnit.MINUTES); } - metadataFaultHandler.maybeRethrowFirstException(); - fatalFaultHandler.maybeRethrowFirstException(); + faultHandlerFactory.fatalFaultHandler().maybeRethrowFirstException(); + faultHandlerFactory.nonFatalFaultHandler().maybeRethrowFirstException(); } private void waitForAllFutures(List>> futureEntries) diff --git a/core/src/test/scala/integration/kafka/api/BaseAdminIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/BaseAdminIntegrationTest.scala index 293b198c0ecdb..6fa8bc9b2bc55 100644 --- a/core/src/test/scala/integration/kafka/api/BaseAdminIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/BaseAdminIntegrationTest.scala @@ -215,11 +215,6 @@ abstract class BaseAdminIntegrationTest extends IntegrationTestHarness with Logg override def kraftControllerConfigs(): Seq[Properties] = { val controllerConfig = new Properties() - if (testInfo.getTestMethod.toString.contains("testCreateTopicsReturnsConfigs")) { - // For testCreateTopicsReturnsConfigs, set the controller's ID to 1 so that the dynamic - // config we set for node 1 will apply to it. - controllerConfig.setProperty(KafkaConfig.NodeIdProp, "1") - } val controllerConfigs = Seq(controllerConfig) modifyConfigs(controllerConfigs) controllerConfigs diff --git a/core/src/test/scala/integration/kafka/api/PlaintextAdminIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextAdminIntegrationTest.scala index 1656af08bc15e..c3854ffa29368 100644 --- a/core/src/test/scala/integration/kafka/api/PlaintextAdminIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/PlaintextAdminIntegrationTest.scala @@ -44,12 +44,14 @@ import org.apache.kafka.common.requests.{DeleteRecordsRequest, MetadataResponse} import org.apache.kafka.common.resource.{PatternType, ResourcePattern, ResourceType} import org.apache.kafka.common.utils.{Time, Utils} import org.apache.kafka.common.{ConsumerGroupState, ElectionType, TopicCollection, TopicPartition, TopicPartitionInfo, TopicPartitionReplica, Uuid} +import org.apache.kafka.controller.ControllerRequestContextUtil.ANONYMOUS_CONTEXT import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Disabled, TestInfo} import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.ValueSource import org.slf4j.LoggerFactory +import java.util.AbstractMap.SimpleImmutableEntry import scala.annotation.nowarn import scala.collection.Seq import scala.compat.java8.OptionConverters._ @@ -2500,22 +2502,32 @@ class PlaintextAdminIntegrationTest extends BaseAdminIntegrationTest { * Test that createTopics returns the dynamic configurations of the topics that were created. * * Note: this test requires some custom static broker and controller configurations, which are set up in - * BaseAdminIntegrationTest.modifyConfigs and BaseAdminIntegrationTest.kraftControllerConfigs. + * BaseAdminIntegrationTest.modifyConfigs. */ @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumName) @ValueSource(strings = Array("zk", "kraft")) def testCreateTopicsReturnsConfigs(quorum: String): Unit = { client = Admin.create(super.createConfig) - val newLogRetentionProperties = new Properties newLogRetentionProperties.put(KafkaConfig.LogRetentionTimeMillisProp, "10800000") TestUtils.incrementalAlterConfigs(null, client, newLogRetentionProperties, perBrokerConfig = false) .all().get(15, TimeUnit.SECONDS) - val newLogCleanerDeleteRetention = new Properties - newLogCleanerDeleteRetention.put(KafkaConfig.LogCleanerDeleteRetentionMsProp, "34") - TestUtils.incrementalAlterConfigs(brokers, client, newLogCleanerDeleteRetention, perBrokerConfig = true) - .all().get(15, TimeUnit.SECONDS) + if (isKRaftTest()) { + // In KRaft mode, we don't yet support altering configs on controller nodes, except by setting + // default node configs. Therefore, we have to set the dynamic config directly to test this. + val controllerNodeResource = new ConfigResource(ConfigResource.Type.BROKER, + controllerServer.config.nodeId.toString) + controllerServer.controller.incrementalAlterConfigs(ANONYMOUS_CONTEXT, + Collections.singletonMap(controllerNodeResource, + Collections.singletonMap(KafkaConfig.LogCleanerDeleteRetentionMsProp, + new SimpleImmutableEntry(AlterConfigOp.OpType.SET, "34"))), false).get() + } else { + val newLogCleanerDeleteRetention = new Properties + newLogCleanerDeleteRetention.put(KafkaConfig.LogCleanerDeleteRetentionMsProp, "34") + TestUtils.incrementalAlterConfigs(brokers, client, newLogCleanerDeleteRetention, perBrokerConfig = true) + .all().get(15, TimeUnit.SECONDS) + } if (isKRaftTest()) { ensureConsistentKRaftMetadata() diff --git a/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala b/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala index f52c2a72da25c..bacdd6e666bd4 100644 --- a/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala +++ b/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala @@ -78,7 +78,7 @@ class KRaftClusterTest { cluster.startup() TestUtils.waitUntilTrue(() => cluster.brokers().get(0).brokerState == BrokerState.RUNNING, "Broker never made it to RUNNING state.") - TestUtils.waitUntilTrue(() => cluster.raftManagers().get(0).client.leaderAndEpoch().leaderId.isPresent, + TestUtils.waitUntilTrue(() => cluster.brokers().get(0).jointServer.raftManager.client.leaderAndEpoch().leaderId.isPresent, "RaftManager was not initialized.") val admin = Admin.create(cluster.clientProperties()) try { @@ -104,7 +104,7 @@ class KRaftClusterTest { cluster.waitForReadyBrokers() TestUtils.waitUntilTrue(() => cluster.brokers().get(0).brokerState == BrokerState.RUNNING, "Broker never made it to RUNNING state.") - TestUtils.waitUntilTrue(() => cluster.raftManagers().get(0).client.leaderAndEpoch().leaderId.isPresent, + TestUtils.waitUntilTrue(() => cluster.brokers().get(0).jointServer.raftManager.client.leaderAndEpoch().leaderId.isPresent, "RaftManager was not initialized.") val admin = Admin.create(cluster.clientProperties()) @@ -141,7 +141,7 @@ class KRaftClusterTest { cluster.waitForReadyBrokers() TestUtils.waitUntilTrue(() => cluster.brokers().get(0).brokerState == BrokerState.RUNNING, "Broker never made it to RUNNING state.") - TestUtils.waitUntilTrue(() => cluster.raftManagers().get(0).client.leaderAndEpoch().leaderId.isPresent, + TestUtils.waitUntilTrue(() => cluster.brokers().get(0).jointServer.raftManager.client.leaderAndEpoch().leaderId.isPresent, "RaftManager was not initialized.") val admin = Admin.create(cluster.clientProperties()) try { @@ -559,10 +559,10 @@ class KRaftClusterTest { assertEquals(Seq(ApiError.NONE), incrementalAlter(admin, Seq( (new ConfigResource(Type.BROKER, ""), Seq( new AlterConfigOp(new ConfigEntry("log.roll.ms", "1234567"), OpType.SET), - new AlterConfigOp(new ConfigEntry("max.connections.per.ip", "6"), OpType.SET)))))) + new AlterConfigOp(new ConfigEntry("max.connections.per.ip", "600"), OpType.SET)))))) validateConfigs(admin, Map(new ConfigResource(Type.BROKER, "") -> Seq( ("log.roll.ms", "1234567"), - ("max.connections.per.ip", "6"))), true) + ("max.connections.per.ip", "600"))), true) admin.createTopics(Arrays.asList( new NewTopic("foo", 2, 3.toShort), @@ -588,10 +588,10 @@ class KRaftClusterTest { assertEquals(Seq(ApiError.NONE), incrementalAlter(admin, Seq( (new ConfigResource(Type.BROKER, "2"), Seq( - new AlterConfigOp(new ConfigEntry("max.connections.per.ip", "7"), OpType.SET)))))) + new AlterConfigOp(new ConfigEntry("max.connections.per.ip", "700"), OpType.SET)))))) validateConfigs(admin, Map(new ConfigResource(Type.BROKER, "2") -> Seq( - ("max.connections.per.ip", "7")))) + ("max.connections.per.ip", "700")))) } finally { admin.close() } diff --git a/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala b/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala index c0a55948cfd2c..632ad36054a4f 100755 --- a/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala +++ b/core/src/test/scala/integration/kafka/server/QuorumTestHarness.scala @@ -23,23 +23,18 @@ import java.util import java.util.{Collections, Properties} import java.util.concurrent.CompletableFuture import javax.security.auth.login.Configuration -import kafka.raft.KafkaRaftManager -import kafka.server.metadata.BrokerServerMetrics import kafka.tools.StorageTool import kafka.utils.{CoreUtils, Logging, TestInfoUtils, TestUtils} import kafka.zk.{AdminZkClient, EmbeddedZookeeper, KafkaZkClient} import org.apache.kafka.common.metrics.Metrics -import org.apache.kafka.common.{TopicPartition, Uuid} +import org.apache.kafka.common.Uuid import org.apache.kafka.common.security.JaasUtils import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.utils.{Exit, Time} -import org.apache.kafka.controller.QuorumControllerMetrics -import org.apache.kafka.metadata.MetadataRecordSerde import org.apache.kafka.metadata.bootstrap.BootstrapMetadata import org.apache.kafka.raft.RaftConfig.{AddressSpec, InetAddressSpec} import org.apache.kafka.server.common.{ApiMessageAndVersion, MetadataVersion} import org.apache.kafka.server.fault.{FaultHandler, MockFaultHandler} -import org.apache.kafka.server.metrics.KafkaYammerMetrics import org.apache.zookeeper.client.ZKClientConfig import org.apache.zookeeper.{WatchedEvent, Watcher, ZooKeeper} import org.junit.jupiter.api.Assertions._ @@ -85,8 +80,8 @@ class ZooKeeperQuorumImplementation( } class KRaftQuorumImplementation( - val raftManager: KafkaRaftManager[ApiMessageAndVersion], val controllerServer: ControllerServer, + val faultHandlerFactory: FaultHandlerFactory, val metadataDir: File, val controllerQuorumVotersFuture: CompletableFuture[util.Map[Integer, AddressSpec]], val clusterId: String, @@ -99,25 +94,28 @@ class KRaftQuorumImplementation( startup: Boolean, threadNamePrefix: Option[String], ): KafkaBroker = { - val metrics = new Metrics() - val broker = new BrokerServer(config = config, - metaProps = new MetaProperties(clusterId, config.nodeId), - raftManager = raftManager, - time = time, - metrics = metrics, - brokerMetrics = BrokerServerMetrics(metrics), - threadNamePrefix = Some("Broker%02d_".format(config.nodeId)), - initialOfflineDirs = Seq(), - controllerQuorumVotersFuture = controllerQuorumVotersFuture, - fatalFaultHandler = faultHandler, - metadataLoadingFaultHandler = faultHandler, - metadataPublishingFaultHandler = faultHandler) - if (startup) broker.startup() - broker + val jointServer = new JointServer(config, + new MetaProperties(clusterId, config.nodeId), + Time.SYSTEM, + new Metrics(), + Option("Broker%02d_".format(config.nodeId)), + controllerQuorumVotersFuture, + faultHandlerFactory) + var broker: BrokerServer = null + try { + broker = new BrokerServer(jointServer, + initialOfflineDirs = Seq()) + if (startup) broker.startup() + broker + } catch { + case e: Throwable => { + if (broker != null) CoreUtils.swallow(broker.shutdown(), log) + throw e + } + } } override def shutdown(): Unit = { - CoreUtils.swallow(raftManager.shutdown(), log) CoreUtils.swallow(controllerServer.shutdown(), log) } } @@ -201,6 +199,10 @@ abstract class QuorumTestHarness extends Logging { val faultHandler = new MockFaultHandler("quorumTestHarnessFaultHandler") + val faultHandlerFactory = new FaultHandlerFactory { + override def build(name: String, fatal: Boolean, action: Runnable): FaultHandler = faultHandler + } + // Note: according to the junit documentation: "JUnit Jupiter does not guarantee the execution // order of multiple @BeforeEach methods that are declared within a single test class or test // interface." Therefore, if you have things you would like to do before each test case runs, it @@ -288,7 +290,6 @@ abstract class QuorumTestHarness extends Logging { val metadataDir = TestUtils.tempDir() val metaProperties = new MetaProperties(Uuid.randomUuid().toString, nodeId) formatDirectories(immutable.Seq(metadataDir.getAbsolutePath), metaProperties) - val controllerMetrics = new Metrics() props.setProperty(KafkaConfig.MetadataLogDirProp, metadataDir.getAbsolutePath) val proto = controllerListenerSecurityProtocol.toString props.setProperty(KafkaConfig.ListenerSecurityProtocolMapProp, s"CONTROLLER:${proto}") @@ -296,34 +297,20 @@ abstract class QuorumTestHarness extends Logging { props.setProperty(KafkaConfig.ControllerListenerNamesProp, "CONTROLLER") props.setProperty(KafkaConfig.QuorumVotersProp, s"${nodeId}@localhost:0") val config = new KafkaConfig(props) - val threadNamePrefix = "Controller_" + testInfo.getDisplayName val controllerQuorumVotersFuture = new CompletableFuture[util.Map[Integer, AddressSpec]] - val raftManager = new KafkaRaftManager( - metaProperties = metaProperties, - config = config, - recordSerde = MetadataRecordSerde.INSTANCE, - topicPartition = new TopicPartition(KafkaRaftServer.MetadataTopic, 0), - topicId = KafkaRaftServer.MetadataTopicId, - time = Time.SYSTEM, - metrics = controllerMetrics, - threadNamePrefixOpt = Option(threadNamePrefix), - controllerQuorumVotersFuture = controllerQuorumVotersFuture) + val jointServer = new JointServer(config, + metaProperties, + Time.SYSTEM, + new Metrics(), + Option("Controller_" + testInfo.getDisplayName), + controllerQuorumVotersFuture, + faultHandlerFactory) var controllerServer: ControllerServer = null try { controllerServer = new ControllerServer( - metaProperties = metaProperties, - config = config, - raftManager = raftManager, - time = Time.SYSTEM, - metrics = controllerMetrics, - controllerMetrics = new QuorumControllerMetrics(KafkaYammerMetrics.defaultRegistry(), Time.SYSTEM), - threadNamePrefix = Option(threadNamePrefix), - controllerQuorumVotersFuture = controllerQuorumVotersFuture, - configSchema = KafkaRaftServer.configSchema, - raftApiVersions = raftManager.apiVersions, - bootstrapMetadata = BootstrapMetadata.fromVersion(metadataVersion, "test harness"), - metadataFaultHandler = faultHandler, - fatalFaultHandler = faultHandler + jointServer, + KafkaRaftServer.configSchema, + BootstrapMetadata.fromVersion(metadataVersion, "test harness") ) controllerServer.socketServerFirstBoundPortFuture.whenComplete((port, e) => { if (e != null) { @@ -335,15 +322,13 @@ abstract class QuorumTestHarness extends Logging { } }) controllerServer.startup() - raftManager.startup() } catch { case e: Throwable => - CoreUtils.swallow(raftManager.shutdown(), this) if (controllerServer != null) CoreUtils.swallow(controllerServer.shutdown(), this) throw e } - new KRaftQuorumImplementation(raftManager, - controllerServer, + new KRaftQuorumImplementation(controllerServer, + faultHandlerFactory, metadataDir, controllerQuorumVotersFuture, metaProperties.clusterId, diff --git a/core/src/test/scala/integration/kafka/server/RaftClusterSnapshotTest.scala b/core/src/test/scala/integration/kafka/server/RaftClusterSnapshotTest.scala index f1c47ff1bea97..4dddbf5823fb3 100644 --- a/core/src/test/scala/integration/kafka/server/RaftClusterSnapshotTest.scala +++ b/core/src/test/scala/integration/kafka/server/RaftClusterSnapshotTest.scala @@ -17,6 +17,8 @@ package kafka.server +import kafka.raft.KafkaRaftManager + import java.util.Collections import kafka.testkit.KafkaClusterTestKit import kafka.testkit.TestKitNodes @@ -24,16 +26,21 @@ import kafka.utils.TestUtils import kafka.server.KafkaConfig.{MetadataMaxIdleIntervalMsProp, MetadataSnapshotMaxNewRecordBytesProp} import org.apache.kafka.common.utils.BufferSupplier import org.apache.kafka.metadata.MetadataRecordSerde +import org.apache.kafka.server.common.ApiMessageAndVersion import org.apache.kafka.snapshot.RecordsSnapshotReader -import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.junit.jupiter.api.Timeout + import scala.jdk.CollectionConverters._ @Timeout(120) class RaftClusterSnapshotTest { + private def allRaftManagers(cluster: KafkaClusterTestKit): Iterable[KafkaRaftManager[ApiMessageAndVersion]] = { + (cluster.brokers().values().asScala.map(_.jointServer.raftManager) ++ + cluster.controllers().values().asScala.map(_.jointServer.raftManager)) + } @Test def testSnapshotsGenerated(): Unit = { @@ -58,21 +65,17 @@ class RaftClusterSnapshotTest { // Check that every controller and broker has a snapshot TestUtils.waitUntilTrue( () => { - cluster.raftManagers().asScala.forall { case (_, raftManager) => - raftManager.replicatedLog.latestSnapshotId.isPresent + allRaftManagers(cluster).forall { + case raftManager => raftManager.replicatedLog.latestSnapshotId().isPresent } }, s"Expected for every controller and broker to generate a snapshot: ${ - cluster.raftManagers().asScala.map { case (id, raftManager) => - (id, raftManager.replicatedLog.latestSnapshotId) - } + allRaftManagers(cluster).map(_.replicatedLog.latestSnapshotId()) }" ) - assertEquals(numberOfControllers + numberOfBrokers, cluster.raftManagers.size()) - // For every controller and broker perform some sanity checks against the lastest snapshot - for ((_, raftManager) <- cluster.raftManagers().asScala) { + allRaftManagers(cluster).foreach(raftManager => { TestUtils.resource( RecordsSnapshotReader.of( raftManager.replicatedLog.latestSnapshot.get(), @@ -92,7 +95,7 @@ class RaftClusterSnapshotTest { assertNotEquals(Collections.emptyList(), batch.records()) } } - } + }) } } } diff --git a/core/src/test/scala/kafka/server/metadata/BrokerServerMetricsTest.scala b/core/src/test/scala/kafka/server/metadata/BrokerServerMetricsTest.scala index ea2b439c166ae..6dd9f05fce022 100644 --- a/core/src/test/scala/kafka/server/metadata/BrokerServerMetricsTest.scala +++ b/core/src/test/scala/kafka/server/metadata/BrokerServerMetricsTest.scala @@ -22,12 +22,38 @@ import kafka.utils.TestUtils import org.apache.kafka.common.MetricName import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.utils.MockTime -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertTrue +import org.apache.kafka.image.MetadataProvenance +import org.apache.kafka.server.metrics.KafkaYammerMetrics +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} import org.junit.jupiter.api.Test + +import scala.collection.mutable import scala.jdk.CollectionConverters._ final class BrokerServerMetricsTest { + @Test + def testYammerMetrics(): Unit = { + val expectedYammerMetrics = Set( + "kafka.server:type=BrokerMetadataListener,name=MetadataBatchProcessingTimeUs" + ) + TestUtils.resource(BrokerServerMetrics(new Metrics())) { brokerMetrics => + val notFound = new mutable.HashSet[String] ++ expectedYammerMetrics + KafkaYammerMetrics.defaultRegistry.allMetrics.keySet().forEach(bean => { + expectedYammerMetrics.foreach(expected => { + if (bean.getMBeanName().equals(expected)) { + notFound -= expected + } + }) + }) + assertEquals(Set.empty, notFound, "Failed to find some Yammer metrics") + } + KafkaYammerMetrics.defaultRegistry.allMetrics.keySet().forEach(bean => { + expectedYammerMetrics.foreach(expected => { + assertFalse(bean.getMBeanName().equals(expected), s"${expected} did not get cleaned up.") + }) + }) + } + @Test def testMetricsExported(): Unit = { val metrics = new Metrics() @@ -59,11 +85,11 @@ final class BrokerServerMetricsTest { val metrics = new Metrics() TestUtils.resource(BrokerServerMetrics(metrics)) { brokerMetrics => val offsetMetric = metrics.metrics().get(brokerMetrics.lastAppliedRecordOffsetName) - assertEquals(0, offsetMetric.metricValue.asInstanceOf[Long]) + assertEquals(-1L, offsetMetric.metricValue.asInstanceOf[Long]) // Update metric value and check val expectedValue = 1000 - brokerMetrics.lastAppliedRecordOffset.set(expectedValue) + brokerMetrics.lastAppliedImageProvenance.set(new MetadataProvenance(expectedValue, 0, 0L)) assertEquals(expectedValue, offsetMetric.metricValue.asInstanceOf[Long]) } } @@ -82,7 +108,7 @@ final class BrokerServerMetricsTest { // Update metric value and check val timestamp = 500 - brokerMetrics.lastAppliedRecordTimestamp.set(timestamp) + brokerMetrics.lastAppliedImageProvenance.set(new MetadataProvenance(0L, 0, timestamp)) assertEquals(timestamp, timestampMetric.metricValue.asInstanceOf[Long]) assertEquals(time.milliseconds - timestamp, lagMetric.metricValue.asInstanceOf[Long]) } diff --git a/core/src/test/scala/unit/kafka/server/DynamicConfigChangeTest.scala b/core/src/test/scala/unit/kafka/server/DynamicConfigChangeTest.scala index 84d6f5a2ef93d..9e7e0d2f17d0f 100644 --- a/core/src/test/scala/unit/kafka/server/DynamicConfigChangeTest.scala +++ b/core/src/test/scala/unit/kafka/server/DynamicConfigChangeTest.scala @@ -43,7 +43,7 @@ import org.apache.kafka.common.record.{CompressionType, RecordVersion} import org.apache.kafka.common.security.auth.KafkaPrincipal import org.apache.kafka.server.common.MetadataVersion.IBP_3_0_IV1 import org.junit.jupiter.api.Assertions._ -import org.junit.jupiter.api.Test +import org.junit.jupiter.api.{Test, Timeout} import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.ValueSource import org.mockito.ArgumentMatchers.{any, anyString} @@ -53,6 +53,7 @@ import scala.annotation.nowarn import scala.collection.{Map, Seq} import scala.jdk.CollectionConverters._ +@Timeout(60) class DynamicConfigChangeTest extends KafkaServerTestHarness { def generateConfigs = List(KafkaConfig.fromProps(TestUtils.createBrokerConfig(0, zkConnectOrNull))) diff --git a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala index d92c76f71189d..838a3c0c4e550 100644 --- a/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala +++ b/core/src/test/scala/unit/kafka/server/MetadataCacheTest.scala @@ -21,7 +21,6 @@ import org.apache.kafka.common.{Node, TopicPartition, Uuid} import java.util import java.util.Arrays.asList import java.util.Collections - import kafka.api.LeaderAndIsr import kafka.server.metadata.{KRaftMetadataCache, ZkMetadataCache} import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadataBroker, UpdateMetadataEndpoint, UpdateMetadataPartitionState, UpdateMetadataTopicState} @@ -32,10 +31,8 @@ import org.apache.kafka.common.requests.UpdateMetadataRequest import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.metadata.{BrokerRegistrationChangeRecord, PartitionRecord, RegisterBrokerRecord, RemoveTopicRecord, TopicRecord} import org.apache.kafka.common.metadata.RegisterBrokerRecord.{BrokerEndpoint, BrokerEndpointCollection} -import org.apache.kafka.image.{ClusterImage, MetadataDelta, MetadataImage} +import org.apache.kafka.image.{ClusterImage, MetadataDelta, MetadataImage, MetadataProvenance} import org.apache.kafka.server.common.MetadataVersion -import org.apache.kafka.raft.{OffsetAndEpoch => RaftOffsetAndEpoch} - import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.MethodSource @@ -65,7 +62,7 @@ object MetadataCacheTest { // contains no brokers, but which contains the previous partitions. val image = c.currentImage() val partialImage = new MetadataImage( - new RaftOffsetAndEpoch(100, 10), + new MetadataProvenance(100L, 10, 1000L), image.features(), ClusterImage.EMPTY, image.topics(), @@ -73,7 +70,7 @@ object MetadataCacheTest { image.clientQuotas(), image.producerIds(), image.acls()) - val delta = new MetadataDelta(partialImage) + val delta = new MetadataDelta.Builder().setImage(partialImage).build() def toRecord(broker: UpdateMetadataBroker): RegisterBrokerRecord = { val endpoints = new BrokerEndpointCollection() @@ -100,7 +97,7 @@ object MetadataCacheTest { setFenced(fenced) } request.liveBrokers().iterator().asScala.foreach { brokerInfo => - delta.replay(100, 10, toRecord(brokerInfo)) + delta.replay(toRecord(brokerInfo)) } def toRecords(topic: UpdateMetadataTopicState): Seq[ApiMessage] = { @@ -125,9 +122,9 @@ object MetadataCacheTest { results } request.topicStates().forEach { topic => - toRecords(topic).foreach(delta.replay(100, 10, _)) + toRecords(topic).foreach(delta.replay) } - c.setImage(delta.apply()) + c.setImage(delta.apply(new MetadataProvenance(100L, 10, 1000L))) } case _ => throw new RuntimeException("Unsupported cache type") } @@ -646,12 +643,12 @@ class MetadataCacheTest { def testIsBrokerFenced(): Unit = { val metadataCache = MetadataCache.kRaftMetadataCache(0) - val delta = new MetadataDelta(MetadataImage.EMPTY) + val delta = new MetadataDelta.Builder().build() delta.replay(new RegisterBrokerRecord() .setBrokerId(0) .setFenced(false)) - metadataCache.setImage(delta.apply()) + metadataCache.setImage(delta.apply(MetadataProvenance.EMPTY)) assertFalse(metadataCache.isBrokerFenced(0)) @@ -659,7 +656,7 @@ class MetadataCacheTest { .setBrokerId(0) .setFenced(1.toByte)) - metadataCache.setImage(delta.apply()) + metadataCache.setImage(delta.apply(MetadataProvenance.EMPTY)) assertTrue(metadataCache.isBrokerFenced(0)) } @@ -668,12 +665,12 @@ class MetadataCacheTest { def testIsBrokerInControlledShutdown(): Unit = { val metadataCache = MetadataCache.kRaftMetadataCache(0) - val delta = new MetadataDelta(MetadataImage.EMPTY) + val delta = new MetadataDelta.Builder().build() delta.replay(new RegisterBrokerRecord() .setBrokerId(0) .setInControlledShutdown(false)) - metadataCache.setImage(delta.apply()) + metadataCache.setImage(delta.apply(MetadataProvenance.EMPTY)) assertFalse(metadataCache.isBrokerShuttingDown(0)) @@ -681,7 +678,7 @@ class MetadataCacheTest { .setBrokerId(0) .setInControlledShutdown(1.toByte)) - metadataCache.setImage(delta.apply()) + metadataCache.setImage(delta.apply(MetadataProvenance.EMPTY)) assertTrue(metadataCache.isBrokerShuttingDown(0)) } diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerConcurrencyTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerConcurrencyTest.scala index 651451afad430..559485ff4ddd6 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerConcurrencyTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerConcurrencyTest.scala @@ -349,7 +349,7 @@ class ReplicaManagerConcurrencyTest { override def doWork(): Unit = { channel.poll() match { case InitializeEvent => - val delta = new MetadataDelta(latestImage) + val delta = new MetadataDelta.Builder().setImage(latestImage).build() brokerIds.foreach { brokerId => delta.replay(new RegisterBrokerRecord() .setBrokerId(brokerId) @@ -357,14 +357,14 @@ class ReplicaManagerConcurrencyTest { ) } topic.initialize(delta) - latestImage = delta.apply() + latestImage = delta.apply(latestImage.provenance()) metadataCache.setImage(latestImage) replicaManager.applyDelta(delta.topicsDelta, latestImage) case AlterIsrEvent(future, topicPartition, leaderAndIsr) => - val delta = new MetadataDelta(latestImage) + val delta = new MetadataDelta.Builder().setImage(latestImage).build() val updatedLeaderAndIsr = topic.alterIsr(topicPartition, leaderAndIsr, delta) - latestImage = delta.apply() + latestImage = delta.apply(latestImage.provenance()) future.complete(updatedLeaderAndIsr) replicaManager.applyDelta(delta.topicsDelta, latestImage) diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index 19199ff71b925..f5437df2b51c8 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -53,10 +53,9 @@ import org.apache.kafka.common.requests._ import org.apache.kafka.common.security.auth.KafkaPrincipal import org.apache.kafka.common.utils.{LogContext, Time, Utils} import org.apache.kafka.common.{IsolationLevel, Node, TopicIdPartition, TopicPartition, Uuid} -import org.apache.kafka.image.{AclsImage, ClientQuotasImage, ClusterImageTest, ConfigurationsImage, FeaturesImage, MetadataImage, ProducerIdsImage, TopicsDelta, TopicsImage} +import org.apache.kafka.image.{AclsImage, ClientQuotasImage, ClusterImageTest, ConfigurationsImage, FeaturesImage, MetadataImage, MetadataProvenance, ProducerIdsImage, TopicsDelta, TopicsImage} import org.apache.kafka.metadata.LeaderConstants.NO_LEADER import org.apache.kafka.metadata.LeaderRecoveryState -import org.apache.kafka.raft.{OffsetAndEpoch => RaftOffsetAndEpoch} import org.apache.kafka.server.common.MetadataVersion.IBP_2_6_IV0 import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} @@ -4041,7 +4040,7 @@ class ReplicaManagerTest { private def imageFromTopics(topicsImage: TopicsImage): MetadataImage = { new MetadataImage( - new RaftOffsetAndEpoch(100, 10), + new MetadataProvenance(100L, 10, 1000L), FeaturesImage.EMPTY, ClusterImageTest.IMAGE1, topicsImage, diff --git a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataListenerTest.scala b/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataListenerTest.scala deleted file mode 100644 index a589c0572ddd8..0000000000000 --- a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataListenerTest.scala +++ /dev/null @@ -1,413 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.server.metadata - -import java.util -import java.util.concurrent.atomic.AtomicReference -import java.util.{Collections, Optional} -import org.apache.kafka.common.metadata.{FeatureLevelRecord, PartitionChangeRecord, PartitionRecord, RegisterBrokerRecord, TopicRecord} -import org.apache.kafka.common.metrics.Metrics -import org.apache.kafka.common.utils.Time -import org.apache.kafka.common.{Endpoint, Uuid} -import org.apache.kafka.image.{MetadataDelta, MetadataImage} -import org.apache.kafka.metadata.util.SnapshotReason -import org.apache.kafka.metadata.{BrokerRegistration, RecordTestUtils, VersionRange} -import org.apache.kafka.server.common.{ApiMessageAndVersion, MetadataVersion} -import org.apache.kafka.server.fault.{FaultHandler, MockFaultHandler} -import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue} -import org.junit.jupiter.api.{AfterEach, Test} - -import scala.jdk.CollectionConverters._ - -class BrokerMetadataListenerTest { - private val metadataLoadingFaultHandler = new MockFaultHandler("metadata loading") - - @AfterEach - def verifyNoFaults(): Unit = { - metadataLoadingFaultHandler.maybeRethrowFirstException() - } - - private def newBrokerMetadataListener( - metrics: BrokerServerMetrics = BrokerServerMetrics(new Metrics()), - snapshotter: Option[MetadataSnapshotter] = None, - maxBytesBetweenSnapshots: Long = 1000000L, - faultHandler: FaultHandler = metadataLoadingFaultHandler - ): BrokerMetadataListener = { - new BrokerMetadataListener( - brokerId = 0, - time = Time.SYSTEM, - threadNamePrefix = None, - maxBytesBetweenSnapshots = maxBytesBetweenSnapshots, - snapshotter = snapshotter, - brokerMetrics = metrics, - _metadataLoadingFaultHandler = faultHandler - ) - } - - @Test - def testCreateAndClose(): Unit = { - val listener = newBrokerMetadataListener() - listener.close() - } - - @Test - def testPublish(): Unit = { - val metrics = BrokerServerMetrics(new Metrics()) - val listener = newBrokerMetadataListener(metrics = metrics) - try { - val unfencedTimestamp = 300L - listener.handleCommit( - RecordTestUtils.mockBatchReader( - 100, - unfencedTimestamp, - util.Arrays.asList(new ApiMessageAndVersion(new RegisterBrokerRecord(). - setBrokerId(0). - setBrokerEpoch(100L). - setFenced(false). - setRack(null). - setIncarnationId(Uuid.fromString("GFBwlTcpQUuLYQ2ig05CSg")), 0.toShort)) - ) - ) - val imageRecords = listener.getImageRecords().get() - assertEquals(0, imageRecords.size()) - assertEquals(100L, listener.highestMetadataOffset) - assertEquals(0L, metrics.lastAppliedRecordOffset.get) - assertEquals(0L, metrics.lastAppliedRecordTimestamp.get) - assertEquals(0L, metrics.metadataLoadErrorCount.get) - assertEquals(0L, metrics.metadataApplyErrorCount.get) - - val fencedTimestamp = 500L - val fencedLastOffset = 200L - listener.handleCommit( - RecordTestUtils.mockBatchReader( - fencedLastOffset, - fencedTimestamp, - util.Arrays.asList(new ApiMessageAndVersion(new RegisterBrokerRecord(). - setBrokerId(1). - setBrokerEpoch(200L). - setFenced(true). - setRack(null). - setIncarnationId(Uuid.fromString("QkOQtNKVTYatADcaJ28xDg")), 0.toShort)) - ) - ) - listener.startPublishing(new MetadataPublisher { - override def publish(delta: MetadataDelta, newImage: MetadataImage): Unit = { - assertEquals(200L, newImage.highestOffsetAndEpoch().offset) - assertEquals(new BrokerRegistration(0, 100L, - Uuid.fromString("GFBwlTcpQUuLYQ2ig05CSg"), Collections.emptyList[Endpoint](), - Collections.emptyMap[String, VersionRange](), Optional.empty[String](), false, false), - delta.clusterDelta().broker(0)) - assertEquals(new BrokerRegistration(1, 200L, - Uuid.fromString("QkOQtNKVTYatADcaJ28xDg"), Collections.emptyList[Endpoint](), - Collections.emptyMap[String, VersionRange](), Optional.empty[String](), true, false), - delta.clusterDelta().broker(1)) - } - - override def publishedOffset: Long = -1 - }).get() - - assertEquals(fencedLastOffset, metrics.lastAppliedRecordOffset.get) - assertEquals(fencedTimestamp, metrics.lastAppliedRecordTimestamp.get) - assertEquals(0L, metrics.metadataLoadErrorCount.get) - assertEquals(0L, metrics.metadataApplyErrorCount.get) - } finally { - listener.close() - } - } - - class MockMetadataSnapshotter extends MetadataSnapshotter { - var image = MetadataImage.EMPTY - val failure = new AtomicReference[Throwable](null) - var activeSnapshotOffset = -1L - var prevCommittedOffset = -1L - var prevCommittedEpoch = -1 - var prevLastContainedLogTime = -1L - - override def maybeStartSnapshot(lastContainedLogTime: Long, newImage: MetadataImage, reason: Set[SnapshotReason]): Boolean = { - try { - if (activeSnapshotOffset == -1L) { - assertTrue(prevCommittedOffset <= newImage.highestOffsetAndEpoch().offset) - assertTrue(prevCommittedEpoch <= newImage.highestOffsetAndEpoch().epoch) - assertTrue(prevLastContainedLogTime <= lastContainedLogTime) - prevCommittedOffset = newImage.highestOffsetAndEpoch().offset - prevCommittedEpoch = newImage.highestOffsetAndEpoch().epoch - prevLastContainedLogTime = lastContainedLogTime - image = newImage - activeSnapshotOffset = newImage.highestOffsetAndEpoch().offset - true - } else { - false - } - } catch { - case t: Throwable => failure.compareAndSet(null, t) - } - } - } - - class MockMetadataPublisher extends MetadataPublisher { - var image = MetadataImage.EMPTY - - override def publish(delta: MetadataDelta, newImage: MetadataImage): Unit = { - image = newImage - } - - override def publishedOffset: Long = -1 - } - - private val FOO_ID = Uuid.fromString("jj1G9utnTuCegi_gpnRgYw") - private val BAR_ID = Uuid.fromString("SzN5j0LvSEaRIJHrxfMAlg") - - private def generateManyRecords(listener: BrokerMetadataListener, - endOffset: Long): Unit = { - (0 to 10000).foreach { _ => - listener.handleCommit( - RecordTestUtils.mockBatchReader( - endOffset, - 0, - util.Arrays.asList( - new ApiMessageAndVersion(new PartitionChangeRecord(). - setPartitionId(0). - setTopicId(FOO_ID). - setRemovingReplicas(Collections.singletonList(1)), 0.toShort), - new ApiMessageAndVersion(new PartitionChangeRecord(). - setPartitionId(0). - setTopicId(FOO_ID). - setRemovingReplicas(Collections.emptyList()), 0.toShort) - ) - ) - ) - } - listener.getImageRecords().get() - } - - private def generateBadRecords(listener: BrokerMetadataListener, - endOffset: Long): Unit = { - listener.handleCommit( - RecordTestUtils.mockBatchReader( - endOffset, - 0, - util.Arrays.asList( - new ApiMessageAndVersion(new PartitionChangeRecord(). - setPartitionId(0). - setTopicId(BAR_ID). - setRemovingReplicas(Collections.singletonList(1)), 0.toShort), - new ApiMessageAndVersion(new PartitionChangeRecord(). - setPartitionId(0). - setTopicId(BAR_ID). - setRemovingReplicas(Collections.emptyList()), 0.toShort) - ) - ) - ) - listener.getImageRecords().get() - } - - @Test - def testHandleCommitsWithNoSnapshotterDefined(): Unit = { - val listener = newBrokerMetadataListener(maxBytesBetweenSnapshots = 1000L) - try { - val brokerIds = 0 to 3 - - registerBrokers(listener, brokerIds, endOffset = 100L) - createTopicWithOnePartition(listener, replicas = brokerIds, endOffset = 200L) - listener.getImageRecords().get() - assertEquals(200L, listener.highestMetadataOffset) - - generateManyRecords(listener, endOffset = 1000L) - assertEquals(1000L, listener.highestMetadataOffset) - } finally { - listener.close() - } - } - - @Test - def testCreateSnapshot(): Unit = { - val snapshotter = new MockMetadataSnapshotter() - val listener = newBrokerMetadataListener(snapshotter = Some(snapshotter), - maxBytesBetweenSnapshots = 1000L) - try { - val brokerIds = 0 to 3 - - registerBrokers(listener, brokerIds, endOffset = 100L) - createTopicWithOnePartition(listener, replicas = brokerIds, endOffset = 200L) - listener.getImageRecords().get() - assertEquals(200L, listener.highestMetadataOffset) - - // Check that we generate at least one snapshot once we see enough records. - assertEquals(-1L, snapshotter.prevCommittedOffset) - generateManyRecords(listener, 1000L) - assertEquals(1000L, snapshotter.prevCommittedOffset) - assertEquals(1000L, snapshotter.activeSnapshotOffset) - snapshotter.activeSnapshotOffset = -1L - - // Test creating a new snapshot after publishing it. - val publisher = new MockMetadataPublisher() - listener.startPublishing(publisher).get() - generateManyRecords(listener, 2000L) - listener.getImageRecords().get() - assertEquals(2000L, snapshotter.activeSnapshotOffset) - assertEquals(2000L, snapshotter.prevCommittedOffset) - - // Test how we handle the snapshotter returning false. - generateManyRecords(listener, 3000L) - assertEquals(2000L, snapshotter.activeSnapshotOffset) - generateManyRecords(listener, 4000L) - assertEquals(2000L, snapshotter.activeSnapshotOffset) - snapshotter.activeSnapshotOffset = -1L - generateManyRecords(listener, 5000L) - assertEquals(5000L, snapshotter.activeSnapshotOffset) - assertEquals(null, snapshotter.failure.get()) - } finally { - listener.close() - } - } - - @Test - def testNotSnapshotAfterMetadataVersionChangeBeforePublishing(): Unit = { - val snapshotter = new MockMetadataSnapshotter() - val listener = newBrokerMetadataListener(snapshotter = Some(snapshotter), - maxBytesBetweenSnapshots = 1000L) - - updateFeature(listener, feature = MetadataVersion.FEATURE_NAME, MetadataVersion.latest.featureLevel(), 100L) - listener.getImageRecords().get() - assertEquals(-1L, snapshotter.activeSnapshotOffset, "We won't generate snapshot on metadata version change before starting publishing") - } - - @Test - def testSnapshotAfterMetadataVersionChangeWhenStarting(): Unit = { - val snapshotter = new MockMetadataSnapshotter() - val listener = newBrokerMetadataListener(snapshotter = Some(snapshotter), - maxBytesBetweenSnapshots = 1000L) - - val endOffset = 100L - updateFeature(listener, feature = MetadataVersion.FEATURE_NAME, MetadataVersion.latest.featureLevel(), endOffset) - listener.startPublishing(new MockMetadataPublisher()).get() - assertEquals(endOffset, snapshotter.activeSnapshotOffset, "We should try to generate snapshot when starting publishing") - } - - @Test - def testSnapshotAfterMetadataVersionChange(): Unit = { - val snapshotter = new MockMetadataSnapshotter() - val listener = newBrokerMetadataListener(snapshotter = Some(snapshotter), - maxBytesBetweenSnapshots = 1000L) - listener.startPublishing(new MockMetadataPublisher()).get() - - val endOffset = 100L - updateFeature(listener, feature = MetadataVersion.FEATURE_NAME, (MetadataVersion.latest().featureLevel() - 1).toShort, endOffset) - // Waiting for the metadata version update to get processed - listener.getImageRecords().get() - assertEquals(endOffset, snapshotter.activeSnapshotOffset, "We should generate snapshot on feature update") - } - - @Test - def testNoSnapshotAfterError(): Unit = { - val snapshotter = new MockMetadataSnapshotter() - val faultHandler = new MockFaultHandler("metadata loading") - - val listener = newBrokerMetadataListener( - snapshotter = Some(snapshotter), - maxBytesBetweenSnapshots = 1000L, - faultHandler = faultHandler) - try { - val brokerIds = 0 to 3 - - registerBrokers(listener, brokerIds, endOffset = 100L) - createTopicWithOnePartition(listener, replicas = brokerIds, endOffset = 200L) - listener.getImageRecords().get() - assertEquals(200L, listener.highestMetadataOffset) - assertEquals(-1L, snapshotter.prevCommittedOffset) - assertEquals(-1L, snapshotter.activeSnapshotOffset) - - // Append invalid records that will normally trigger a snapshot - generateBadRecords(listener, 1000L) - assertEquals(-1L, snapshotter.prevCommittedOffset) - assertEquals(-1L, snapshotter.activeSnapshotOffset) - - // Generate some records that will not throw an error, verify still no snapshots - generateManyRecords(listener, 2000L) - assertEquals(-1L, snapshotter.prevCommittedOffset) - assertEquals(-1L, snapshotter.activeSnapshotOffset) - } finally { - listener.close() - } - } - - private def registerBrokers( - listener: BrokerMetadataListener, - brokerIds: Iterable[Int], - endOffset: Long - ): Unit = { - brokerIds.foreach { brokerId => - listener.handleCommit( - RecordTestUtils.mockBatchReader( - endOffset, - 0, - util.Arrays.asList(new ApiMessageAndVersion(new RegisterBrokerRecord(). - setBrokerId(brokerId). - setBrokerEpoch(100L). - setFenced(false). - setRack(null). - setIncarnationId(Uuid.fromString("GFBwlTcpQUuLYQ2ig05CS" + brokerId)), 0.toShort)) - ) - ) - } - } - - private def createTopicWithOnePartition( - listener: BrokerMetadataListener, - replicas: Seq[Int], - endOffset: Long - ): Unit = { - listener.handleCommit( - RecordTestUtils.mockBatchReader( - endOffset, - 0, - util.Arrays.asList( - new ApiMessageAndVersion(new TopicRecord(). - setName("foo"). - setTopicId(FOO_ID), 0.toShort), - new ApiMessageAndVersion(new PartitionRecord(). - setPartitionId(0). - setTopicId(FOO_ID). - setIsr(replicas.map(Int.box).asJava). - setLeader(0). - setReplicas(replicas.map(Int.box).asJava), 0.toShort) - ) - ) - ) - } - - private def updateFeature( - listener: BrokerMetadataListener, - feature: String, - version: Short, - endOffset: Long - ): Unit = { - listener.handleCommit( - RecordTestUtils.mockBatchReader( - endOffset, - 0, - util.Arrays.asList( - new ApiMessageAndVersion(new FeatureLevelRecord(). - setName(feature). - setFeatureLevel(version), 0.toShort) - ) - ) - ) - } - -} diff --git a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala b/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala index 472d7ef550b44..d5c1ae6ff63cb 100644 --- a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala +++ b/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataPublisherTest.scala @@ -18,7 +18,7 @@ package kafka.server.metadata import java.util.Collections.{singleton, singletonList, singletonMap} -import java.util.Properties +import java.util.{Collections, Properties} import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} import kafka.log.UnifiedLog import kafka.server.{BrokerServer, KafkaConfig} @@ -35,7 +35,7 @@ import org.apache.kafka.metadata.LeaderRecoveryState import org.apache.kafka.metadata.PartitionRegistration import org.apache.kafka.server.fault.{FaultHandler, MockFaultHandler} import org.junit.jupiter.api.Assertions.{assertEquals, assertNotNull, assertTrue} -import org.junit.jupiter.api.{AfterEach, BeforeEach, Disabled, Test} +import org.junit.jupiter.api.{AfterEach, BeforeEach, Disabled, Test, Timeout} import org.mockito.ArgumentMatchers.any import org.mockito.Mockito import org.mockito.Mockito.doThrow @@ -44,6 +44,7 @@ import org.mockito.stubbing.Answer import scala.jdk.CollectionConverters._ +@Timeout(60) class BrokerMetadataPublisherTest { val exitException = new AtomicReference[Throwable](null) @@ -176,23 +177,13 @@ class BrokerMetadataPublisherTest { new TopicsImage(idsMap.asJava, namesMap.asJava) } - private def newMockPublisher( + private def newMockDynamicConfigPublisher( broker: BrokerServer, errorHandler: FaultHandler = new MockFaultHandler("publisher") - ): BrokerMetadataPublisher = { - Mockito.spy(new BrokerMetadataPublisher( - conf = broker.config, - metadataCache = broker.metadataCache, - logManager = broker.logManager, - replicaManager = broker.replicaManager, - groupCoordinator = broker.groupCoordinator, - txnCoordinator = broker.transactionCoordinator, - clientQuotaMetadataManager = broker.clientQuotaMetadataManager, - dynamicConfigHandlers = broker.dynamicConfigHandlers.toMap, - _authorizer = Option.empty, - errorHandler, - errorHandler - )) + ): DynamicConfigPublisher = { + Mockito.spy(new DynamicConfigPublisher(conf = broker.config, + faultHandler = errorHandler, + dynamicConfigHandlers = broker.dynamicConfigHandlers.toMap)) } @Disabled @@ -207,13 +198,13 @@ class BrokerMetadataPublisherTest { cluster.startup() cluster.waitForReadyBrokers() val broker = cluster.brokers().values().iterator().next() - val publisher = newMockPublisher(broker) + val publisher = newMockDynamicConfigPublisher(broker) val numTimesReloadCalled = new AtomicInteger(0) Mockito.when(publisher.reloadUpdatedFilesWithoutConfigChange(any[Properties]())). thenAnswer(new Answer[Unit]() { override def answer(invocation: InvocationOnMock): Unit = numTimesReloadCalled.addAndGet(1) }) - broker.metadataListener.alterPublisher(publisher).get() + broker.metadataPublisher.dynamicConfigPublisher = publisher val admin = Admin.create(cluster.clientProperties()) try { assertEquals(0, numTimesReloadCalled.get()) @@ -240,12 +231,11 @@ class BrokerMetadataPublisherTest { @Test def testExceptionInUpdateCoordinator(): Unit = { - val errorHandler = new MockFaultHandler("publisher") val cluster = new KafkaClusterTestKit.Builder( new TestKitNodes.Builder(). setNumBrokerNodes(1). setNumControllerNodes(1).build()). - setMetadataFaultHandler(errorHandler).build() + build() try { cluster.format() cluster.startup() @@ -256,7 +246,8 @@ class BrokerMetadataPublisherTest { } val publisher = Mockito.spy(broker.metadataPublisher) doThrow(new RuntimeException("injected failure")).when(publisher).updateCoordinator(any(), any(), any(), any(), any()) - broker.metadataListener.alterPublisher(publisher).get() + broker.jointServer.metadataLoader.removeAndClosePublisher(broker.metadataPublisher).get() + broker.jointServer.metadataLoader.installPublishers(Collections.singletonList(publisher)).get() val admin = Admin.create(cluster.clientProperties()) try { admin.createTopics(singletonList(new NewTopic("foo", 1, 1.toShort))).all().get() @@ -264,11 +255,11 @@ class BrokerMetadataPublisherTest { admin.close() } TestUtils.retry(60000) { - assertTrue(Option(errorHandler.firstException()). + assertTrue(Option(cluster.nonFatalFaultHandler().firstException()). flatMap(e => Option(e.getMessage())).getOrElse("(none)").contains("injected failure")) } } finally { - errorHandler.setIgnore(true) + cluster.nonFatalFaultHandler().setIgnore(true) cluster.close() } } diff --git a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataSnapshotterTest.scala b/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataSnapshotterTest.scala deleted file mode 100644 index e93846c0cef35..0000000000000 --- a/core/src/test/scala/unit/kafka/server/metadata/BrokerMetadataSnapshotterTest.scala +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.server.metadata - -import java.nio.ByteBuffer -import java.util.Optional -import java.util.concurrent.{CompletableFuture, CountDownLatch} -import org.apache.kafka.common.memory.MemoryPool -import org.apache.kafka.common.protocol.ByteBufferAccessor -import org.apache.kafka.common.record.{CompressionType, MemoryRecords} -import org.apache.kafka.common.utils.Time -import org.apache.kafka.image.{MetadataDelta, MetadataImage, MetadataImageTest} -import org.apache.kafka.metadata.MetadataRecordSerde -import org.apache.kafka.metadata.util.SnapshotReason -import org.apache.kafka.queue.EventQueue -import org.apache.kafka.raft.OffsetAndEpoch -import org.apache.kafka.server.common.ApiMessageAndVersion -import org.apache.kafka.snapshot.{MockRawSnapshotWriter, RecordsSnapshotWriter, SnapshotWriter} -import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} -import org.junit.jupiter.api.Test - -import java.util -import scala.compat.java8.OptionConverters._ - -class BrokerMetadataSnapshotterTest { - @Test - def testCreateAndClose(): Unit = { - val snapshotter = new BrokerMetadataSnapshotter(0, Time.SYSTEM, None, - (_, _, _) => throw new RuntimeException("unimplemented")) - snapshotter.close() - } - - class MockSnapshotWriterBuilder extends SnapshotWriterBuilder { - var image = new CompletableFuture[MetadataImage] - - override def build(committedOffset: Long, - committedEpoch: Int, - lastContainedLogTime: Long): Option[SnapshotWriter[ApiMessageAndVersion]] = { - val offsetAndEpoch = new OffsetAndEpoch(committedOffset, committedEpoch) - RecordsSnapshotWriter.createWithHeader( - () => { - Optional.of( - new MockRawSnapshotWriter(offsetAndEpoch, consumeSnapshotBuffer(committedOffset, committedEpoch)) - ) - }, - 1024, - MemoryPool.NONE, - Time.SYSTEM, - lastContainedLogTime, - CompressionType.NONE, - MetadataRecordSerde.INSTANCE - ).asScala - } - - def consumeSnapshotBuffer(committedOffset: Long, committedEpoch: Int)(buffer: ByteBuffer): Unit = { - val delta = new MetadataDelta(MetadataImage.EMPTY) - val memoryRecords = MemoryRecords.readableRecords(buffer) - val batchIterator = memoryRecords.batchIterator() - while (batchIterator.hasNext) { - val batch = batchIterator.next() - if (!batch.isControlBatch()) { - batch.forEach(record => { - val recordBuffer = record.value().duplicate() - val messageAndVersion = MetadataRecordSerde.INSTANCE.read( - new ByteBufferAccessor(recordBuffer), recordBuffer.remaining()) - delta.replay(committedOffset, committedEpoch, messageAndVersion.message()) - }) - } - } - image.complete(delta.apply()) - } - } - - class BlockingEvent extends EventQueue.Event { - val latch = new CountDownLatch(1) - override def run(): Unit = latch.await() - } - - @Test - def testCreateSnapshot(): Unit = { - val writerBuilder = new MockSnapshotWriterBuilder() - val snapshotter = new BrokerMetadataSnapshotter(0, Time.SYSTEM, None, writerBuilder) - - try { - val blockingEvent = new BlockingEvent() - val reasons = Set(SnapshotReason.UnknownReason) - - snapshotter.eventQueue.append(blockingEvent) - assertTrue(snapshotter.maybeStartSnapshot(10000L, MetadataImageTest.IMAGE1, reasons)) - assertFalse(snapshotter.maybeStartSnapshot(11000L, MetadataImageTest.IMAGE2, reasons)) - blockingEvent.latch.countDown() - assertEquals(MetadataImageTest.IMAGE1, writerBuilder.image.get()) - } finally { - snapshotter.close() - } - } - - @Test - def testCreateSnapshotMultipleReasons(): Unit = { - val writerBuilder = new MockSnapshotWriterBuilder() - val snapshotter = new BrokerMetadataSnapshotter(0, Time.SYSTEM, None, writerBuilder) - - try { - val blockingEvent = new BlockingEvent() - val reasons = Set(SnapshotReason.MaxBytesExceeded, SnapshotReason.MetadataVersionChanged) - - snapshotter.eventQueue.append(blockingEvent) - assertTrue(snapshotter.maybeStartSnapshot(10000L, MetadataImageTest.IMAGE1, reasons)) - assertFalse(snapshotter.maybeStartSnapshot(11000L, MetadataImageTest.IMAGE2, reasons)) - blockingEvent.latch.countDown() - assertEquals(MetadataImageTest.IMAGE1, writerBuilder.image.get()) - } finally { - snapshotter.close() - } - } - - class MockSnapshotWriter extends SnapshotWriter[ApiMessageAndVersion] { - val batches = new util.ArrayList[util.List[ApiMessageAndVersion]] - override def snapshotId(): OffsetAndEpoch = new OffsetAndEpoch(0, 0) - override def lastContainedLogOffset(): Long = 0 - override def lastContainedLogEpoch(): Int = 0 - override def isFrozen: Boolean = false - override def append(batch: util.List[ApiMessageAndVersion]): Unit = batches.add(batch) - override def freeze(): Unit = {} - override def close(): Unit = {} - } -} diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala index efdee12147b00..2d3b9ee8cb092 100755 --- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala +++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala @@ -1224,7 +1224,7 @@ object TestUtils extends Logging { TestUtils.waitUntilTrue( () => { brokers.forall { broker => - val metadataOffset = broker.asInstanceOf[BrokerServer].metadataPublisher.publishedOffset + val metadataOffset = broker.asInstanceOf[BrokerServer].jointServer.metadataLoader.lastAppliedOffset() metadataOffset >= controllerOffset } }, msg) diff --git a/metadata/src/main/java/org/apache/kafka/controller/AclControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/AclControlManager.java index d3fc0fe76edd8..af94568552df9 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/AclControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/AclControlManager.java @@ -28,7 +28,6 @@ import org.apache.kafka.common.requests.ApiError; import org.apache.kafka.metadata.authorizer.ClusterMetadataAuthorizer; import org.apache.kafka.metadata.authorizer.StandardAcl; -import org.apache.kafka.metadata.authorizer.StandardAclRecordIterator; import org.apache.kafka.metadata.authorizer.StandardAclWithId; import org.apache.kafka.raft.OffsetAndEpoch; import org.apache.kafka.server.authorizer.AclCreateResult; @@ -42,7 +41,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -213,23 +211,8 @@ public void replay(RemoveAccessControlEntryRecord record, } } + // VisibleForTesting Map idToAcl() { return Collections.unmodifiableMap(idToAcl); } - - Iterator> iterator(long epoch) { - Iterator> iterator = idToAcl.entrySet(epoch).iterator(); - return new StandardAclRecordIterator(new Iterator() { - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public StandardAclWithId next() { - Entry entry = iterator.next(); - return new StandardAclWithId(entry.getKey(), entry.getValue()); - } - }); - } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java index 504994b3a5b5c..b859bbfd65d20 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java @@ -36,11 +36,9 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.NoSuchElementException; import java.util.Objects; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -288,44 +286,4 @@ private ApiError validateEntity(ClientQuotaEntity entity, Map va return ApiError.NONE; } - - class ClientQuotaControlIterator implements Iterator> { - private final long epoch; - private final Iterator>> iterator; - - ClientQuotaControlIterator(long epoch) { - this.epoch = epoch; - this.iterator = clientQuotaData.entrySet(epoch).iterator(); - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public List next() { - if (!hasNext()) throw new NoSuchElementException(); - Entry> entry = iterator.next(); - ClientQuotaEntity entity = entry.getKey(); - List records = new ArrayList<>(); - for (Entry quotaEntry : entry.getValue().entrySet(epoch)) { - ClientQuotaRecord record = new ClientQuotaRecord(); - for (Entry entityEntry : entity.entries().entrySet()) { - record.entity().add(new EntityData(). - setEntityType(entityEntry.getKey()). - setEntityName(entityEntry.getValue())); - } - record.setKey(quotaEntry.getKey()); - record.setValue(quotaEntry.getValue()); - record.setRemove(false); - records.add(new ApiMessageAndVersion(record, (short) 0)); - } - return records; - } - } - - ClientQuotaControlIterator iterator(long epoch) { - return new ClientQuotaControlIterator(epoch); - } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java index 5bc5bd2c3af28..df1851de12eb1 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java @@ -28,9 +28,7 @@ import org.apache.kafka.common.metadata.FenceBrokerRecord; import org.apache.kafka.common.metadata.RegisterBrokerRecord; import org.apache.kafka.common.metadata.RegisterBrokerRecord.BrokerEndpoint; -import org.apache.kafka.common.metadata.RegisterBrokerRecord.BrokerEndpointCollection; import org.apache.kafka.common.metadata.RegisterBrokerRecord.BrokerFeature; -import org.apache.kafka.common.metadata.RegisterBrokerRecord.BrokerFeatureCollection; import org.apache.kafka.common.metadata.UnfenceBrokerRecord; import org.apache.kafka.common.metadata.UnregisterBrokerRecord; import org.apache.kafka.common.protocol.ApiMessage; @@ -58,7 +56,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.NoSuchElementException; import java.util.Optional; import java.util.OptionalLong; import java.util.Random; @@ -67,7 +64,6 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static java.util.Collections.singletonList; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -632,57 +628,4 @@ public void addReadyBrokersFuture(CompletableFuture future, int minBrokers readyBrokersFuture = Optional.empty(); } } - - class ClusterControlIterator implements Iterator> { - private final Iterator> iterator; - private final MetadataVersion metadataVersion; - - ClusterControlIterator(long epoch) { - this.iterator = brokerRegistrations.entrySet(epoch).iterator(); - this.metadataVersion = featureControl.metadataVersion(); - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public List next() { - if (!hasNext()) throw new NoSuchElementException(); - Entry entry = iterator.next(); - int brokerId = entry.getKey(); - BrokerRegistration registration = entry.getValue(); - BrokerEndpointCollection endpoints = new BrokerEndpointCollection(); - for (Entry endpointEntry : registration.listeners().entrySet()) { - endpoints.add(new BrokerEndpoint().setName(endpointEntry.getKey()). - setHost(endpointEntry.getValue().host()). - setPort(endpointEntry.getValue().port()). - setSecurityProtocol(endpointEntry.getValue().securityProtocol().id)); - } - BrokerFeatureCollection features = new BrokerFeatureCollection(); - for (Entry featureEntry : registration.supportedFeatures().entrySet()) { - features.add(new BrokerFeature().setName(featureEntry.getKey()). - setMaxSupportedVersion(featureEntry.getValue().max()). - setMinSupportedVersion(featureEntry.getValue().min())); - } - RegisterBrokerRecord record = new RegisterBrokerRecord(). - setBrokerId(brokerId). - setIncarnationId(registration.incarnationId()). - setBrokerEpoch(registration.epoch()). - setEndPoints(endpoints). - setFeatures(features). - setRack(registration.rack().orElse(null)). - setFenced(registration.fenced()); - if (metadataVersion.isInControlledShutdownStateSupported()) { - record.setInControlledShutdown(registration.inControlledShutdown()); - } - return singletonList(new ApiMessageAndVersion(record, - metadataVersion.registerBrokerRecordVersion())); - } - } - - ClusterControlIterator iterator(long epoch) { - return new ClusterControlIterator(epoch); - } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java index d569ae1a6cab7..b5d71230cf473 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java @@ -43,7 +43,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; @@ -474,39 +473,4 @@ Map currentControllerConfig() { Map result = configData.get(currentController); return (result == null) ? Collections.emptyMap() : result; } - - class ConfigurationControlIterator implements Iterator> { - private final long epoch; - private final Iterator>> iterator; - - ConfigurationControlIterator(long epoch) { - this.epoch = epoch; - this.iterator = configData.entrySet(epoch).iterator(); - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public List next() { - if (!hasNext()) throw new NoSuchElementException(); - List records = new ArrayList<>(); - Entry> entry = iterator.next(); - ConfigResource resource = entry.getKey(); - for (Entry configEntry : entry.getValue().entrySet(epoch)) { - records.add(new ApiMessageAndVersion(new ConfigRecord(). - setResourceName(resource.name()). - setResourceType(resource.type().id()). - setName(configEntry.getKey()). - setValue(configEntry.getValue()), (short) 0)); - } - return records; - } - } - - ConfigurationControlIterator iterator(long epoch) { - return new ConfigurationControlIterator(epoch); - } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/Controller.java b/metadata/src/main/java/org/apache/kafka/controller/Controller.java index 3622fe225dc8f..ed6c523753304 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/Controller.java +++ b/metadata/src/main/java/org/apache/kafka/controller/Controller.java @@ -316,14 +316,6 @@ CompletableFuture updateFeatures( UpdateFeaturesRequestData request ); - /** - * Begin writing a controller snapshot. If there was already an ongoing snapshot, it - * simply returns information about that snapshot rather than starting a new one. - * - * @return A future yielding the epoch of the snapshot. - */ - CompletableFuture beginWritingSnapshot(); - /** * Create partitions on certain topics. * diff --git a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java index 97942835720c0..d7f235d15ec97 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java @@ -20,11 +20,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map.Entry; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Optional; import java.util.TreeMap; import java.util.function.Consumer; @@ -306,45 +304,7 @@ public void replay(FeatureLevelRecord record) { } } - class FeatureControlIterator implements Iterator> { - private final Iterator> iterator; - private final MetadataVersion metadataVersion; - private boolean wroteVersion = false; - - FeatureControlIterator(long epoch) { - this.iterator = finalizedVersions.entrySet(epoch).iterator(); - this.metadataVersion = FeatureControlManager.this.metadataVersion.get(epoch); - } - - @Override - public boolean hasNext() { - return needsWriteMetadataVersion() || iterator.hasNext(); - } - - private boolean needsWriteMetadataVersion() { - return !wroteVersion && metadataVersion.isAtLeast(minimumBootstrapVersion); - } - - @Override - public List next() { - // Write the metadata.version first - if (needsWriteMetadataVersion()) { - wroteVersion = true; - return Collections.singletonList(new ApiMessageAndVersion(new FeatureLevelRecord() - .setName(MetadataVersion.FEATURE_NAME) - .setFeatureLevel(metadataVersion.featureLevel()), FEATURE_LEVEL_RECORD.lowestSupportedVersion())); - } - - // Then write the rest of the features - if (!iterator.hasNext()) throw new NoSuchElementException(); - Entry entry = iterator.next(); - return Collections.singletonList(new ApiMessageAndVersion(new FeatureLevelRecord() - .setName(entry.getKey()) - .setFeatureLevel(entry.getValue()), (short) 0)); - } - } - - FeatureControlIterator iterator(long epoch) { - return new FeatureControlIterator(epoch); + boolean isQuorumNodeId(int nodeId) { + return quorumFeatures.isQuorumNodeId(nodeId); } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ProducerIdControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ProducerIdControlManager.java index 178ef46bdb55e..47e4e1b430fdb 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ProducerIdControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ProducerIdControlManager.java @@ -25,14 +25,10 @@ import org.apache.kafka.timeline.TimelineLong; import org.apache.kafka.timeline.TimelineObject; -import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; -import java.util.List; public class ProducerIdControlManager { - private final ClusterControlManager clusterControlManager; private final TimelineObject nextProducerBlock; private final TimelineLong brokerEpoch; @@ -62,6 +58,11 @@ ControllerResult generateNextProducerId(int brokerId, long bro return ControllerResult.of(Collections.singletonList(new ApiMessageAndVersion(record, (short) 0)), block); } + // VisibleForTesting + ProducerIdsBlock nextProducerBlock() { + return nextProducerBlock.get(); + } + void replay(ProducerIdsRecord record) { long currentNextProducerId = nextProducerBlock.get().firstProducerId(); if (record.nextProducerId() <= currentNextProducerId) { @@ -72,19 +73,4 @@ void replay(ProducerIdsRecord record) { brokerEpoch.set(record.brokerEpoch()); } } - - Iterator> iterator(long epoch) { - List records = new ArrayList<>(1); - - ProducerIdsBlock producerIdBlock = nextProducerBlock.get(epoch); - if (producerIdBlock.firstProducerId() > 0) { - records.add(new ApiMessageAndVersion( - new ProducerIdsRecord() - .setNextProducerId(producerIdBlock.firstProducerId()) - .setBrokerId(producerIdBlock.assignedBrokerId()) - .setBrokerEpoch(brokerEpoch.get(epoch)), - (short) 0)); - } - return Collections.singleton(records).iterator(); - } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index 9aaea73082cd6..fae44019b639f 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -72,7 +72,6 @@ import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; -import org.apache.kafka.controller.SnapshotGenerator.Section; import org.apache.kafka.metadata.BrokerHeartbeatReply; import org.apache.kafka.metadata.BrokerRegistrationReply; import org.apache.kafka.metadata.FinalizedControllerFeatures; @@ -89,7 +88,6 @@ import org.apache.kafka.raft.LeaderAndEpoch; import org.apache.kafka.raft.OffsetAndEpoch; import org.apache.kafka.raft.RaftClient; -import org.apache.kafka.metadata.util.SnapshotReason; import org.apache.kafka.server.authorizer.AclCreateResult; import org.apache.kafka.server.authorizer.AclDeleteResult; import org.apache.kafka.server.common.ApiMessageAndVersion; @@ -98,7 +96,6 @@ import org.apache.kafka.server.policy.AlterConfigPolicy; import org.apache.kafka.server.policy.CreateTopicPolicy; import org.apache.kafka.snapshot.SnapshotReader; -import org.apache.kafka.snapshot.SnapshotWriter; import org.apache.kafka.timeline.SnapshotRegistry; import org.slf4j.Logger; @@ -167,7 +164,6 @@ static public class Builder { private short defaultReplicationFactor = 3; private int defaultNumPartitions = 1; private ReplicaPlacer replicaPlacer = new StripedReplicaPlacer(new Random()); - private long snapshotMaxNewRecordBytes = Long.MAX_VALUE; private OptionalLong leaderImbalanceCheckIntervalNs = OptionalLong.empty(); private OptionalLong maxIdleIntervalNs = OptionalLong.empty(); private long sessionTimeoutNs = ClusterControlManager.DEFAULT_SESSION_TIMEOUT_NS; @@ -239,11 +235,6 @@ public Builder setReplicaPlacer(ReplicaPlacer replicaPlacer) { return this; } - public Builder setSnapshotMaxNewRecordBytes(long value) { - this.snapshotMaxNewRecordBytes = value; - return this; - } - public Builder setLeaderImbalanceCheckIntervalNs(OptionalLong value) { this.leaderImbalanceCheckIntervalNs = value; return this; @@ -338,7 +329,6 @@ public QuorumController build() throws Exception { defaultReplicationFactor, defaultNumPartitions, replicaPlacer, - snapshotMaxNewRecordBytes, leaderImbalanceCheckIntervalNs, maxIdleIntervalNs, sessionTimeoutNs, @@ -374,16 +364,17 @@ public void accept(ConfigResource configResource) { if (configResource.name().isEmpty()) break; // Otherwise, check that the broker ID is valid. - int brokerId; + int nodeId; try { - brokerId = Integer.parseInt(configResource.name()); + nodeId = Integer.parseInt(configResource.name()); } catch (NumberFormatException e) { throw new InvalidRequestException("Invalid broker name " + configResource.name()); } - if (!clusterControl.brokerRegistrations().containsKey(brokerId)) { + if (!(clusterControl.brokerRegistrations().containsKey(nodeId) || + featureControl.isQuorumNodeId(nodeId))) { throw new BrokerIdNotRegisteredException("No broker with id " + - brokerId + " found."); + nodeId + " found."); } break; case TOPIC: @@ -505,110 +496,6 @@ private void appendControlEvent(String name, Runnable handler) { queue.append(event); } - private static final String GENERATE_SNAPSHOT = "generateSnapshot"; - - private static final int MAX_BATCHES_PER_GENERATE_CALL = 10; - - class SnapshotGeneratorManager implements Runnable { - private SnapshotGenerator generator = null; - - void createSnapshotGenerator(long committedOffset, int committedEpoch, long committedTimestamp) { - if (generator != null) { - throw new RuntimeException("Snapshot generator already exists."); - } - if (!snapshotRegistry.hasSnapshot(committedOffset)) { - throw new RuntimeException( - String.format( - "Cannot generate a snapshot at committed offset %d because it does not exists in the snapshot registry.", - committedOffset - ) - ); - } - Optional> writer = raftClient.createSnapshot( - committedOffset, - committedEpoch, - committedTimestamp - ); - if (writer.isPresent()) { - generator = new SnapshotGenerator( - logContext, - writer.get(), - MAX_BATCHES_PER_GENERATE_CALL, - Arrays.asList( - new Section("features", featureControl.iterator(committedOffset)), - new Section("cluster", clusterControl.iterator(committedOffset)), - new Section("replication", replicationControl.iterator(committedOffset)), - new Section("configuration", configurationControl.iterator(committedOffset)), - new Section("clientQuotas", clientQuotaControlManager.iterator(committedOffset)), - new Section("producerIds", producerIdControlManager.iterator(committedOffset)), - new Section("acls", aclControlManager.iterator(committedOffset)) - ) - ); - reschedule(0); - } else { - log.info( - "Skipping generation of snapshot for committed offset {} and epoch {} since it already exists", - committedOffset, - committedEpoch - ); - } - } - - void cancel() { - if (generator == null) return; - log.error("Cancelling snapshot {}", generator.lastContainedLogOffset()); - generator.writer().close(); - generator = null; - - // Delete every in-memory snapshot up to the committed offset. They are not needed since this - // snapshot generation was canceled. - snapshotRegistry.deleteSnapshotsUpTo(lastCommittedOffset); - - queue.cancelDeferred(GENERATE_SNAPSHOT); - } - - void reschedule(long delayNs) { - ControlEvent event = new ControlEvent(GENERATE_SNAPSHOT, this); - queue.scheduleDeferred(event.name, - new EarliestDeadlineFunction(time.nanoseconds() + delayNs), event); - } - - @Override - public void run() { - if (generator == null) { - log.debug("No snapshot is in progress."); - return; - } - OptionalLong nextDelay; - try { - nextDelay = generator.generateBatches(); - } catch (Exception e) { - log.error("Error while generating snapshot {}", generator.lastContainedLogOffset(), e); - generator.writer().close(); - generator = null; - return; - } - if (!nextDelay.isPresent()) { - log.info("Finished generating snapshot {}.", generator.lastContainedLogOffset()); - generator.writer().close(); - generator = null; - - // Delete every in-memory snapshot up to the committed offset. They are not needed since this - // snapshot generation finished. - snapshotRegistry.deleteSnapshotsUpTo(lastCommittedOffset); - return; - } - reschedule(nextDelay.getAsLong()); - } - - OptionalLong snapshotLastOffsetFromLog() { - if (generator == null) { - return OptionalLong.empty(); - } - return OptionalLong.of(generator.lastContainedLogOffset()); - } - } - /** * A controller event that reads the committed internal state in order to expose it * to an API. @@ -931,7 +818,6 @@ public void handleCommit(BatchReader reader) { appendRaftEvent("handleCommit[baseOffset=" + reader.baseOffset() + "]", () -> { try { maybeCompleteAuthorizerInitialLoad(); - long processedRecordsSize = 0; boolean isActive = isActiveController(); while (reader.hasNext()) { Batch batch = reader.next(); @@ -948,10 +834,7 @@ public void handleCommit(BatchReader reader) { purgatory.completeUpTo(offset); // Delete all the in-memory snapshots that we no longer need. - // If we are writing a new snapshot, then we need to keep that around; - // otherwise, we should delete up to the current committed offset. - snapshotRegistry.deleteSnapshotsUpTo( - snapshotGeneratorManager.snapshotLastOffsetFromLog().orElse(offset)); + snapshotRegistry.deleteSnapshotsUpTo(offset); } else { // If the controller is a standby, replay the records that were // created by the active controller. @@ -980,15 +863,9 @@ public void handleCommit(BatchReader reader) { i++; } } - updateLastCommittedState( - offset, - epoch, - batch.appendTimestamp(), - committedBytesSinceLastSnapshot + batch.sizeInBytes() - ); + updateLastCommittedState(offset, epoch, batch.appendTimestamp()); } - maybeGenerateSnapshot(); } finally { reader.close(); } @@ -1040,12 +917,9 @@ public void handleSnapshot(SnapshotReader reader) { i++; } } - updateLastCommittedState( - reader.lastContainedLogOffset(), + updateLastCommittedState(reader.lastContainedLogOffset(), reader.lastContainedLogEpoch(), - reader.lastContainedLogTimestamp(), - 0 - ); + reader.lastContainedLogTimestamp()); snapshotRegistry.getOrCreateSnapshot(lastCommittedOffset); authorizer.ifPresent(a -> a.loadSnapshot(aclControlManager.idToAcl())); } finally { @@ -1201,13 +1075,11 @@ public void processBatchEndOffset(long offset) { private void updateLastCommittedState( long offset, int epoch, - long timestamp, - long bytesSinceLastSnapshot + long timestamp ) { lastCommittedOffset = offset; lastCommittedEpoch = epoch; lastCommittedTimestamp = timestamp; - committedBytesSinceLastSnapshot = bytesSinceLastSnapshot; controllerMetrics.setLastCommittedRecordOffset(offset); if (!isActiveController()) { @@ -1454,42 +1326,12 @@ private void replay(ApiMessage message, Optional snapshotId, lon } } - private void maybeGenerateSnapshot() { - if (committedBytesSinceLastSnapshot >= snapshotMaxNewRecordBytes && - snapshotGeneratorManager.generator == null - ) { - if (!isActiveController()) { - // The active controller creates in-memory snapshot every time an uncommitted - // batch gets appended. The in-active controller can be more efficient and only - // create an in-memory snapshot when needed. - snapshotRegistry.getOrCreateSnapshot(lastCommittedOffset); - } - - log.info( - "Generating a snapshot that includes (epoch={}, offset={}) after {} committed bytes since the last snapshot because, {}.", - lastCommittedEpoch, - lastCommittedOffset, - committedBytesSinceLastSnapshot, - SnapshotReason.MaxBytesExceeded - ); - - snapshotGeneratorManager.createSnapshotGenerator( - lastCommittedOffset, - lastCommittedEpoch, - lastCommittedTimestamp - ); - committedBytesSinceLastSnapshot = 0; - } - } - /** * Clear all data structures and reset all KRaft state. */ private void resetToEmptyState() { - snapshotGeneratorManager.cancel(); snapshotRegistry.reset(); - - updateLastCommittedState(-1, -1, -1, 0); + updateLastCommittedState(-1, -1, -1); } /** @@ -1603,11 +1445,6 @@ private void resetToEmptyState() { */ private final LogReplayTracker logReplayTracker; - /** - * Manages generating controller snapshots. - */ - private final SnapshotGeneratorManager snapshotGeneratorManager = new SnapshotGeneratorManager(); - /** * The interface that we use to mutate the Raft log. */ @@ -1654,16 +1491,6 @@ private void resetToEmptyState() { */ private long writeOffset; - /** - * Maximum number of bytes processed through handling commits before generating a snapshot. - */ - private final long snapshotMaxNewRecordBytes; - - /** - * Number of bytes processed through handling commits since the last snapshot was generated. - */ - private long committedBytesSinceLastSnapshot = 0; - /** * How long to delay partition leader balancing operations. */ @@ -1716,7 +1543,6 @@ private QuorumController( short defaultReplicationFactor, int defaultNumPartitions, ReplicaPlacer replicaPlacer, - long snapshotMaxNewRecordBytes, OptionalLong leaderImbalanceCheckIntervalNs, OptionalLong maxIdleIntervalNs, long sessionTimeoutNs, @@ -1773,7 +1599,6 @@ private QuorumController( setFeatureControlManager(featureControl). build(); this.producerIdControlManager = new ProducerIdControlManager(clusterControl, snapshotRegistry); - this.snapshotMaxNewRecordBytes = snapshotMaxNewRecordBytes; this.leaderImbalanceCheckIntervalNs = leaderImbalanceCheckIntervalNs; this.maxIdleIntervalNs = maxIdleIntervalNs; this.replicationControl = new ReplicationControlManager.Builder(). @@ -2112,29 +1937,6 @@ public CompletableFuture> createPartitions( }); } - @Override - public CompletableFuture beginWritingSnapshot() { - CompletableFuture future = new CompletableFuture<>(); - appendControlEvent("beginWritingSnapshot", () -> { - if (snapshotGeneratorManager.generator == null) { - log.info( - "Generating a snapshot that includes (epoch={}, offset={}) after {} committed bytes since the last snapshot because, {}.", - lastCommittedEpoch, - lastCommittedOffset, - committedBytesSinceLastSnapshot, - SnapshotReason.UnknownReason - ); - snapshotGeneratorManager.createSnapshotGenerator( - lastCommittedOffset, - lastCommittedEpoch, - lastCommittedTimestamp - ); - } - future.complete(snapshotGeneratorManager.generator.lastContainedLogOffset()); - }); - return future; - } - @Override public CompletableFuture> createAcls( ControllerRequestContext context, diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumFeatures.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumFeatures.java index 36725c251857c..85a121a045840 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumFeatures.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumFeatures.java @@ -124,4 +124,8 @@ public Optional reasonNotSupported(String featureName, short level) { VersionRange localSupportedFeature(String featureName) { return localSupportedFeatures.getOrDefault(featureName, DISABLED); } + + boolean isQuorumNodeId(int nodeId) { + return quorumNodeIds.contains(nodeId); + } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java index 934434a5cf71d..6a8d7cf2d9a39 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java @@ -106,7 +106,6 @@ import java.util.ListIterator; import java.util.Map.Entry; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Optional; import java.util.OptionalInt; import java.util.Set; @@ -1908,39 +1907,6 @@ private void listReassigningTopic(ListPartitionReassignmentsResponseData respons setReplicas(Replicas.toList(partition.replicas))); } - class ReplicationControlIterator implements Iterator> { - private final long epoch; - private final Iterator iterator; - - ReplicationControlIterator(long epoch) { - this.epoch = epoch; - this.iterator = topics.values(epoch).iterator(); - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public List next() { - if (!hasNext()) throw new NoSuchElementException(); - TopicControlInfo topic = iterator.next(); - List records = new ArrayList<>(); - records.add(new ApiMessageAndVersion(new TopicRecord(). - setName(topic.name). - setTopicId(topic.id), (short) 0)); - for (Entry entry : topic.parts.entrySet(epoch)) { - records.add(entry.getValue().toRecord(topic.id, entry.getKey())); - } - return records; - } - } - - ReplicationControlIterator iterator(long epoch) { - return new ReplicationControlIterator(epoch); - } - private static final class IneligibleReplica { private final int replicaId; private final String reason; diff --git a/metadata/src/main/java/org/apache/kafka/controller/SnapshotGenerator.java b/metadata/src/main/java/org/apache/kafka/controller/SnapshotGenerator.java deleted file mode 100644 index d34696ef44f11..0000000000000 --- a/metadata/src/main/java/org/apache/kafka/controller/SnapshotGenerator.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kafka.controller; - -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.OptionalLong; - -import org.apache.kafka.common.utils.LogContext; -import org.apache.kafka.server.common.ApiMessageAndVersion; -import org.apache.kafka.snapshot.SnapshotWriter; -import org.slf4j.Logger; - - -final class SnapshotGenerator { - static class Section { - private final String name; - private final Iterator> iterator; - - Section(String name, Iterator> iterator) { - this.name = name; - this.iterator = iterator; - } - - String name() { - return name; - } - - Iterator> iterator() { - return iterator; - } - } - - private final Logger log; - private final SnapshotWriter writer; - private final int maxBatchesPerGenerateCall; - private final List
sections; - private final Iterator
sectionIterator; - private Iterator> batchIterator; - private List batch; - private Section section; - private long numRecords; - - SnapshotGenerator(LogContext logContext, - SnapshotWriter writer, - int maxBatchesPerGenerateCall, - List
sections) { - this.log = logContext.logger(SnapshotGenerator.class); - this.writer = writer; - this.maxBatchesPerGenerateCall = maxBatchesPerGenerateCall; - this.sections = sections; - this.sectionIterator = this.sections.iterator(); - this.batchIterator = Collections.emptyIterator(); - this.batch = null; - this.section = null; - this.numRecords = 0; - } - - /** - * Returns the last offset from the log that will be included in the snapshot. - */ - long lastContainedLogOffset() { - return writer.lastContainedLogOffset(); - } - - SnapshotWriter writer() { - return writer; - } - - /** - * Generate and write the next batch of records. - * - * @return true if the last batch was generated, otherwise false - */ - private boolean generateBatch() throws Exception { - if (batch == null) { - while (!batchIterator.hasNext()) { - if (section != null) { - log.info("Generated {} record(s) for the {} section of snapshot {}.", - numRecords, section.name(), writer.snapshotId()); - section = null; - numRecords = 0; - } - if (!sectionIterator.hasNext()) { - writer.freeze(); - return true; - } - section = sectionIterator.next(); - log.info("Generating records for the {} section of snapshot {}.", - section.name(), writer.snapshotId()); - batchIterator = section.iterator(); - } - batch = batchIterator.next(); - } - - writer.append(batch); - numRecords += batch.size(); - batch = null; - return false; - } - - /** - * Generate the next few batches of records. - * - * @return The number of nanoseconds to delay before rescheduling the - * generateBatches event, or empty if the snapshot is done. - */ - OptionalLong generateBatches() throws Exception { - for (int numBatches = 0; numBatches < maxBatchesPerGenerateCall; numBatches++) { - if (generateBatch()) { - return OptionalLong.empty(); - } - } - return OptionalLong.of(0); - } -} diff --git a/metadata/src/main/java/org/apache/kafka/image/FeaturesDelta.java b/metadata/src/main/java/org/apache/kafka/image/FeaturesDelta.java index 7f431c2d0615f..ac67c87af4bb6 100644 --- a/metadata/src/main/java/org/apache/kafka/image/FeaturesDelta.java +++ b/metadata/src/main/java/org/apache/kafka/image/FeaturesDelta.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Optional; +import java.util.stream.Collectors; /** @@ -34,7 +35,7 @@ public final class FeaturesDelta { private final Map> changes = new HashMap<>(); - private MetadataVersion metadataVersionChange = null; + private Optional metadataVersionChange = Optional.empty(); public FeaturesDelta(FeaturesImage image) { this.image = image; @@ -44,8 +45,8 @@ public Map> changes() { return changes; } - public Optional metadataVersionChange() { - return Optional.ofNullable(metadataVersionChange); + public Optional metadataVersionChange() { + return metadataVersionChange; } public void finishSnapshot() { @@ -54,17 +55,39 @@ public void finishSnapshot() { changes.put(featureName, Optional.empty()); } } + // Note: we do not need to do anything for metadataVersionChange here, since that is + // handled in the constructor of MetadataDelta. + } + + public Optional calculateMetadataVersionChange(FeatureLevelRecord record) { + // Check if the record alters the metadata version. + if (!record.name().equals(MetadataVersion.FEATURE_NAME)) { + return Optional.empty(); + } + // Check if the new version is the same as the previous one. + MetadataVersion oldVersion = metadataVersionChange.isPresent() ? + metadataVersionChange.get().newVersion() : image.metadataVersion(); + MetadataVersion newVersion = MetadataVersion.fromFeatureLevel(record.featureLevel()); + if (oldVersion.equals(newVersion)) return Optional.empty(); + return Optional.of(new MetadataVersionChange(oldVersion, newVersion)); + } + + public void replayMetadataVersionChange(MetadataVersion newVersion) { + if (image.metadataVersion().equals(newVersion)) { + metadataVersionChange = Optional.empty(); + } else { + metadataVersionChange = Optional.of( + new MetadataVersionChange(image.metadataVersion(), newVersion)); + } } public void replay(FeatureLevelRecord record) { if (record.name().equals(MetadataVersion.FEATURE_NAME)) { - metadataVersionChange = MetadataVersion.fromFeatureLevel(record.featureLevel()); + replayMetadataVersionChange(MetadataVersion.fromFeatureLevel(record.featureLevel())); + } else if (record.featureLevel() == 0) { + changes.put(record.name(), Optional.empty()); } else { - if (record.featureLevel() == 0) { - changes.put(record.name(), Optional.empty()); - } else { - changes.put(record.name(), Optional.of(record.featureLevel())); - } + changes.put(record.name(), Optional.of(record.featureLevel())); } } @@ -89,21 +112,18 @@ public FeaturesImage apply() { } } } - - final MetadataVersion metadataVersion; - if (metadataVersionChange == null) { - metadataVersion = image.metadataVersion(); - } else { - metadataVersion = metadataVersionChange; - } - return new FeaturesImage(newFinalizedVersions, metadataVersion); + return new FeaturesImage(newFinalizedVersions, metadataVersionChange.isPresent() ? + metadataVersionChange.get().newVersion() : image.metadataVersion()); } @Override public String toString() { return "FeaturesDelta(" + - "changes=" + changes + - ", metadataVersionChange=" + metadataVersionChange + - ')'; + "changes=" + changes.keySet().stream(). + sorted(). + map(k -> "k=" + changes.get(k)). + collect(Collectors.joining(", ")) + + ", metadataVersionChange=" + metadataVersionChange.toString() + + ")"; } } diff --git a/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java b/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java index a82216769d03f..8df79a1ee6f7e 100644 --- a/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataDelta.java @@ -34,26 +34,37 @@ import org.apache.kafka.common.metadata.UnfenceBrokerRecord; import org.apache.kafka.common.metadata.UnregisterBrokerRecord; import org.apache.kafka.common.protocol.ApiMessage; -import org.apache.kafka.raft.OffsetAndEpoch; -import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.server.common.MetadataVersion; -import java.util.Iterator; -import java.util.List; import java.util.Optional; /** * A change to the broker metadata image. - * - * This class is thread-safe. */ public final class MetadataDelta { - private final MetadataImage image; + public static class Builder { + private MetadataImage image = MetadataImage.EMPTY; + private MetadataVersion metadataVersion = null; + + public Builder setImage(MetadataImage image) { + this.image = image; + return this; + } - private long highestOffset; + public Builder setMetadataVersion(MetadataVersion metadataVersion) { + this.metadataVersion = metadataVersion; + return this; + } + + public MetadataDelta build() { + if (metadataVersion == null) metadataVersion = image.features().metadataVersion(); + return new MetadataDelta(image, + metadataVersion); + } + } - private int highestEpoch; + private final MetadataImage image; private FeaturesDelta featuresDelta = null; @@ -69,10 +80,24 @@ public final class MetadataDelta { private AclsDelta aclsDelta = null; - public MetadataDelta(MetadataImage image) { + private MetadataDelta( + MetadataImage image, + MetadataVersion metadataVersion + ) { this.image = image; - this.highestOffset = image.highestOffsetAndEpoch().offset(); - this.highestEpoch = image.highestOffsetAndEpoch().epoch(); + if (!image.features().metadataVersion().equals(metadataVersion)) { + getOrCreateFeaturesDelta().replayMetadataVersionChange(metadataVersion); + } + } + + public void clear() { + this.featuresDelta = null; + this.clusterDelta = null; + this.topicsDelta = null; + this.configsDelta = null; + this.clientQuotasDelta = null; + this.producerIdsDelta = null; + this.aclsDelta = null; } public MetadataImage image() { @@ -144,27 +169,7 @@ public AclsDelta getOrCreateAclsDelta() { return aclsDelta; } - public Optional metadataVersionChanged() { - if (featuresDelta == null) { - return Optional.empty(); - } else { - return featuresDelta.metadataVersionChange(); - } - } - - public void read(long highestOffset, int highestEpoch, Iterator> reader) { - while (reader.hasNext()) { - List batch = reader.next(); - for (ApiMessageAndVersion messageAndVersion : batch) { - replay(highestOffset, highestEpoch, messageAndVersion.message()); - } - } - } - - public void replay(long offset, int epoch, ApiMessage record) { - highestOffset = offset; - highestEpoch = epoch; - + public void replay(ApiMessage record) { MetadataRecordType type = MetadataRecordType.fromId(record.apiKey()); switch (type) { case REGISTER_BROKER_RECORD: @@ -223,13 +228,11 @@ public void replay(long offset, int epoch, ApiMessage record) { } public void replay(RegisterBrokerRecord record) { - if (clusterDelta == null) clusterDelta = new ClusterDelta(image.cluster()); - clusterDelta.replay(record); + getOrCreateClusterDelta().replay(record); } public void replay(UnregisterBrokerRecord record) { - if (clusterDelta == null) clusterDelta = new ClusterDelta(image.cluster()); - clusterDelta.replay(record); + getOrCreateClusterDelta().replay(record); } public void replay(TopicRecord record) { @@ -261,17 +264,32 @@ public void replay(RemoveTopicRecord record) { getOrCreateConfigsDelta().replay(record, topicName); } + public void setMetadataVersion(MetadataVersion metadataVersion) { + getOrCreateFeaturesDelta().replay(new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(metadataVersion.featureLevel())); + } + public void replay(FeatureLevelRecord record) { - getOrCreateFeaturesDelta().replay(record); - featuresDelta.metadataVersionChange().ifPresent(changedMetadataVersion -> { - // If any feature flags change, need to immediately check if any metadata needs to be downgraded. - getOrCreateClusterDelta().handleMetadataVersionChange(changedMetadataVersion); - getOrCreateConfigsDelta().handleMetadataVersionChange(changedMetadataVersion); - getOrCreateTopicsDelta().handleMetadataVersionChange(changedMetadataVersion); - getOrCreateClientQuotasDelta().handleMetadataVersionChange(changedMetadataVersion); - getOrCreateProducerIdsDelta().handleMetadataVersionChange(changedMetadataVersion); - getOrCreateAclsDelta().handleMetadataVersionChange(changedMetadataVersion); - }); + FeaturesDelta delta = getOrCreateFeaturesDelta(); + Optional change = delta.calculateMetadataVersionChange(record); + if (change.isPresent()) { + // + // If this record would change the metadata version, and the base image is not empty, + // we throw an exception here. The caller is expected to handle this exception by saving + // the image to a series of records with the new metadata version, and then reloading it. + // + // The rationale for handling it "out-of-band" like this is that: + // 1) the broker and controller need to trigger a save of the metadata image right + // before any version change, so we need to interrupt the normal control flow anyway. + // 2) it is easier to maintain one downgrade/upgrade code path in the image write + // functions than to try to handle every conceivable change that a metadata version + // could bring in an event-driven fashion. + // 3) metadata version changes are rare so the overhead of this is not a problem. + // + throw new MetadataVersionChangeException(change.get()); + } + delta.replay(record); } public void replay(BrokerRegistrationChangeRecord record) { @@ -308,7 +326,7 @@ public void finishSnapshot() { getOrCreateAclsDelta().finishSnapshot(); } - public MetadataImage apply() { + public MetadataImage apply(MetadataProvenance provenance) { FeaturesImage newFeatures; if (featuresDelta == null) { newFeatures = image.features(); @@ -352,7 +370,7 @@ public MetadataImage apply() { newAcls = aclsDelta.apply(); } return new MetadataImage( - new OffsetAndEpoch(highestOffset, highestEpoch), + provenance, newFeatures, newCluster, newTopics, @@ -366,9 +384,7 @@ public MetadataImage apply() { @Override public String toString() { return "MetadataDelta(" + - "highestOffset=" + highestOffset + - ", highestEpoch=" + highestEpoch + - ", featuresDelta=" + featuresDelta + + "featuresDelta=" + featuresDelta + ", clusterDelta=" + clusterDelta + ", topicsDelta=" + topicsDelta + ", configsDelta=" + configsDelta + diff --git a/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java b/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java index 18c836ee3bd66..36db5dab1de6b 100644 --- a/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataImage.java @@ -31,7 +31,7 @@ */ public final class MetadataImage { public final static MetadataImage EMPTY = new MetadataImage( - new OffsetAndEpoch(0, 0), + MetadataProvenance.EMPTY, FeaturesImage.EMPTY, ClusterImage.EMPTY, TopicsImage.EMPTY, @@ -40,7 +40,7 @@ public final class MetadataImage { ProducerIdsImage.EMPTY, AclsImage.EMPTY); - private final OffsetAndEpoch highestOffsetAndEpoch; + private final MetadataProvenance provenance; private final FeaturesImage features; @@ -57,7 +57,7 @@ public final class MetadataImage { private final AclsImage acls; public MetadataImage( - OffsetAndEpoch highestOffsetAndEpoch, + MetadataProvenance provenance, FeaturesImage features, ClusterImage cluster, TopicsImage topics, @@ -66,7 +66,7 @@ public MetadataImage( ProducerIdsImage producerIds, AclsImage acls ) { - this.highestOffsetAndEpoch = highestOffsetAndEpoch; + this.provenance = provenance; this.features = features; this.cluster = cluster; this.topics = topics; @@ -86,8 +86,16 @@ public boolean isEmpty() { acls.isEmpty(); } + public MetadataProvenance provenance() { + return provenance; + } + public OffsetAndEpoch highestOffsetAndEpoch() { - return highestOffsetAndEpoch; + return new OffsetAndEpoch(provenance.offset(), provenance.epoch()); + } + + public long offset() { + return provenance.offset(); } public FeaturesImage features() { @@ -135,7 +143,7 @@ public void write(ImageWriter writer, ImageWriterOptions options) { public boolean equals(Object o) { if (o == null || !o.getClass().equals(this.getClass())) return false; MetadataImage other = (MetadataImage) o; - return highestOffsetAndEpoch.equals(other.highestOffsetAndEpoch) && + return provenance.equals(other.provenance) && features.equals(other.features) && cluster.equals(other.cluster) && topics.equals(other.topics) && @@ -147,7 +155,8 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(highestOffsetAndEpoch, + return Objects.hash( + provenance, features, cluster, topics, @@ -159,7 +168,8 @@ public int hashCode() { @Override public String toString() { - return "MetadataImage(highestOffsetAndEpoch=" + highestOffsetAndEpoch + + return "MetadataImage(" + + "provenance=" + provenance + ", features=" + features + ", cluster=" + cluster + ", topics=" + topics + diff --git a/metadata/src/main/java/org/apache/kafka/image/MetadataProvenance.java b/metadata/src/main/java/org/apache/kafka/image/MetadataProvenance.java new file mode 100644 index 0000000000000..09ec10eb9d038 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataProvenance.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image; + +import org.apache.kafka.raft.OffsetAndEpoch; + +import java.util.Objects; + + +/** + * Information about the source of a metadata image. + */ +public final class MetadataProvenance { + public static final MetadataProvenance EMPTY = new MetadataProvenance(-1L, 0, 0L); + + private final long offset; + private final int epoch; + private final long lastContainedLogTimeMs; + + public MetadataProvenance( + long offset, + int epoch, + long lastContainedLogTimeMs + ) { + this.offset = offset; + this.epoch = epoch; + this.lastContainedLogTimeMs = lastContainedLogTimeMs; + } + + public OffsetAndEpoch offsetAndEpoch() { + return new OffsetAndEpoch(offset, epoch); + } + + public long offset() { + return offset; + } + + public int epoch() { + return epoch; + } + + public long lastContainedLogTimeMs() { + return lastContainedLogTimeMs; + } + + public String snapshotName() { + return String.format("snapshot %020d-%010d", offset, epoch); + } + + @Override + public boolean equals(Object o) { + if (o == null || !o.getClass().equals(this.getClass())) return false; + MetadataProvenance other = (MetadataProvenance) o; + return offset == other.offset && + epoch == other.epoch && + lastContainedLogTimeMs == other.lastContainedLogTimeMs; + } + + @Override + public int hashCode() { + return Objects.hash(offset, + epoch, + lastContainedLogTimeMs); + } + + @Override + public String toString() { + return "MetadataProvenance(" + + "offset=" + offset + + ", epoch=" + epoch + + ", lastContainedLogTimeMs=" + lastContainedLogTimeMs + + ")"; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/MetadataVersionChange.java b/metadata/src/main/java/org/apache/kafka/image/MetadataVersionChange.java new file mode 100644 index 0000000000000..4585c2512a3a0 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataVersionChange.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image; + +import org.apache.kafka.server.common.MetadataVersion; + +import java.util.Objects; + + +/** + * A change in the MetadataVersion. + */ +public final class MetadataVersionChange { + private final MetadataVersion oldVersion; + private final MetadataVersion newVersion; + + public MetadataVersionChange( + MetadataVersion oldVersion, + MetadataVersion newVersion + ) { + this.oldVersion = oldVersion; + this.newVersion = newVersion; + } + + public MetadataVersion oldVersion() { + return oldVersion; + } + + public MetadataVersion newVersion() { + return newVersion; + } + + @Override + public boolean equals(Object o) { + if (o == null || !o.getClass().equals(this.getClass())) return false; + MetadataVersionChange other = (MetadataVersionChange) o; + return oldVersion.equals(other.oldVersion) && + newVersion.equals(other.newVersion); + } + + @Override + public int hashCode() { + return Objects.hash(oldVersion, + newVersion); + } + + @Override + public String toString() { + return "MetadataVersionChange(" + + "oldVersion=" + oldVersion + + ", newVersion=" + newVersion + + ")"; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/metadata/util/SnapshotReason.java b/metadata/src/main/java/org/apache/kafka/image/MetadataVersionChangeException.java similarity index 58% rename from metadata/src/main/java/org/apache/kafka/metadata/util/SnapshotReason.java rename to metadata/src/main/java/org/apache/kafka/image/MetadataVersionChangeException.java index 59d95035f2ddd..a4c931f77b7f6 100644 --- a/metadata/src/main/java/org/apache/kafka/metadata/util/SnapshotReason.java +++ b/metadata/src/main/java/org/apache/kafka/image/MetadataVersionChangeException.java @@ -14,21 +14,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.metadata.util; -public enum SnapshotReason { - UnknownReason("unknown reason"), - MaxBytesExceeded("max bytes were exceeded"), - MetadataVersionChanged("metadata version was changed"); +package org.apache.kafka.image; - private final String snapshotReason; - SnapshotReason(String snapshotReason) { - this.snapshotReason = snapshotReason; +/** + * Indicates that the metadata version has changed. + */ +public final class MetadataVersionChangeException extends RuntimeException { + private static final long serialVersionUID = 1L; + private final MetadataVersionChange change; + + public MetadataVersionChangeException(MetadataVersionChange change) { + super("The metadata version is changing from " + change.oldVersion() + " to " + + change.newVersion()); + this.change = change; } - @Override - public String toString() { - return snapshotReason; + public MetadataVersionChange change() { + return change; } } diff --git a/metadata/src/main/java/org/apache/kafka/image/loader/LogDeltaManifest.java b/metadata/src/main/java/org/apache/kafka/image/loader/LogDeltaManifest.java new file mode 100644 index 0000000000000..c3ab72b3b79f2 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/loader/LogDeltaManifest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.loader; + +import org.apache.kafka.image.MetadataProvenance; + +import java.util.Objects; + + +/** + * Contains information about a set of changes that were loaded from the metadata log. + */ +public class LogDeltaManifest { + /** + * The highest offset and epoch included in this delta, inclusive. + */ + private final MetadataProvenance provenance; + + /** + * The number of batches that were loaded. + */ + private final int numBatches; + + /** + * The time in nanoseconds that it took to load the changes. + */ + private final long elapsedNs; + + /** + * The total size of the records in bytes that we read while creating the delta. + */ + private final long numBytes; + + public LogDeltaManifest( + MetadataProvenance provenance, + int numBatches, + long elapsedNs, + long numBytes + ) { + this.provenance = provenance; + this.numBatches = numBatches; + this.elapsedNs = elapsedNs; + this.numBytes = numBytes; + } + + + public MetadataProvenance provenance() { + return provenance; + } + + public int numBatches() { + return numBatches; + } + + public long elapsedNs() { + return elapsedNs; + } + + public long numBytes() { + return numBytes; + } + + @Override + public int hashCode() { + return Objects.hash( + provenance, + numBatches, + elapsedNs, + numBytes); + } + + @Override + public boolean equals(Object o) { + if (o == null || !o.getClass().equals(this.getClass())) return false; + LogDeltaManifest other = (LogDeltaManifest) o; + return provenance.equals(other.provenance) && + numBatches == other.numBatches && + elapsedNs == other.elapsedNs && + numBytes == other.numBytes; + } + + @Override + public String toString() { + return "LogDeltaManifest(" + + "provenance=" + provenance + + ", numBatches=" + numBatches + + ", elapsedNs=" + elapsedNs + + ", numBytes=" + numBytes + + ")"; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java b/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java new file mode 100644 index 0000000000000..a7e2de83d65f1 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java @@ -0,0 +1,541 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.loader; + +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.image.MetadataDelta; +import org.apache.kafka.image.MetadataImage; +import org.apache.kafka.image.MetadataProvenance; +import org.apache.kafka.image.MetadataVersionChange; +import org.apache.kafka.image.MetadataVersionChangeException; +import org.apache.kafka.image.publisher.MetadataPublisher; +import org.apache.kafka.image.writer.ImageReWriter; +import org.apache.kafka.image.writer.ImageWriterOptions; +import org.apache.kafka.queue.KafkaEventQueue; +import org.apache.kafka.raft.Batch; +import org.apache.kafka.raft.BatchReader; +import org.apache.kafka.raft.LeaderAndEpoch; +import org.apache.kafka.raft.RaftClient; +import org.apache.kafka.server.common.ApiMessageAndVersion; +import org.apache.kafka.server.fault.FaultHandler; +import org.apache.kafka.snapshot.SnapshotReader; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; + + +/** + * The MetadataLoader follows changes provided by a RaftClient, and packages them into metadata + * deltas and images that can be consumed by publishers. + * + * The Loader maintains its own thread, which is used to make all callbacks into publishers. If a + * publisher A is installed before B, A will receive all callbacks before B. This is also true if + * A and B are installed as part of a list [A, B]. + * + * Publishers should not modify any data structures passed to them. + * + * It is possible to change the list of publishers dynamically over time. Whenever a new publisher is + * added, it receives a catch-up delta which contains the full state. Any publisher installed when the + * loader is closed will itself be closed. + */ +public class MetadataLoader implements RaftClient.Listener, AutoCloseable { + public static class Builder { + private int nodeId = -1; + private Time time = Time.SYSTEM; + private LogContext logContext = null; + private String threadNamePrefix = ""; + private FaultHandler faultHandler = null; + private MetadataLoaderMetrics metrics = new MetadataLoaderMetrics() { + private volatile long lastAppliedOffset = -1L; + + @Override + public void updateBatchProcessingTime(long elapsedNs) { } + + @Override + public void updateBatchSize(int size) { } + + @Override + public void updateLastAppliedImageProvenance(MetadataProvenance provenance) { + this.lastAppliedOffset = provenance.offset(); + } + + @Override + public long lastAppliedOffset() { + return lastAppliedOffset; + } + + @Override + public void close() throws Exception { } + }; + + public Builder setNodeId(int nodeId) { + this.nodeId = nodeId; + return this; + } + + public Builder setTime(Time time) { + this.time = time; + return this; + } + + public Builder setThreadNamePrefix(String threadNamePrefix) { + this.threadNamePrefix = threadNamePrefix; + return this; + } + + public Builder setFaultHandler(FaultHandler faultHandler) { + this.faultHandler = faultHandler; + return this; + } + + public Builder setMetadataLoaderMetrics(MetadataLoaderMetrics metrics) { + this.metrics = metrics; + return this; + } + + public MetadataLoader build() { + if (logContext == null) { + logContext = new LogContext("[MetadataLoader " + nodeId + "] "); + } + if (faultHandler == null) throw new RuntimeException("You must set a fault handler."); + return new MetadataLoader( + time, + logContext, + threadNamePrefix, + faultHandler, + metrics); + } + } + + /** + * The log4j logger for this loader. + */ + private final Logger log; + + /** + * The clock used by this loader. + */ + private final Time time; + + /** + * The fault handler to use if metadata loading fails. + */ + private final FaultHandler faultHandler; + + /** + * Callbacks for updating metrics. + */ + private final MetadataLoaderMetrics metrics; + + /** + * The publishers which should receive cluster metadata updates. + */ + private final List publishers; + + /** + * The current leader and epoch. + */ + private LeaderAndEpoch currentLeaderAndEpoch = LeaderAndEpoch.UNKNOWN; + + /** + * The current metadata image. Accessed only from the event queue thread. + */ + private MetadataImage image; + + /** + * The event queue which runs this loader. + */ + private final KafkaEventQueue eventQueue; + + private MetadataLoader( + Time time, + LogContext logContext, + String threadNamePrefix, + FaultHandler faultHandler, + MetadataLoaderMetrics metrics + ) { + this.log = logContext.logger(MetadataLoader.class); + this.time = time; + this.faultHandler = faultHandler; + this.metrics = metrics; + this.publishers = new ArrayList<>(); + this.image = MetadataImage.EMPTY; + this.eventQueue = new KafkaEventQueue(time, logContext, threadNamePrefix); + } + + @Override + public void handleCommit(BatchReader reader) { + eventQueue.append(() -> { + try { + MetadataDelta delta = new MetadataDelta.Builder(). + setImage(image). + build(); + LogDeltaManifest manifest = loadLogDelta(delta, reader); + if (log.isDebugEnabled()) { + log.debug("Generated a metadata delta between {} and {} from {} batch(es) " + + "in {} us.", image.offset(), manifest.provenance().offset(), + manifest.numBatches(), NANOSECONDS.toMicros(manifest.elapsedNs())); + } + try { + image = delta.apply(manifest.provenance()); + } catch (Throwable e) { + faultHandler.handleFault("Error generating new metadata image from " + + "metadata delta between offset " + image.offset() + + " and " + manifest.provenance().offset(), e); + return; + } + log.debug("Publishing new image with provenance {}.", image.provenance()); + for (MetadataPublisher publisher : publishers) { + try { + publisher.publishLogDelta(delta, image, manifest); + } catch (Throwable e) { + faultHandler.handleFault("Unhandled error publishing the new metadata " + + "image ending at " + manifest.provenance().offset() + + " with publisher " + publisher.name(), e); + } + } + metrics.updateLastAppliedImageProvenance(image.provenance()); + } catch (Throwable e) { + // This is a general catch-all block where we don't expect to end up; + // failure-prone operations should have individual try/catch blocks around them. + faultHandler.handleFault("Unhandled fault in MetadataLoader#handleCommit. " + + "Last image offset was " + image.offset(), e); + } finally { + reader.close(); + } + }); + } + + /** + * Load some batches of records from the log. We have to do some bookkeeping here to + * translate between batch offsets and record offsets, and track the number of bytes we + * have read. Additionally, there is the chance that one of the records is a metadata + * version change which needs to be handled differently. + * + * @param delta The metadata delta we are preparing. + * @param reader The reader which yields the batches. + * @return A manifest of what was loaded. + */ + LogDeltaManifest loadLogDelta( + MetadataDelta delta, + BatchReader reader + ) { + long startNs = time.nanoseconds(); + int numBatches = 0; + long numBytes = 0L; + long lastOffset = image.provenance().offset(); + int lastEpoch = image.provenance().epoch(); + long lastContainedLogTimeMs = image.provenance().lastContainedLogTimeMs(); + + while (reader.hasNext()) { + Batch batch = reader.next(); + int indexWithinBatch = 0; + for (ApiMessageAndVersion record : batch.records()) { + try { + delta.replay(record.message()); + } catch (MetadataVersionChangeException e) { + MetadataProvenance preChangeProvenance; + if (indexWithinBatch == 0) { + preChangeProvenance = new MetadataProvenance( + lastOffset, + lastEpoch, + lastContainedLogTimeMs); + } else { + preChangeProvenance = new MetadataProvenance( + batch.baseOffset() + indexWithinBatch - 1, + batch.epoch(), + batch.appendTimestamp()); + } + handleMetadataVersionChange(delta, e.change(), preChangeProvenance); + } catch (Throwable e) { + faultHandler.handleFault("Error loading metadata log record from offset " + + batch.baseOffset() + indexWithinBatch, e); + } + indexWithinBatch++; + } + metrics.updateBatchSize(batch.records().size()); + lastOffset = batch.lastOffset(); + lastEpoch = batch.epoch(); + lastContainedLogTimeMs = batch.appendTimestamp(); + numBytes += batch.sizeInBytes(); + numBatches++; + } + MetadataProvenance provenance = + new MetadataProvenance(lastOffset, lastEpoch, lastContainedLogTimeMs); + long elapsedNs = time.nanoseconds() - startNs; + metrics.updateBatchProcessingTime(elapsedNs); + return new LogDeltaManifest(provenance, + numBatches, + elapsedNs, + numBytes); + } + + /** + * Handle a change in the metadata version. + * + * @param delta The metadata delta we're working with. + * @param change The change we're handling. + * @param preChangeProvenance The pre-change provenance. This will be the offset, epoch, + * timestamp, etc. right before the MV change. + */ + void handleMetadataVersionChange( + MetadataDelta delta, + MetadataVersionChange change, + MetadataProvenance preChangeProvenance + ) { + if (image.isEmpty()) { + // If the previous image was empty, this is not a true metadata version transition. + // It just indicates that we hadn't loaded any records before, and now we have. + // Print out the metadata version and return to the loading process. + log.info("Loaded initial metadata version as {}.", change.newVersion()); + delta.setMetadataVersion(change.newVersion()); + return; + } + // First, we materialize the current metadata image and send it to all the publishers that + // are interested in preVersionChange images. The most important one is the publisher which + // writes snapshots out to disk. + PreVersionChangeManifest manifest = new PreVersionChangeManifest(preChangeProvenance, change); + MetadataImage preVersionChangeImage = delta.apply(preChangeProvenance); + log.info("Publishing pre-version change image with provenance {}.", preChangeProvenance); + for (MetadataPublisher publisher : publishers) { + try { + publisher.publishPreVersionChangeImage(delta, preVersionChangeImage, manifest); + } catch (Throwable e) { + faultHandler.handleFault("Error publishing pre-version change image at offset " + + preChangeProvenance.offset() + " with publisher " + publisher.name(), e); + } + } + // Then, we clear the current delta and write out the current image to it in the new format. + // If any metadata was lost, we just log it here. We cannot prevent the losses because the + // decision to change the metadata version was already taken by the controller. + delta.clear(); + delta.setMetadataVersion(change.newVersion()); + ImageReWriter writer = new ImageReWriter(delta); + preVersionChangeImage.write(writer, new ImageWriterOptions.Builder(). + setMetadataVersion(change.newVersion()). + setLossHandler(loss -> { + log.warn("{}", loss.getMessage()); + }). + build()); + } + + @Override + public void handleSnapshot(SnapshotReader reader) { + eventQueue.append(() -> { + try { + MetadataDelta delta = new MetadataDelta.Builder(). + setImage(image). + build(); + SnapshotManifest manifest = loadSnapshot(delta, reader); + if (log.isDebugEnabled()) { + log.debug("Generated a metadata delta from a snapshot at offset {} " + + "in {} us.", manifest.provenance().offset(), + NANOSECONDS.toMicros(manifest.elapsedNs())); + } + try { + image = delta.apply(manifest.provenance()); + } catch (Throwable e) { + faultHandler.handleFault("Error generating new metadata image from " + + "snapshot at offset " + reader.lastContainedLogOffset(), e); + return; + } + log.debug("Publishing new snapshot image with provenance {}.", image.provenance()); + for (MetadataPublisher publisher : publishers) { + try { + publisher.publishSnapshot(delta, image, manifest); + } catch (Throwable e) { + faultHandler.handleFault("Unhandled error publishing the new metadata " + + "image from snapshot at offset " + reader.lastContainedLogOffset() + + " with publisher " + publisher.name(), e); + } + } + metrics.updateLastAppliedImageProvenance(image.provenance()); + } catch (Throwable e) { + // This is a general catch-all block where we don't expect to end up; + // failure-prone operations should have individual try/catch blocks around them. + faultHandler.handleFault("Unhandled fault in MetadataLoader#handleSnapshot. " + + "Snapshot offset was " + reader.lastContainedLogOffset(), e); + } finally { + reader.close(); + } + }); + } + + /** + * Load a snapshot. This is relatively straightforward since we don't track as many things as + * we do in loadLogDelta. The main complication here is that we have to maintain an index + * of what record we are processing so that we can give useful error messages. + * + * @param delta The metadata delta we are preparing. + * @param reader The reader which yields the snapshot batches. + * @return A manifest of what was loaded. + */ + SnapshotManifest loadSnapshot( + MetadataDelta delta, + SnapshotReader reader + ) { + long startNs = time.nanoseconds(); + int snapshotIndex = 0; + while (reader.hasNext()) { + Batch batch = reader.next(); + for (ApiMessageAndVersion record : batch.records()) { + try { + delta.replay(record.message()); + } catch (MetadataVersionChangeException e) { + handleMetadataVersionChange(delta, e.change(), image.provenance()); + } catch (Throwable e) { + faultHandler.handleFault("Error loading metadata log record " + snapshotIndex + + " in snapshot at offset " + reader.lastContainedLogOffset(), e); + } + snapshotIndex++; + } + } + MetadataProvenance provenance = new MetadataProvenance(reader.lastContainedLogOffset(), + reader.lastContainedLogEpoch(), + reader.lastContainedLogTimestamp()); + return new SnapshotManifest(provenance, + time.nanoseconds() - startNs); + } + + @Override + public void handleLeaderChange(LeaderAndEpoch leaderAndEpoch) { + eventQueue.append(() -> { + currentLeaderAndEpoch = leaderAndEpoch; + }); + } + + /** + * Install a list of publishers. When a publisher is installed, we will publish a MetadataDelta + * to it which contains the entire current image. + * + * @param newPublishers The publishers to install. + * + * @return A future which yields null when the publishers have been added, or + * an exception if the installation failed. + */ + public CompletableFuture installPublishers(List newPublishers) { + if (newPublishers.isEmpty()) return CompletableFuture.completedFuture(null); + CompletableFuture future = new CompletableFuture<>(); + eventQueue.append(() -> { + try { + installNewPublishers(newPublishers); + future.complete(null); + } catch (Throwable e) { + future.completeExceptionally(faultHandler.handleFault("Unhandled fault in " + + "MetadataLoader#installPublishers", e)); + } + }); + return future; + } + + void installNewPublishers( + List newPublishers + ) { + long startNs = time.nanoseconds(); + // Publishers can't be re-installed if they're already present. + for (MetadataPublisher publisher : newPublishers) { + if (publishers.contains(publisher)) { + throw faultHandler.handleFault("Attempted to install publisher " + publisher.name() + + ", which is already installed."); + } + } + MetadataDelta delta = new MetadataDelta.Builder(). + setImage(image). + build(); + ImageReWriter writer = new ImageReWriter(delta); + image.write(writer, new ImageWriterOptions.Builder(). + setMetadataVersion(image.features().metadataVersion()). + build()); + SnapshotManifest manifest = new SnapshotManifest( + image.provenance(), + time.nanoseconds() - startNs); + for (MetadataPublisher publisher : newPublishers) { + try { + log.info("Publishing initial snapshot at offset {} to {}", + image.highestOffsetAndEpoch().offset(), publisher.name()); + publisher.publishSnapshot(delta, image, manifest); + publishers.add(publisher); + } catch (Throwable e) { + faultHandler.handleFault("Unhandled error publishing the initial metadata " + + "image from snapshot at offset " + image.highestOffsetAndEpoch().offset() + + " with publisher " + publisher.name(), e); + } + } + } + + // VisibleForTesting + void waitForAllEventsToBeHandled() throws Exception { + CompletableFuture future = new CompletableFuture<>(); + eventQueue.append(() -> future.complete(null)); + future.get(); + } + + /** + * Remove a publisher and close it. + * + * @param publisher The publisher to remove and close. + * + * @return A future which yields null when the publisher has been removed + * and closed, or an exception if the removal failed. + */ + public CompletableFuture removeAndClosePublisher(MetadataPublisher publisher) { + CompletableFuture future = new CompletableFuture<>(); + eventQueue.append(() -> { + try { + if (!publishers.remove(publisher)) { + throw faultHandler.handleFault("Attempted to remove publisher " + publisher.name() + + ", which is not installed."); + } + publisher.close(); + future.complete(null); + } catch (Throwable e) { + future.completeExceptionally(e); + } + }); + return future; + } + + public long lastAppliedOffset() { + return metrics.lastAppliedOffset(); + } + + @Override + public void beginShutdown() { + eventQueue.beginShutdown("beginShutdown", () -> { + for (MetadataPublisher publisher : publishers) { + try { + publisher.close(); + } catch (Throwable e) { + faultHandler.handleFault("Got unexpected exception while closing " + + "publisher " + publisher.name(), e); + } + } + }); + } + + @Override + public void close() throws Exception { + beginShutdown(); + eventQueue.close(); + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoaderMetrics.java b/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoaderMetrics.java new file mode 100644 index 0000000000000..f933703bba23a --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoaderMetrics.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.loader; + + +import org.apache.kafka.image.MetadataProvenance; + +/** + * An interface for the metadata loader metrics. + */ +public interface MetadataLoaderMetrics extends AutoCloseable { + /** + * Update the batch processing time histogram. + */ + void updateBatchProcessingTime(long elapsedNs); + + /** + * Update the batch size histogram. + */ + void updateBatchSize(int size); + + /** + * Set the provenance of the last image which has been processed by all publishers. + */ + void updateLastAppliedImageProvenance(MetadataProvenance provenance); + + /** + * Retrieve the last offset which has been processed by all publishers. + */ + long lastAppliedOffset(); +} diff --git a/metadata/src/main/java/org/apache/kafka/image/loader/PreVersionChangeManifest.java b/metadata/src/main/java/org/apache/kafka/image/loader/PreVersionChangeManifest.java new file mode 100644 index 0000000000000..87684e72291b4 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/loader/PreVersionChangeManifest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.loader; + +import org.apache.kafka.image.MetadataProvenance; +import org.apache.kafka.image.MetadataVersionChange; + +import java.util.Objects; + + +/** + * Contains information about a snapshot that we want to publish right before changing the metadata + * version. + */ +public class PreVersionChangeManifest { + /** + * The source of this snapshot. + */ + private final MetadataProvenance provenance; + + /** + * The new metadata version. + */ + private final MetadataVersionChange change; + + public PreVersionChangeManifest( + MetadataProvenance provenance, + MetadataVersionChange change + ) { + this.provenance = provenance; + this.change = change; + } + + public MetadataProvenance provenance() { + return provenance; + } + + public MetadataVersionChange change() { + return change; + } + + @Override + public int hashCode() { + return Objects.hash( + provenance, + change); + } + + @Override + public boolean equals(Object o) { + if (o == null || !o.getClass().equals(this.getClass())) return false; + PreVersionChangeManifest other = (PreVersionChangeManifest) o; + return provenance.equals(other.provenance) && + change.equals(other.change); + } + + @Override + public String toString() { + return "PreVersionChangeManifest(" + + "provenance=" + provenance + + ", change=" + change + + ")"; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/loader/SnapshotManifest.java b/metadata/src/main/java/org/apache/kafka/image/loader/SnapshotManifest.java new file mode 100644 index 0000000000000..b6c6dcce4d5ea --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/loader/SnapshotManifest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.loader; + +import org.apache.kafka.image.MetadataProvenance; + +import java.util.Objects; + + +/** + * Contains information about a snapshot that was loaded. + */ +public class SnapshotManifest { + /** + * The source of this snapshot. + */ + private final MetadataProvenance provenance; + + /** + * The time in microseconds that it took to load the snapshot. + */ + private final long elapsedNs; + + public SnapshotManifest( + MetadataProvenance provenance, + long elapsedNs + ) { + this.provenance = provenance; + this.elapsedNs = elapsedNs; + } + + public MetadataProvenance provenance() { + return provenance; + } + + public long elapsedNs() { + return elapsedNs; + } + + @Override + public int hashCode() { + return Objects.hash( + provenance, + elapsedNs); + } + + @Override + public boolean equals(Object o) { + if (o == null || !o.getClass().equals(this.getClass())) return false; + SnapshotManifest other = (SnapshotManifest) o; + return provenance.equals(other.provenance) && + elapsedNs == other.elapsedNs; + } + + @Override + public String toString() { + return "SnapshotManifest(" + + "provenance=" + provenance + + ", elapsedNs=" + elapsedNs + + ")"; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/publisher/MetadataPublisher.java b/metadata/src/main/java/org/apache/kafka/image/publisher/MetadataPublisher.java new file mode 100644 index 0000000000000..2870952104fdf --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/publisher/MetadataPublisher.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.publisher; + +import org.apache.kafka.image.MetadataDelta; +import org.apache.kafka.image.MetadataImage; +import org.apache.kafka.image.loader.LogDeltaManifest; +import org.apache.kafka.image.loader.PreVersionChangeManifest; +import org.apache.kafka.image.loader.SnapshotManifest; + + +/** + * Publishes metadata deltas which we have loaded from the log and snapshots. + * + * Publishers receive a stream of callbacks from the metadata loader which keeps them notified + * of the latest cluster metadata. This interface abstracts away some of the complications of + * following the cluster metadata. For example, if the loader needs to read a snapshot, it will + * present the contents of the snapshot in the form of a delta from the previous state. + */ +public interface MetadataPublisher extends AutoCloseable { + /** + * Returns the name of this publisher. + * + * @return The publisher name. + */ + String name(); + + /** + * Publish a new cluster metadata snapshot that we loaded. + * + * @param delta The delta between the previous state and the new one. + * @param newImage The complete new state. + * @param manifest The contents of what was published. + */ + void publishSnapshot( + MetadataDelta delta, + MetadataImage newImage, + SnapshotManifest manifest + ); + + /** + * Publish a cluster metadata snapshot that we created prior to changing the metadata version. + * + * @param delta The delta between the previous state and the new one. + * @param preVersionChangeImage The complete new state. + * @param manifest The contents of what was published. + */ + default void publishPreVersionChangeImage( + MetadataDelta delta, + MetadataImage preVersionChangeImage, + PreVersionChangeManifest manifest + ) { + } + + /** + * Publish a change to the cluster metadata. + * + * @param delta The delta between the previous state and the new one. + * @param newImage The complete new state. + * @param manifest The contents of what was published. + */ + void publishLogDelta( + MetadataDelta delta, + MetadataImage newImage, + LogDeltaManifest manifest + ); + + /** + * Close this metadata publisher. + */ + void close() throws Exception; +} diff --git a/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotEmitter.java b/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotEmitter.java new file mode 100644 index 0000000000000..1dc95d873c5bd --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotEmitter.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.publisher; + +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.image.MetadataImage; +import org.apache.kafka.image.MetadataProvenance; +import org.apache.kafka.image.writer.ImageWriterOptions; +import org.apache.kafka.image.writer.RaftSnapshotWriter; +import org.apache.kafka.raft.RaftClient; +import org.apache.kafka.server.common.ApiMessageAndVersion; +import org.apache.kafka.snapshot.SnapshotWriter; +import org.slf4j.Logger; + +import java.util.Optional; + + +public class SnapshotEmitter implements SnapshotGenerator.Emitter { + private final static int DEFAULT_BATCH_SIZE = 100; + + public static class Builder { + private int nodeId = 0; + private RaftClient raftClient = null; + private int batchSize = DEFAULT_BATCH_SIZE; + + public Builder setNodeId(int nodeId) { + this.nodeId = nodeId; + return this; + } + + public Builder setRaftClient(RaftClient raftClient) { + this.raftClient = raftClient; + return this; + } + + public Builder setBatchSize(int batchSize) { + this.batchSize = batchSize; + return this; + } + + public SnapshotEmitter build() { + if (raftClient == null) throw new RuntimeException("You must set the raftClient."); + return new SnapshotEmitter(nodeId, + raftClient, + batchSize); + } + } + + /** + * The slf4j logger to use. + */ + private final Logger log; + + /** + * The RaftClient to use. + */ + private final RaftClient raftClient; + + /** + * The maximum number of records to put in each batch. + */ + private final int batchSize; + + private SnapshotEmitter( + int nodeId, + RaftClient raftClient, + int batchSize + ) { + this.log = new LogContext("[SnapshotEmitter id=" + nodeId + "] ").logger(SnapshotEmitter.class); + this.raftClient = raftClient; + this.batchSize = batchSize; + } + + @Override + public void maybeEmit(MetadataImage image) { + MetadataProvenance provenance = image.provenance(); + Optional> snapshotWriter = + raftClient.createSnapshot(provenance.offset(), + provenance.epoch(), + provenance.lastContainedLogTimeMs()); + if (!snapshotWriter.isPresent()) { + log.error("Not generating {} because it already exists.", provenance.snapshotName()); + return; + } + RaftSnapshotWriter writer = new RaftSnapshotWriter(snapshotWriter.get(), batchSize); + try { + image.write(writer, new ImageWriterOptions.Builder(). + setMetadataVersion(image.features().metadataVersion()). + build()); + writer.close(true); + } catch (Throwable e) { + log.error("Encountered error while writing {}", provenance.snapshotName(), e); + throw e; + } finally { + Utils.closeQuietly(writer, "RaftSnapshotWriter"); + Utils.closeQuietly(snapshotWriter.get(), "SnapshotWriter"); + } + log.info("Successfully wrote {}", provenance.snapshotName()); + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java b/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java new file mode 100644 index 0000000000000..bd340f2615110 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/publisher/SnapshotGenerator.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.publisher; + +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.image.MetadataDelta; +import org.apache.kafka.image.MetadataImage; +import org.apache.kafka.image.loader.LogDeltaManifest; +import org.apache.kafka.image.loader.PreVersionChangeManifest; +import org.apache.kafka.image.loader.SnapshotManifest; +import org.apache.kafka.queue.EventQueue; +import org.apache.kafka.queue.KafkaEventQueue; +import org.apache.kafka.server.fault.FaultHandler; +import org.slf4j.Logger; + +import java.util.Objects; +import java.util.concurrent.TimeUnit; + + +/** + * A metadata publisher that generates snapshots when appropriate. + */ +public class SnapshotGenerator implements MetadataPublisher { + public static class Builder { + private final Emitter emitter; + private int nodeId = 0; + private Time time = Time.SYSTEM; + private FaultHandler faultHandler = (m, e) -> null; + private long minBytesSinceLastSnapshot = 100 * 1024L * 1024L; + private long maxTimeSinceLastSnapshotNs = TimeUnit.DAYS.toNanos(1); + + public Builder(Emitter emitter) { + this.emitter = emitter; + } + + public Builder setNodeId(int nodeId) { + this.nodeId = nodeId; + return this; + } + + public Builder setTime(Time time) { + this.time = time; + return this; + } + + public Builder setFaultHandler(FaultHandler faultHandler) { + this.faultHandler = faultHandler; + return this; + } + + public Builder setMinBytesSinceLastSnapshot(long minBytesSinceLastSnapshot) { + this.minBytesSinceLastSnapshot = minBytesSinceLastSnapshot; + return this; + } + + public Builder setMaxTimeSinceLastSnapshotNs(long maxTimeSinceLastSnapshotNs) { + this.maxTimeSinceLastSnapshotNs = maxTimeSinceLastSnapshotNs; + return this; + } + + public SnapshotGenerator build() { + return new SnapshotGenerator( + nodeId, + time, + emitter, + faultHandler, + minBytesSinceLastSnapshot, + maxTimeSinceLastSnapshotNs + ); + } + } + + /** + * The callback which actually generates the snapshot. + */ + public interface Emitter { + /** + * Emit a snapshot for the given image. + * + * Note: if a snapshot has already been emitted for the given offset and epoch pair, this + * function will not recreate it. + * + * @param image The metadata image to emit. + */ + void maybeEmit(MetadataImage image); + } + + /** + * The node ID. + */ + private final int nodeId; + + /** + * The clock to use. + */ + private final Time time; + + /** + * The emitter callback, which actually generates the snapshot. + */ + private final Emitter emitter; + + /** + * The slf4j logger to use. + */ + private final Logger log; + + /** + * The fault handler to use. + */ + private final FaultHandler faultHandler; + + /** + * The minimum number of bytes we will wait to see before emitting a snapshot. + */ + private final long minBytesSinceLastSnapshot; + + /** + * The maximum amount of time we will wait before emitting a snapshot, or 0 to disable + * time-based snapshotting. + */ + private final long maxTimeSinceLastSnapshotNs; + + /** + * If non-null, the reason why snapshots have been disabled. + */ + private volatile String disabledReason; + + /** + * The event queue used to schedule emitting snapshots. + */ + private final EventQueue eventQueue; + + /** + * The log bytes that we have read since the last snapshot. + */ + private long bytesSinceLastSnapshot; + + /** + * The time at which we created the last snapshot. + */ + private long lastSnapshotTimeNs; + + private SnapshotGenerator( + int nodeId, + Time time, + Emitter emitter, + FaultHandler faultHandler, + long minBytesSinceLastSnapshot, + long maxTimeSinceLastSnapshotNs + ) { + this.nodeId = nodeId; + this.time = time; + this.emitter = emitter; + this.faultHandler = faultHandler; + this.minBytesSinceLastSnapshot = minBytesSinceLastSnapshot; + this.maxTimeSinceLastSnapshotNs = maxTimeSinceLastSnapshotNs; + LogContext logContext = new LogContext("[SnapshotGenerator " + nodeId + "] "); + this.log = logContext.logger(SnapshotGenerator.class); + this.disabledReason = null; + this.eventQueue = new KafkaEventQueue(time, logContext, "SnapshotGenerator" + nodeId); + resetSnapshotCounters(); + log.debug("Starting SnapshotGenerator."); + } + + @Override + public String name() { + return "SnapshotGenerator"; + } + + void resetSnapshotCounters() { + this.bytesSinceLastSnapshot = 0L; + this.lastSnapshotTimeNs = time.nanoseconds(); + } + + @Override + public void publishSnapshot( + MetadataDelta delta, + MetadataImage newImage, + SnapshotManifest manifest + ) { + log.debug("Resetting the snapshot counters because we just read a snapshot at offset {}.", + newImage.provenance().offset()); + resetSnapshotCounters(); + } + + @Override + public void publishPreVersionChangeImage( + MetadataDelta delta, + MetadataImage preVersionChangeImage, + PreVersionChangeManifest manifest + ) { + log.info("Scheduling snapshot on metadata version change from {} to {}.", + manifest.change().oldVersion(), manifest.change().newVersion()); + scheduleEmit("the metadata version changed", preVersionChangeImage); + } + + @Override + public void publishLogDelta( + MetadataDelta delta, + MetadataImage newImage, + LogDeltaManifest manifest + ) { + bytesSinceLastSnapshot += manifest.numBytes(); + if (bytesSinceLastSnapshot >= minBytesSinceLastSnapshot) { + if (eventQueue.isEmpty()) { + scheduleEmit("we have replayed at least " + minBytesSinceLastSnapshot + + " bytes", newImage); + } else if (log.isTraceEnabled()) { + log.trace("Not scheduling bytes-based snapshot because event queue is not empty yet."); + } + } else if (maxTimeSinceLastSnapshotNs != 0 && + (time.nanoseconds() - lastSnapshotTimeNs >= maxTimeSinceLastSnapshotNs)) { + if (eventQueue.isEmpty()) { + scheduleEmit("we have waited at least " + + TimeUnit.NANOSECONDS.toMinutes(maxTimeSinceLastSnapshotNs) + " minute(s)", newImage); + } else if (log.isTraceEnabled()) { + log.trace("Not scheduling time-based snapshot because event queue is not empty yet."); + } + } else if (log.isTraceEnabled()) { + log.trace("Neither time-based nor bytes-based criteria are met; not scheduling snapshot."); + } + } + + void scheduleEmit( + String reason, + MetadataImage image + ) { + resetSnapshotCounters(); + eventQueue.append(() -> { + String currentDisabledReason = disabledReason; + if (currentDisabledReason != null) { + log.error("Not emitting {} despite the fact that {} because snapshots are " + + "disabled; {}", image.provenance().snapshotName(), reason, + currentDisabledReason); + } else { + log.info("Creating new KRaft snapshot file {} because {}.", + image.provenance().snapshotName(), reason); + try { + emitter.maybeEmit(image); + } catch (Throwable e) { + faultHandler.handleFault("KRaft snapshot file generation error", e); + } + } + }); + } + + public void beginShutdown() { + log.debug("Beginning shutdown of SnapshotGenerator."); + this.disabledReason = "we are shutting down"; + eventQueue.beginShutdown("beginShutdown"); + } + + public void disable(String disabledReason) { + log.error("Disabling periodic KRaft snapshot generation because {}.", disabledReason); + this.disabledReason = Objects.requireNonNull(disabledReason); + } + + @Override + public void close() throws InterruptedException { + eventQueue.beginShutdown("close"); + log.debug("Closing SnapshotGenerator."); + eventQueue.close(); + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/writer/ImageReWriter.java b/metadata/src/main/java/org/apache/kafka/image/writer/ImageReWriter.java new file mode 100644 index 0000000000000..42a0aaa93a182 --- /dev/null +++ b/metadata/src/main/java/org/apache/kafka/image/writer/ImageReWriter.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.writer; + +import org.apache.kafka.image.MetadataDelta; +import org.apache.kafka.image.MetadataImage; +import org.apache.kafka.server.common.ApiMessageAndVersion; + + +/** + * ImageReWriter writes a metadata image out to another metadata image. + * + * There are a few reasons why you might want to do this. One is to obtain a MetadataDelta + * object which contains everything in the image. Another is to translate an image from + * one metadata version to another. + */ +public class ImageReWriter implements ImageWriter { + private final MetadataDelta delta; + private boolean closed = false; + private MetadataImage image = null; + + public ImageReWriter(MetadataDelta delta) { + this.delta = delta; + } + + @Override + public void write(ApiMessageAndVersion record) { + if (closed) throw new ImageWriterClosedException(); + delta.replay(record.message()); + } + + @Override + public void close(boolean complete) { + if (closed) return; + closed = true; + if (complete) { + image = delta.apply(delta.image().provenance()); + } + } + + public MetadataImage image() { + return image; + } +} diff --git a/metadata/src/main/java/org/apache/kafka/image/writer/ImageWriterOptions.java b/metadata/src/main/java/org/apache/kafka/image/writer/ImageWriterOptions.java index 635a6c77c406e..906d5420b1bf0 100644 --- a/metadata/src/main/java/org/apache/kafka/image/writer/ImageWriterOptions.java +++ b/metadata/src/main/java/org/apache/kafka/image/writer/ImageWriterOptions.java @@ -43,12 +43,16 @@ public Builder setMetadataVersion(MetadataVersion metadataVersion) { return this; } - // Package-private for testing + // Visible for testing public Builder setRawMetadataVersion(MetadataVersion metadataVersion) { this.metadataVersion = metadataVersion; return this; } + public MetadataVersion metadataVersion() { + return metadataVersion; + } + public Builder setLossHandler(Consumer lossHandler) { this.lossHandler = lossHandler; return this; diff --git a/metadata/src/test/java/org/apache/kafka/controller/AclControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/AclControlManagerTest.java index fdc0327645176..566fa4acd5471 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/AclControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/AclControlManagerTest.java @@ -215,11 +215,9 @@ public void testLoadSnapshot() { // Verify that the ACLs stored in the AclControlManager match the ones we expect. Set foundAcls = new HashSet<>(); - for (Iterator> iterator = manager.iterator(Long.MAX_VALUE); - iterator.hasNext(); ) { - for (ApiMessageAndVersion apiMessageAndVersion : iterator.next()) { - assertTrue(foundAcls.add(apiMessageAndVersion)); - } + for (Map.Entry entry : manager.idToAcl().entrySet()) { + foundAcls.add(new ApiMessageAndVersion( + new StandardAclWithId(entry.getKey(), entry.getValue()).toRecord(), (short) 0)); } assertEquals(loadedAcls, foundAcls); @@ -233,7 +231,7 @@ public void testLoadSnapshot() { // a cluster metadata authorizer. snapshotRegistry.revertToSnapshot(0); authorizer.loadSnapshot(manager.idToAcl()); - assertFalse(manager.iterator(Long.MAX_VALUE).hasNext()); + assertTrue(manager.idToAcl().isEmpty()); } @Test @@ -243,11 +241,9 @@ public void testAddAndDelete() { MockClusterMetadataAuthorizer authorizer = new MockClusterMetadataAuthorizer(); authorizer.loadSnapshot(manager.idToAcl()); manager.replay(StandardAclWithIdTest.TEST_ACLS.get(0).toRecord(), Optional.empty()); - assertEquals(new ApiMessageAndVersion(TEST_ACLS.get(0).toRecord(), (short) 0), - manager.iterator(Long.MAX_VALUE).next().get(0)); manager.replay(new RemoveAccessControlEntryRecord(). setId(TEST_ACLS.get(0).id()), Optional.empty()); - assertFalse(manager.iterator(Long.MAX_VALUE).hasNext()); + assertTrue(manager.idToAcl().isEmpty()); } @Test @@ -284,7 +280,7 @@ public void testCreateAclDeleteAcl() { } } RecordTestUtils.replayAll(manager, createResult.records()); - assertTrue(manager.iterator(Long.MAX_VALUE).hasNext()); + assertFalse(manager.idToAcl().isEmpty()); ControllerResult> deleteResult = manager.deleteAcls(Arrays.asList( @@ -308,12 +304,8 @@ public void testCreateAclDeleteAcl() { deleteResult.response().get(1).exception().get().getClass()); RecordTestUtils.replayAll(manager, deleteResult.records()); - Iterator> iterator = manager.iterator(Long.MAX_VALUE); - assertTrue(iterator.hasNext()); - List list = iterator.next(); - assertEquals(1, list.size()); - assertEquals(TEST_ACLS.get(1).toBinding(), StandardAcl.fromRecord( - (AccessControlEntryRecord) list.get(0).message()).toBinding()); + Iterator> iterator = manager.idToAcl().entrySet().iterator(); + assertEquals(TEST_ACLS.get(1).acl(), iterator.next().getValue()); assertFalse(iterator.hasNext()); } diff --git a/metadata/src/test/java/org/apache/kafka/controller/ClientQuotaControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ClientQuotaControlManagerTest.java index b915db3fe0d49..1fb81cbf7a473 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ClientQuotaControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ClientQuotaControlManagerTest.java @@ -205,49 +205,50 @@ public void testEntityTypes() throws Exception { List alters = new ArrayList<>(); quotasToTest.forEach((entity, quota) -> entityQuotaToAlterations(entity, quota, alters::add)); - alterQuotas(alters, manager); - - RecordTestUtils.assertBatchIteratorContains(Arrays.asList( - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + List records = alterQuotas(alters, manager); + List expectedRecords = Arrays.asList( + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName("user-1"), new EntityData().setEntityType("client-id").setEntityName("client-id-1"))). - setKey("request_percentage").setValue(50.5).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(50.5).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName("user-2"), new EntityData().setEntityType("client-id").setEntityName("client-id-1"))). - setKey("request_percentage").setValue(51.51).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(51.51).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName("user-3"), new EntityData().setEntityType("client-id").setEntityName("client-id-2"))). - setKey("request_percentage").setValue(52.52).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(52.52).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName(null), new EntityData().setEntityType("client-id").setEntityName("client-id-1"))). - setKey("request_percentage").setValue(53.53).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(53.53).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName("user-1"), new EntityData().setEntityType("client-id").setEntityName(null))). - setKey("request_percentage").setValue(54.54).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(54.54).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName("user-3"), new EntityData().setEntityType("client-id").setEntityName(null))). - setKey("request_percentage").setValue(55.55).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(55.55).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName("user-1"))). - setKey("request_percentage").setValue(56.56).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(56.56).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName("user-2"))). - setKey("request_percentage").setValue(57.57).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(57.57).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName("user-3"))). - setKey("request_percentage").setValue(58.58).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(58.58).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("user").setEntityName(null))). - setKey("request_percentage").setValue(59.59).setRemove(false), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( + setKey("request_percentage").setValue(59.59).setRemove(false), (short) 0), + new ApiMessageAndVersion(new ClientQuotaRecord().setEntity(Arrays.asList( new EntityData().setEntityType("client-id").setEntityName("client-id-2"))). - setKey("request_percentage").setValue(60.60).setRemove(false), (short) 0))), - manager.iterator(Long.MAX_VALUE)); + setKey("request_percentage").setValue(60.60).setRemove(false), (short) 0)); + RecordTestUtils.deepSortRecords(records); + RecordTestUtils.deepSortRecords(expectedRecords); + assertEquals(expectedRecords, records); } static void entityQuotaToAlterations(ClientQuotaEntity entity, Map quota, @@ -258,11 +259,15 @@ static void entityQuotaToAlterations(ClientQuotaEntity entity, Map alterations, ClientQuotaControlManager manager) { + static List alterQuotas( + List alterations, + ClientQuotaControlManager manager + ) { ControllerResult> result = manager.alterClientQuotas(alterations); assertTrue(result.response().values().stream().allMatch(ApiError::isSuccess)); result.records().forEach(apiMessageAndVersion -> manager.replay((ClientQuotaRecord) apiMessageAndVersion.message())); + return result.records(); } static Map quotas(String key, Double value) { diff --git a/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java index ceb12cc62542d..7a73788cf3627 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ClusterControlManagerTest.java @@ -34,12 +34,12 @@ import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.image.writer.ImageWriterOptions; import org.apache.kafka.metadata.BrokerRegistration; import org.apache.kafka.metadata.BrokerRegistrationFencingChange; import org.apache.kafka.metadata.BrokerRegistrationInControlledShutdownChange; import org.apache.kafka.metadata.BrokerRegistrationReply; import org.apache.kafka.metadata.FinalizedControllerFeatures; -import org.apache.kafka.metadata.RecordTestUtils; import org.apache.kafka.metadata.VersionRange; import org.apache.kafka.metadata.placement.ClusterDescriber; import org.apache.kafka.metadata.placement.PlacementSpec; @@ -425,7 +425,7 @@ public Iterator usableBrokers() { @ParameterizedTest @EnumSource(value = MetadataVersion.class, names = {"IBP_3_3_IV2", "IBP_3_3_IV3"}) - public void testIterator(MetadataVersion metadataVersion) throws Exception { + public void testRegistrationsToRecords(MetadataVersion metadataVersion) throws Exception { MockTime time = new MockTime(0, 0, 0); SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); FeatureControlManager featureControl = new FeatureControlManager.Builder(). @@ -467,8 +467,12 @@ public void testIterator(MetadataVersion metadataVersion) throws Exception { IN_CONTROLLED_SHUTDOWN.value()); clusterControl.replay(registrationChangeRecord); short expectedVersion = metadataVersion.registerBrokerRecordVersion(); - RecordTestUtils.assertBatchIteratorContains(Arrays.asList( - Arrays.asList(new ApiMessageAndVersion(new RegisterBrokerRecord(). + + ImageWriterOptions options = new ImageWriterOptions.Builder(). + setMetadataVersion(metadataVersion). + setLossHandler(__ -> { }). + build(); + assertEquals(new ApiMessageAndVersion(new RegisterBrokerRecord(). setBrokerEpoch(100).setBrokerId(0).setRack(null). setEndPoints(new BrokerEndpointCollection(Collections.singleton( new BrokerEndpoint().setSecurityProtocol(SecurityProtocol.PLAINTEXT.id). @@ -476,27 +480,28 @@ public void testIterator(MetadataVersion metadataVersion) throws Exception { setName("PLAINTEXT"). setHost("example.com")).iterator())). setInControlledShutdown(metadataVersion.isInControlledShutdownStateSupported()). - setFenced(false), expectedVersion)), - Arrays.asList(new ApiMessageAndVersion(new RegisterBrokerRecord(). + setFenced(false), expectedVersion), + clusterControl.brokerRegistrations().get(0).toRecord(options)); + assertEquals(new ApiMessageAndVersion(new RegisterBrokerRecord(). setBrokerEpoch(100).setBrokerId(1).setRack(null). setEndPoints(new BrokerEndpointCollection(Collections.singleton( new BrokerEndpoint().setSecurityProtocol(SecurityProtocol.PLAINTEXT.id). setPort((short) 9093). setName("PLAINTEXT"). setHost("example.com")).iterator())). - setFenced(false), expectedVersion)), - Arrays.asList(new ApiMessageAndVersion(new RegisterBrokerRecord(). + setFenced(false), expectedVersion), + clusterControl.brokerRegistrations().get(1).toRecord(options)); + assertEquals(new ApiMessageAndVersion(new RegisterBrokerRecord(). setBrokerEpoch(100).setBrokerId(2).setRack(null). setEndPoints(new BrokerEndpointCollection(Collections.singleton( new BrokerEndpoint().setSecurityProtocol(SecurityProtocol.PLAINTEXT.id). setPort((short) 9094). setName("PLAINTEXT"). setHost("example.com")).iterator())). - setFenced(true), expectedVersion))), - clusterControl.iterator(Long.MAX_VALUE)); + setFenced(true), expectedVersion), + clusterControl.brokerRegistrations().get(2).toRecord(options)); } - @Test public void testRegistrationWithUnsupportedMetadataVersion() { SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); diff --git a/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java index 02bd6e4d7e969..bbca7bfbf6b67 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java @@ -135,14 +135,6 @@ public void testReplay() throws Exception { setName("def").setValue("blah")); assertEquals(toMap(entry("abc", "x,y,z"), entry("def", "blah")), manager.getConfigs(MYTOPIC)); - RecordTestUtils.assertBatchIteratorContains(asList( - asList(new ApiMessageAndVersion(new ConfigRecord(). - setResourceType(TOPIC.id()).setResourceName("mytopic"). - setName("abc").setValue("x,y,z"), CONFIG_RECORD.highestSupportedVersion()), - new ApiMessageAndVersion(new ConfigRecord(). - setResourceType(TOPIC.id()).setResourceName("mytopic"). - setName("def").setValue("blah"), CONFIG_RECORD.highestSupportedVersion()))), - manager.iterator(Long.MAX_VALUE)); } @Test diff --git a/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java index 2478f4ce164a4..8345b4e314955 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java @@ -21,6 +21,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -202,25 +203,7 @@ public void testUpdateFeaturesErrorCases() { } @Test - public void testFeatureControlIteratorWithOldMetadataVersion() throws Exception { - // We require minimum of IBP_3_3_IV0 to write metadata version in the snapshot. - - LogContext logContext = new LogContext(); - SnapshotRegistry snapshotRegistry = new SnapshotRegistry(logContext); - FeatureControlManager manager = new FeatureControlManager.Builder() - .setLogContext(logContext) - .setSnapshotRegistry(snapshotRegistry) - .setMetadataVersion(MetadataVersion.IBP_3_2_IV0) - .build(); - - RecordTestUtils.assertBatchIteratorContains( - Collections.emptyList(), - manager.iterator(Long.MAX_VALUE) - ); - } - - @Test - public void testFeatureControlIterator() throws Exception { + public void testReplayRecords() throws Exception { LogContext logContext = new LogContext(); SnapshotRegistry snapshotRegistry = new SnapshotRegistry(logContext); FeatureControlManager manager = new FeatureControlManager.Builder(). @@ -233,17 +216,12 @@ public void testFeatureControlIterator() throws Exception { updateFeatures(updateMap("foo", 5, "bar", 1), Collections.emptyMap(), Collections.emptyMap(), false); RecordTestUtils.replayAll(manager, result.records()); - RecordTestUtils.assertBatchIteratorContains(Arrays.asList( - Arrays.asList(new ApiMessageAndVersion(new FeatureLevelRecord(). - setName("metadata.version"). - setFeatureLevel((short) 4), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new FeatureLevelRecord(). - setName("foo"). - setFeatureLevel((short) 5), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new FeatureLevelRecord(). - setName("bar"). - setFeatureLevel((short) 1), (short) 0))), - manager.iterator(Long.MAX_VALUE)); + assertEquals(MetadataVersion.IBP_3_3_IV0, manager.metadataVersion()); + assertEquals(Optional.of((short) 5), manager.finalizedFeatures(Long.MAX_VALUE).get("foo")); + assertEquals(Optional.of((short) 1), manager.finalizedFeatures(Long.MAX_VALUE).get("bar")); + assertEquals(new HashSet<>(Arrays.asList( + MetadataVersion.FEATURE_NAME, "foo", "bar")), + manager.finalizedFeatures(Long.MAX_VALUE).featureNames()); } private static final FeatureControlManager.Builder TEST_MANAGER_BUILDER1 = diff --git a/metadata/src/test/java/org/apache/kafka/controller/ProducerIdControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ProducerIdControlManagerTest.java index 80c5c505ae0eb..d0c16d1e3b0d7 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ProducerIdControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ProducerIdControlManagerTest.java @@ -26,20 +26,14 @@ import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; -import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.server.common.MetadataVersion; import org.apache.kafka.server.common.ProducerIdsBlock; import org.apache.kafka.timeline.SnapshotRegistry; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.Iterator; -import java.util.List; - import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; public class ProducerIdControlManagerTest { @@ -149,29 +143,11 @@ public void testMaxValue() { } @Test - public void testSnapshotIterator() { - ProducerIdsBlock range = null; + public void testGenerateProducerIds() { for (int i = 0; i < 100; i++) { - range = generateProducerIds(producerIdControlManager, i % 4, 100); - } - - Iterator> snapshotIterator = producerIdControlManager.iterator(Long.MAX_VALUE); - assertTrue(snapshotIterator.hasNext()); - List batch = snapshotIterator.next(); - assertEquals(1, batch.size(), "Producer IDs record batch should only contain a single record"); - assertEquals(range.firstProducerId() + range.size(), ((ProducerIdsRecord) batch.get(0).message()).nextProducerId()); - assertFalse(snapshotIterator.hasNext(), "Producer IDs iterator should only contain a single batch"); - - ProducerIdControlManager newProducerIdManager = new ProducerIdControlManager(clusterControl, snapshotRegistry); - snapshotIterator = producerIdControlManager.iterator(Long.MAX_VALUE); - while (snapshotIterator.hasNext()) { - snapshotIterator.next().forEach(message -> newProducerIdManager.replay((ProducerIdsRecord) message.message())); + generateProducerIds(producerIdControlManager, i % 4, 100); } - - // Verify that after reloading state from this "snapshot", we don't produce any overlapping IDs - long lastProducerID = range.firstProducerId() + range.size() - 1; - range = generateProducerIds(producerIdControlManager, 1, 100); - assertTrue(range.firstProducerId() > lastProducerID); + assertEquals(new ProducerIdsBlock(3, 100000, 1000), producerIdControlManager.nextProducerBlock()); } static ProducerIdsBlock generateProducerIds( diff --git a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java index 90807a2e2a7f6..127f1afa5429a 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/QuorumControllerTest.java @@ -31,8 +31,6 @@ import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; -import java.util.Spliterator; -import java.util.Spliterators; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -42,7 +40,6 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; -import java.util.stream.StreamSupport; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.BrokerIdNotRegisteredException; @@ -85,27 +82,22 @@ import org.apache.kafka.common.metadata.TopicRecord; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ApiError; -import org.apache.kafka.common.utils.BufferSupplier; import org.apache.kafka.controller.QuorumController.ConfigResourceExistenceChecker; import org.apache.kafka.metadata.BrokerHeartbeatReply; +import org.apache.kafka.metadata.BrokerRegistrationFencingChange; import org.apache.kafka.metadata.BrokerRegistrationReply; import org.apache.kafka.metadata.FinalizedControllerFeatures; -import org.apache.kafka.metadata.MetadataRecordSerde; import org.apache.kafka.metadata.PartitionRegistration; -import org.apache.kafka.metadata.RecordTestUtils; import org.apache.kafka.metadata.authorizer.StandardAuthorizer; import org.apache.kafka.metadata.bootstrap.BootstrapMetadata; import org.apache.kafka.metadata.util.BatchFileWriter; import org.apache.kafka.metalog.LocalLogManager; import org.apache.kafka.metalog.LocalLogManagerTestEnv; -import org.apache.kafka.raft.Batch; import org.apache.kafka.raft.OffsetAndEpoch; import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.server.common.MetadataVersion; import org.apache.kafka.snapshot.FileRawSnapshotReader; -import org.apache.kafka.snapshot.SnapshotReader; import org.apache.kafka.snapshot.RawSnapshotReader; -import org.apache.kafka.snapshot.RecordsSnapshotReader; import org.apache.kafka.snapshot.Snapshots; import org.apache.kafka.test.TestUtils; import org.junit.jupiter.api.Disabled; @@ -128,7 +120,6 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; @Timeout(value = 40) @@ -669,345 +660,80 @@ public void testSnapshotSaveAndLoad() throws Throwable { fooId = fooData.topics().find("foo").topicId(); active.allocateProducerIds(ANONYMOUS_CONTEXT, new AllocateProducerIdsRequestData().setBrokerId(0).setBrokerEpoch(brokerEpochs.get(0))).get(); - long snapshotLogOffset = active.beginWritingSnapshot().get(); - reader = logEnv.waitForSnapshot(snapshotLogOffset); - SnapshotReader snapshot = createSnapshotReader(reader); - assertEquals(snapshotLogOffset, snapshot.lastContainedLogOffset()); - checkSnapshotContent(expectedSnapshotContent(fooId, brokerEpochs), snapshot); + controlEnv.close(); + assertEquals(generateTestRecords(fooId, brokerEpochs), logEnv.allRecords()); } - - try ( - LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv.Builder(3). - setSnapshotReader(reader). - build(); - QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv.Builder(logEnv). - setControllerBuilderInitializer(controllerBuilder -> { - controllerBuilder.setConfigSchema(SCHEMA); - }). - build(); - ) { - QuorumController active = controlEnv.activeController(); - long snapshotLogOffset = active.beginWritingSnapshot().get(); - SnapshotReader snapshot = createSnapshotReader( - logEnv.waitForSnapshot(snapshotLogOffset) - ); - assertEquals(snapshotLogOffset, snapshot.lastContainedLogOffset()); - checkSnapshotContent(expectedSnapshotContent(fooId, brokerEpochs), snapshot); - } - } - - @Test - public void testSnapshotConfiguration() throws Throwable { - final int numBrokers = 4; - final int maxNewRecordBytes = 4; - Map brokerEpochs = new HashMap<>(); - Uuid fooId; - try ( - LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv.Builder(3). - build(); - QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv.Builder(logEnv). - setControllerBuilderInitializer(controllerBuilder -> { - controllerBuilder.setConfigSchema(SCHEMA); - controllerBuilder.setSnapshotMaxNewRecordBytes(maxNewRecordBytes); - controllerBuilder.setBootstrapMetadata(SIMPLE_BOOTSTRAP); - }). - build(); - ) { - QuorumController active = controlEnv.activeController(); - for (int i = 0; i < numBrokers; i++) { - BrokerRegistrationReply reply = active.registerBroker(ANONYMOUS_CONTEXT, - new BrokerRegistrationRequestData(). - setBrokerId(i). - setRack(null). - setClusterId(active.clusterId()). - setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_3_IV3)). - setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB" + i)). - setListeners(new ListenerCollection(Arrays.asList(new Listener(). - setName("PLAINTEXT").setHost("localhost"). - setPort(9092 + i)).iterator()))).get(); - brokerEpochs.put(i, reply.epoch()); - } - for (int i = 0; i < numBrokers - 1; i++) { - assertEquals(new BrokerHeartbeatReply(true, false, false, false), - active.processBrokerHeartbeat(ANONYMOUS_CONTEXT, new BrokerHeartbeatRequestData(). - setWantFence(false).setBrokerEpoch(brokerEpochs.get(i)). - setBrokerId(i).setCurrentMetadataOffset(100000L)).get()); - } - CreateTopicsResponseData fooData = active.createTopics(ANONYMOUS_CONTEXT, - new CreateTopicsRequestData().setTopics( - new CreatableTopicCollection(Collections.singleton( - new CreatableTopic().setName("foo").setNumPartitions(-1). - setReplicationFactor((short) -1). - setAssignments(new CreatableReplicaAssignmentCollection( - Arrays.asList(new CreatableReplicaAssignment(). - setPartitionIndex(0). - setBrokerIds(Arrays.asList(0, 1, 2)), - new CreatableReplicaAssignment(). - setPartitionIndex(1). - setBrokerIds(Arrays.asList(1, 2, 0))). - iterator()))).iterator())), - Collections.singleton("foo")).get(); - fooId = fooData.topics().find("foo").topicId(); - active.allocateProducerIds(ANONYMOUS_CONTEXT, - new AllocateProducerIdsRequestData().setBrokerId(0).setBrokerEpoch(brokerEpochs.get(0))).get(); - - SnapshotReader snapshot = createSnapshotReader(logEnv.waitForLatestSnapshot()); - checkSnapshotSubcontent( - expectedSnapshotContent(fooId, brokerEpochs), - snapshot - ); - } - } - - @Test - public void testSnapshotOnlyAfterConfiguredMinBytes() throws Throwable { - final int numBrokers = 4; - final int maxNewRecordBytes = 1000; - Map brokerEpochs = new HashMap<>(); - try ( - LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv.Builder(3). - build(); - QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv.Builder(logEnv). - setControllerBuilderInitializer(controllerBuilder -> { - controllerBuilder.setConfigSchema(SCHEMA); - controllerBuilder.setSnapshotMaxNewRecordBytes(maxNewRecordBytes); - }). - build(); - ) { - QuorumController active = controlEnv.activeController(); - for (int i = 0; i < numBrokers; i++) { - BrokerRegistrationReply reply = active.registerBroker(ANONYMOUS_CONTEXT, - new BrokerRegistrationRequestData(). - setBrokerId(i). - setRack(null). - setClusterId(active.clusterId()). - setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_3_IV3)). - setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB" + i)). - setListeners(new ListenerCollection(Arrays.asList(new Listener(). - setName("PLAINTEXT").setHost("localhost"). - setPort(9092 + i)).iterator()))).get(); - brokerEpochs.put(i, reply.epoch()); - assertEquals(new BrokerHeartbeatReply(true, false, false, false), - active.processBrokerHeartbeat(ANONYMOUS_CONTEXT, new BrokerHeartbeatRequestData(). - setWantFence(false).setBrokerEpoch(brokerEpochs.get(i)). - setBrokerId(i).setCurrentMetadataOffset(100000L)).get()); - } - - assertTrue(logEnv.appendedBytes() < maxNewRecordBytes, - String.format("%s appended bytes is not less than %s max new record bytes", - logEnv.appendedBytes(), - maxNewRecordBytes)); - - // Keep creating topic until we reached the max bytes limit - int counter = 0; - while (logEnv.appendedBytes() < maxNewRecordBytes) { - counter += 1; - String topicName = String.format("foo-%s", counter); - active.createTopics(ANONYMOUS_CONTEXT, new CreateTopicsRequestData().setTopics( - new CreatableTopicCollection(Collections.singleton( - new CreatableTopic().setName(topicName).setNumPartitions(-1). - setReplicationFactor((short) -1). - setAssignments(new CreatableReplicaAssignmentCollection( - Arrays.asList(new CreatableReplicaAssignment(). - setPartitionIndex(0). - setBrokerIds(Arrays.asList(0, 1, 2)), - new CreatableReplicaAssignment(). - setPartitionIndex(1). - setBrokerIds(Arrays.asList(1, 2, 0))). - iterator()))).iterator())), - Collections.singleton(topicName)).get(60, TimeUnit.SECONDS); - } - logEnv.waitForLatestSnapshot(); - } - } - - @Test - public void testSnapshotAfterRepeatedResign() throws Throwable { - final int numBrokers = 4; - final int maxNewRecordBytes = 1000; - Map brokerEpochs = new HashMap<>(); - try ( - LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv.Builder(1). - build(); - QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv.Builder(logEnv). - setControllerBuilderInitializer(controllerBuilder -> { - controllerBuilder.setConfigSchema(SCHEMA); - controllerBuilder.setSnapshotMaxNewRecordBytes(maxNewRecordBytes); - }). - build(); - ) { - QuorumController active = controlEnv.activeController(); - for (int i = 0; i < numBrokers; i++) { - BrokerRegistrationReply reply = active.registerBroker(ANONYMOUS_CONTEXT, - new BrokerRegistrationRequestData(). - setBrokerId(i). - setRack(null). - setClusterId(active.clusterId()). - setFeatures(brokerFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_3_IV3)). - setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB" + i)). - setListeners(new ListenerCollection(Arrays.asList(new Listener(). - setName("PLAINTEXT").setHost("localhost"). - setPort(9092 + i)).iterator()))).get(); - brokerEpochs.put(i, reply.epoch()); - assertEquals(new BrokerHeartbeatReply(true, false, false, false), - active.processBrokerHeartbeat(ANONYMOUS_CONTEXT, new BrokerHeartbeatRequestData(). - setWantFence(false).setBrokerEpoch(brokerEpochs.get(i)). - setBrokerId(i).setCurrentMetadataOffset(100000L)).get()); - } - - assertTrue(logEnv.appendedBytes() < maxNewRecordBytes, - String.format("%s appended bytes is not less than %s max new record bytes", - logEnv.appendedBytes(), - maxNewRecordBytes)); - - // Keep creating topic and resign leader until we reached the max bytes limit - int counter = 0; - while (logEnv.appendedBytes() < maxNewRecordBytes) { - active = controlEnv.activeController(); - - counter += 1; - String topicName = String.format("foo-%s", counter); - active.createTopics(ANONYMOUS_CONTEXT, new CreateTopicsRequestData().setTopics( - new CreatableTopicCollection(Collections.singleton( - new CreatableTopic().setName(topicName).setNumPartitions(-1). - setReplicationFactor((short) -1). - setAssignments(new CreatableReplicaAssignmentCollection( - Arrays.asList(new CreatableReplicaAssignment(). - setPartitionIndex(0). - setBrokerIds(Arrays.asList(0, 1, 2)), - new CreatableReplicaAssignment(). - setPartitionIndex(1). - setBrokerIds(Arrays.asList(1, 2, 0))). - iterator()))).iterator())), - Collections.singleton(topicName)).get(60, TimeUnit.SECONDS); - - LocalLogManager activeLocalLogManager = logEnv.logManagers().get(active.nodeId()); - activeLocalLogManager.resign(activeLocalLogManager.leaderAndEpoch().epoch()); - } - logEnv.waitForLatestSnapshot(); - } - } - - private SnapshotReader createSnapshotReader(RawSnapshotReader reader) { - return RecordsSnapshotReader.of( - reader, - new MetadataRecordSerde(), - BufferSupplier.create(), - Integer.MAX_VALUE, - true - ); } - private List expectedSnapshotContent(Uuid fooId, Map brokerEpochs) { + private List generateTestRecords(Uuid fooId, Map brokerEpochs) { return Arrays.asList( new ApiMessageAndVersion(new FeatureLevelRecord(). setName(MetadataVersion.FEATURE_NAME). setFeatureLevel(MetadataVersion.IBP_3_3_IV3.featureLevel()), (short) 0), - new ApiMessageAndVersion(new TopicRecord(). - setName("foo").setTopicId(fooId), (short) 0), - new ApiMessageAndVersion(new PartitionRecord().setPartitionId(0). - setTopicId(fooId).setReplicas(Arrays.asList(0, 1, 2)). - setIsr(Arrays.asList(0, 1, 2)).setRemovingReplicas(Collections.emptyList()). - setAddingReplicas(Collections.emptyList()).setLeader(0).setLeaderEpoch(0). - setPartitionEpoch(0), (short) 0), - new ApiMessageAndVersion(new PartitionRecord().setPartitionId(1). - setTopicId(fooId).setReplicas(Arrays.asList(1, 2, 0)). - setIsr(Arrays.asList(1, 2, 0)).setRemovingReplicas(Collections.emptyList()). - setAddingReplicas(Collections.emptyList()).setLeader(1).setLeaderEpoch(0). - setPartitionEpoch(0), (short) 0), new ApiMessageAndVersion(new RegisterBrokerRecord(). setBrokerId(0).setBrokerEpoch(brokerEpochs.get(0)). setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB0")). - setEndPoints( - new BrokerEndpointCollection( - Arrays.asList( - new BrokerEndpoint().setName("PLAINTEXT").setHost("localhost"). - setPort(9092).setSecurityProtocol((short) 0)).iterator())). + setEndPoints(new BrokerEndpointCollection( + Arrays.asList(new BrokerEndpoint().setName("PLAINTEXT").setHost("localhost"). + setPort(9092).setSecurityProtocol((short) 0)).iterator())). setFeatures(registrationFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_3_IV3)). setRack(null). - setFenced(false), (short) 1), + setFenced(true), (short) 1), new ApiMessageAndVersion(new RegisterBrokerRecord(). setBrokerId(1).setBrokerEpoch(brokerEpochs.get(1)). setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB1")). - setEndPoints( - new BrokerEndpointCollection( - Arrays.asList( - new BrokerEndpoint().setName("PLAINTEXT").setHost("localhost"). - setPort(9093).setSecurityProtocol((short) 0)).iterator())). + setEndPoints(new BrokerEndpointCollection( + Arrays.asList(new BrokerEndpoint().setName("PLAINTEXT").setHost("localhost"). + setPort(9093).setSecurityProtocol((short) 0)).iterator())). setFeatures(registrationFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_3_IV3)). setRack(null). - setFenced(false), (short) 1), + setFenced(true), (short) 1), new ApiMessageAndVersion(new RegisterBrokerRecord(). setBrokerId(2).setBrokerEpoch(brokerEpochs.get(2)). setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB2")). - setEndPoints( - new BrokerEndpointCollection( - Arrays.asList( - new BrokerEndpoint().setName("PLAINTEXT").setHost("localhost"). - setPort(9094).setSecurityProtocol((short) 0)).iterator())). + setEndPoints(new BrokerEndpointCollection( + Arrays.asList(new BrokerEndpoint().setName("PLAINTEXT").setHost("localhost"). + setPort(9094).setSecurityProtocol((short) 0)).iterator())). setFeatures(registrationFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_3_IV3)). setRack(null). - setFenced(false), (short) 1), + setFenced(true), (short) 1), new ApiMessageAndVersion(new RegisterBrokerRecord(). setBrokerId(3).setBrokerEpoch(brokerEpochs.get(3)). setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB3")). - setEndPoints(new BrokerEndpointCollection(Arrays.asList( - new BrokerEndpoint().setName("PLAINTEXT").setHost("localhost"). + setEndPoints(new BrokerEndpointCollection( + Arrays.asList(new BrokerEndpoint().setName("PLAINTEXT").setHost("localhost"). setPort(9095).setSecurityProtocol((short) 0)).iterator())). setFeatures(registrationFeatures(MetadataVersion.IBP_3_0_IV1, MetadataVersion.IBP_3_3_IV3)). - setRack(null), (short) 1), + setRack(null). + setFenced(true), (short) 1), + new ApiMessageAndVersion(new BrokerRegistrationChangeRecord(). + setBrokerId(0). + setBrokerEpoch(brokerEpochs.get(0)). + setFenced(BrokerRegistrationFencingChange.UNFENCE.value()), (short) 0), + new ApiMessageAndVersion(new BrokerRegistrationChangeRecord(). + setBrokerId(1). + setBrokerEpoch(brokerEpochs.get(1)). + setFenced(BrokerRegistrationFencingChange.UNFENCE.value()), (short) 0), + new ApiMessageAndVersion(new BrokerRegistrationChangeRecord(). + setBrokerId(2). + setBrokerEpoch(brokerEpochs.get(2)). + setFenced(BrokerRegistrationFencingChange.UNFENCE.value()), (short) 0), + new ApiMessageAndVersion(new TopicRecord(). + setName("foo").setTopicId(fooId), (short) 0), + new ApiMessageAndVersion(new PartitionRecord().setPartitionId(0). + setTopicId(fooId).setReplicas(Arrays.asList(0, 1, 2)). + setIsr(Arrays.asList(0, 1, 2)).setRemovingReplicas(Collections.emptyList()). + setAddingReplicas(Collections.emptyList()).setLeader(0).setLeaderEpoch(0). + setPartitionEpoch(0), (short) 0), + new ApiMessageAndVersion(new PartitionRecord().setPartitionId(1). + setTopicId(fooId).setReplicas(Arrays.asList(1, 2, 0)). + setIsr(Arrays.asList(1, 2, 0)).setRemovingReplicas(Collections.emptyList()). + setAddingReplicas(Collections.emptyList()).setLeader(1).setLeaderEpoch(0). + setPartitionEpoch(0), (short) 0), new ApiMessageAndVersion(new ProducerIdsRecord(). setBrokerId(0). setBrokerEpoch(brokerEpochs.get(0)). - setNextProducerId(1000), (short) 0) - ); - } - - private void checkSnapshotContent( - List expected, - Iterator> iterator - ) throws Exception { - RecordTestUtils.assertBatchIteratorContains( - Arrays.asList(expected), - Arrays.asList( - StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false) - .flatMap(batch -> batch.records().stream()) - .collect(Collectors.toList()) - ).iterator() - ); - } - - /** - * This function checks that the iterator is a subset of the expected list. - * - * This is needed because when generating snapshots through configuration is difficult to control exactly when a - * snapshot will be generated and which committed offset will be included in the snapshot. - */ - private void checkSnapshotSubcontent( - List expected, - Iterator> iterator - ) throws Exception { - RecordTestUtils.deepSortRecords(expected); - - List actual = StreamSupport - .stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false) - .flatMap(batch -> batch.records().stream()) - .collect(Collectors.toList()); - - RecordTestUtils.deepSortRecords(actual); - - int expectedIndex = 0; - for (ApiMessageAndVersion current : actual) { - while (expectedIndex < expected.size() && !expected.get(expectedIndex).equals(current)) { - expectedIndex += 1; - } - - if (expectedIndex >= expected.size()) { - fail("Failed to find record " + current + " in the expected record set: " + expected); - } - - expectedIndex += 1; - } + setNextProducerId(1000), (short) 0)); } /** @@ -1341,7 +1067,7 @@ public void testQuorumControllerCompletesAuthorizerInitialLoad() throws Throwabl }). build() ) { - logEnv.appendInitialRecords(expectedSnapshotContent(FOO_ID, ALL_ZERO_BROKER_EPOCHS)); + logEnv.appendInitialRecords(generateTestRecords(FOO_ID, ALL_ZERO_BROKER_EPOCHS)); logEnv.logManagers().forEach(m -> m.setMaxReadOffset(2)); try ( QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv.Builder(logEnv). diff --git a/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java index c81f25e4c000a..0997792cd90f6 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ReplicationControlManagerTest.java @@ -58,7 +58,6 @@ import org.apache.kafka.common.metadata.BrokerRegistrationChangeRecord; import org.apache.kafka.common.metadata.ConfigRecord; import org.apache.kafka.common.metadata.PartitionChangeRecord; -import org.apache.kafka.common.metadata.PartitionRecord; import org.apache.kafka.common.metadata.RegisterBrokerRecord; import org.apache.kafka.common.metadata.RemoveTopicRecord; import org.apache.kafka.common.metadata.TopicRecord; @@ -519,16 +518,6 @@ public void testCreateTopics() throws Exception { setErrorCode(Errors.TOPIC_ALREADY_EXISTS.code()). setErrorMessage("Topic 'foo' already exists.")); assertEquals(expectedResponse4, result4.response()); - Uuid fooId = result3.response().topics().find("foo").topicId(); - RecordTestUtils.assertBatchIteratorContains(asList( - asList(new ApiMessageAndVersion(new PartitionRecord(). - setPartitionId(0).setTopicId(fooId). - setReplicas(asList(1, 2, 0)).setIsr(asList(1, 2, 0)). - setRemovingReplicas(Collections.emptyList()).setAddingReplicas(Collections.emptyList()).setLeader(1). - setLeaderEpoch(0).setPartitionEpoch(0), (short) 0), - new ApiMessageAndVersion(new TopicRecord(). - setTopicId(fooId).setName("foo"), (short) 0))), - ctx.replicationControl.iterator(Long.MAX_VALUE)); } @Test diff --git a/metadata/src/test/java/org/apache/kafka/controller/SnapshotGeneratorTest.java b/metadata/src/test/java/org/apache/kafka/controller/SnapshotGeneratorTest.java deleted file mode 100644 index f7fa18f20a4cf..0000000000000 --- a/metadata/src/test/java/org/apache/kafka/controller/SnapshotGeneratorTest.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kafka.controller; - -import org.apache.kafka.common.Uuid; -import org.apache.kafka.common.config.ConfigResource; -import org.apache.kafka.common.memory.MemoryPool; -import org.apache.kafka.common.metadata.ConfigRecord; -import org.apache.kafka.common.metadata.TopicRecord; -import org.apache.kafka.common.record.CompressionType; -import org.apache.kafka.common.utils.LogContext; -import org.apache.kafka.common.utils.MockTime; -import org.apache.kafka.controller.SnapshotGenerator.Section; -import org.apache.kafka.metadata.MetadataRecordSerde; -import org.apache.kafka.raft.OffsetAndEpoch; -import org.apache.kafka.server.common.ApiMessageAndVersion; -import org.apache.kafka.snapshot.SnapshotWriter; -import org.apache.kafka.snapshot.MockRawSnapshotWriter; -import org.apache.kafka.snapshot.RawSnapshotWriter; -import org.apache.kafka.snapshot.RecordsSnapshotWriter; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; - -import java.util.Arrays; -import java.util.List; -import java.util.OptionalLong; -import java.util.Optional; - -import static org.apache.kafka.raft.KafkaRaftClient.MAX_BATCH_SIZE_BYTES; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - - -@Timeout(40) -public class SnapshotGeneratorTest { - private static final List> BATCHES; - - static { - BATCHES = Arrays.asList( - Arrays.asList(new ApiMessageAndVersion(new TopicRecord(). - setName("foo").setTopicId(Uuid.randomUuid()), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new TopicRecord(). - setName("bar").setTopicId(Uuid.randomUuid()), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new TopicRecord(). - setName("baz").setTopicId(Uuid.randomUuid()), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ConfigRecord(). - setResourceName("foo").setResourceType(ConfigResource.Type.TOPIC.id()). - setName("retention.ms").setValue("10000000"), (short) 0), - new ApiMessageAndVersion(new ConfigRecord(). - setResourceName("foo").setResourceType(ConfigResource.Type.TOPIC.id()). - setName("max.message.bytes").setValue("100000000"), (short) 0)), - Arrays.asList(new ApiMessageAndVersion(new ConfigRecord(). - setResourceName("bar").setResourceType(ConfigResource.Type.TOPIC.id()). - setName("retention.ms").setValue("5000000"), (short) 0))); - } - - @Test - public void testGenerateBatches() throws Exception { - SnapshotWriter writer = createSnapshotWriter(123, 0); - List
sections = Arrays.asList(new Section("replication", - Arrays.asList(BATCHES.get(0), BATCHES.get(1), BATCHES.get(2)).iterator()), - new Section("configuration", - Arrays.asList(BATCHES.get(3), BATCHES.get(4)).iterator())); - SnapshotGenerator generator = new SnapshotGenerator(new LogContext(), - writer, 2, sections); - assertFalse(writer.isFrozen()); - assertEquals(123L, generator.lastContainedLogOffset()); - assertEquals(writer, generator.writer()); - assertEquals(OptionalLong.of(0L), generator.generateBatches()); - assertEquals(OptionalLong.of(0L), generator.generateBatches()); - assertFalse(writer.isFrozen()); - assertEquals(OptionalLong.empty(), generator.generateBatches()); - assertTrue(writer.isFrozen()); - } - - private SnapshotWriter createSnapshotWriter( - long committedOffset, - long lastContainedLogTime - ) { - return RecordsSnapshotWriter.createWithHeader( - () -> createNewSnapshot(new OffsetAndEpoch(committedOffset + 1, 1)), - MAX_BATCH_SIZE_BYTES, - MemoryPool.NONE, - new MockTime(), - lastContainedLogTime, - CompressionType.NONE, - new MetadataRecordSerde() - ).get(); - } - - private Optional createNewSnapshot( - OffsetAndEpoch snapshotId - ) { - return Optional.of(new MockRawSnapshotWriter(snapshotId, buffer -> { })); - } -} diff --git a/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java b/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java index 8c63cbc516d48..f4707a49890d6 100644 --- a/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/FeaturesImageTest.java @@ -30,7 +30,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import static org.apache.kafka.server.common.MetadataVersion.IBP_3_3_IV0; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -49,6 +51,10 @@ public class FeaturesImageTest { IMAGE1 = new FeaturesImage(map1, MetadataVersion.latest()); DELTA1_RECORDS = new ArrayList<>(); + DELTA1_RECORDS.add(new ApiMessageAndVersion(new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(MetadataVersion.latest().featureLevel()), + (short) 0)); DELTA1_RECORDS.add(new ApiMessageAndVersion(new FeatureLevelRecord(). setName("foo").setFeatureLevel((short) 3), (short) 0)); @@ -95,4 +101,29 @@ private void testToImageAndBack(FeaturesImage image) throws Throwable { FeaturesImage nextImage = delta.apply(); assertEquals(image, nextImage); } + + private static final FeatureLevelRecord MV_LATEST_RECORD = new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(MetadataVersion.latest().featureLevel()); + + private static final FeatureLevelRecord MV_3_3_IV0_RECORD = new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(IBP_3_3_IV0.featureLevel()); + + @Test + public void testCalculateMetadataVersionChange() throws Throwable { + FeaturesDelta delta = new FeaturesDelta(IMAGE1); + assertEquals(Optional.empty(), delta.calculateMetadataVersionChange(MV_LATEST_RECORD)); + assertEquals(Optional.of(new MetadataVersionChange( + MetadataVersion.latest(), IBP_3_3_IV0)), delta.calculateMetadataVersionChange(MV_3_3_IV0_RECORD)); + } + + @Test + public void testReplayMetadataVersionChangeRecord() throws Throwable { + FeaturesDelta delta = new FeaturesDelta(IMAGE1); + delta.replay(MV_LATEST_RECORD); + assertEquals(MetadataVersion.latest(), delta.apply().metadataVersion()); + delta.replay(MV_3_3_IV0_RECORD); + assertEquals(IBP_3_3_IV0, delta.apply().metadataVersion()); + } } diff --git a/metadata/src/test/java/org/apache/kafka/image/ImageDowngradeTest.java b/metadata/src/test/java/org/apache/kafka/image/ImageDowngradeTest.java index 5a7c2e6c0b605..ce6cfe3da4cf9 100644 --- a/metadata/src/test/java/org/apache/kafka/image/ImageDowngradeTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/ImageDowngradeTest.java @@ -133,9 +133,11 @@ private static void writeWithExpectedLosses( List expectedOutputs ) { MockLossConsumer lossConsumer = new MockLossConsumer(metadataVersion); - MetadataDelta delta = new MetadataDelta(MetadataImage.EMPTY); + MetadataDelta delta = new MetadataDelta.Builder(). + setMetadataVersion(findMetadataVersion(inputs)). + build(); RecordTestUtils.replayAll(delta, inputs); - MetadataImage image = delta.apply(); + MetadataImage image = delta.apply(MetadataProvenance.EMPTY); RecordListWriter writer = new RecordListWriter(); image.write(writer, new ImageWriterOptions.Builder(). setRawMetadataVersion(metadataVersion). @@ -144,4 +146,16 @@ private static void writeWithExpectedLosses( assertEquals(expectedLosses, lossConsumer.losses, "Failed to get expected metadata losses."); assertEquals(expectedOutputs, writer.records(), "Failed to get expected output records."); } + + private static MetadataVersion findMetadataVersion(List records) { + for (ApiMessageAndVersion messageAndVersion : records) { + if (messageAndVersion.message() instanceof FeatureLevelRecord) { + FeatureLevelRecord record = (FeatureLevelRecord) messageAndVersion.message(); + if (record.name().equals(MetadataVersion.FEATURE_NAME)) { + return MetadataVersion.fromFeatureLevel(record.featureLevel()); + } + } + } + return MetadataVersion.MINIMUM_KRAFT_VERSION; + } } diff --git a/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java b/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java index 290214369ccf6..d15e88fb20000 100644 --- a/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java +++ b/metadata/src/test/java/org/apache/kafka/image/MetadataImageTest.java @@ -20,7 +20,6 @@ import org.apache.kafka.image.writer.ImageWriterOptions; import org.apache.kafka.image.writer.RecordListWriter; import org.apache.kafka.metadata.RecordTestUtils; -import org.apache.kafka.raft.OffsetAndEpoch; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -37,7 +36,7 @@ public class MetadataImageTest { static { IMAGE1 = new MetadataImage( - new OffsetAndEpoch(100, 4), + new MetadataProvenance(100, 4, 2000), FeaturesImageTest.IMAGE1, ClusterImageTest.IMAGE1, TopicsImageTest.IMAGE1, @@ -46,17 +45,19 @@ public class MetadataImageTest { ProducerIdsImageTest.IMAGE1, AclsImageTest.IMAGE1); - DELTA1 = new MetadataDelta(IMAGE1); - RecordTestUtils.replayAll(DELTA1, 200, 5, FeaturesImageTest.DELTA1_RECORDS); - RecordTestUtils.replayAll(DELTA1, 200, 5, ClusterImageTest.DELTA1_RECORDS); - RecordTestUtils.replayAll(DELTA1, 200, 5, TopicsImageTest.DELTA1_RECORDS); - RecordTestUtils.replayAll(DELTA1, 200, 5, ConfigurationsImageTest.DELTA1_RECORDS); - RecordTestUtils.replayAll(DELTA1, 200, 5, ClientQuotasImageTest.DELTA1_RECORDS); - RecordTestUtils.replayAll(DELTA1, 200, 5, ProducerIdsImageTest.DELTA1_RECORDS); - RecordTestUtils.replayAll(DELTA1, 200, 5, AclsImageTest.DELTA1_RECORDS); + DELTA1 = new MetadataDelta.Builder(). + setImage(IMAGE1). + build(); + RecordTestUtils.replayAll(DELTA1, FeaturesImageTest.DELTA1_RECORDS); + RecordTestUtils.replayAll(DELTA1, ClusterImageTest.DELTA1_RECORDS); + RecordTestUtils.replayAll(DELTA1, TopicsImageTest.DELTA1_RECORDS); + RecordTestUtils.replayAll(DELTA1, ConfigurationsImageTest.DELTA1_RECORDS); + RecordTestUtils.replayAll(DELTA1, ClientQuotasImageTest.DELTA1_RECORDS); + RecordTestUtils.replayAll(DELTA1, ProducerIdsImageTest.DELTA1_RECORDS); + RecordTestUtils.replayAll(DELTA1, AclsImageTest.DELTA1_RECORDS); IMAGE2 = new MetadataImage( - new OffsetAndEpoch(200, 5), + new MetadataProvenance(200, 5, 4000), FeaturesImageTest.IMAGE2, ClusterImageTest.IMAGE2, TopicsImageTest.IMAGE2, @@ -78,7 +79,7 @@ public void testImage1RoundTrip() throws Throwable { @Test public void testApplyDelta1() throws Throwable { - assertEquals(IMAGE2, DELTA1.apply()); + assertEquals(IMAGE2, DELTA1.apply(IMAGE2.provenance())); } @Test @@ -88,13 +89,14 @@ public void testImage2RoundTrip() throws Throwable { private void testToImageAndBack(MetadataImage image) throws Throwable { RecordListWriter writer = new RecordListWriter(); - image.write(writer, new ImageWriterOptions.Builder().build()); - MetadataDelta delta = new MetadataDelta(MetadataImage.EMPTY); - RecordTestUtils.replayAll(delta, - image.highestOffsetAndEpoch().offset(), - image.highestOffsetAndEpoch().epoch(), - writer.records()); - MetadataImage nextImage = delta.apply(); + image.write(writer, new ImageWriterOptions.Builder(). + setMetadataVersion(image.features().metadataVersion()). + build()); + MetadataDelta delta = new MetadataDelta.Builder(). + setMetadataVersion(image.features().metadataVersion()). + build(); + RecordTestUtils.replayAll(delta, writer.records()); + MetadataImage nextImage = delta.apply(image.provenance()); assertEquals(image, nextImage); } } diff --git a/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java b/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java new file mode 100644 index 0000000000000..6b26c1c60e2a6 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java @@ -0,0 +1,527 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.loader; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.metadata.FeatureLevelRecord; +import org.apache.kafka.common.metadata.RemoveTopicRecord; +import org.apache.kafka.common.metadata.TopicRecord; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.image.MetadataDelta; +import org.apache.kafka.image.MetadataImage; +import org.apache.kafka.image.MetadataProvenance; +import org.apache.kafka.image.MetadataVersionChange; +import org.apache.kafka.image.publisher.MetadataPublisher; +import org.apache.kafka.raft.Batch; +import org.apache.kafka.raft.BatchReader; +import org.apache.kafka.raft.OffsetAndEpoch; +import org.apache.kafka.server.common.ApiMessageAndVersion; +import org.apache.kafka.server.common.MetadataVersion; +import org.apache.kafka.server.fault.MockFaultHandler; +import org.apache.kafka.snapshot.SnapshotReader; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.OptionalLong; +import java.util.concurrent.ExecutionException; + +import static java.util.Arrays.asList; +import static org.apache.kafka.server.common.MetadataVersion.IBP_3_3_IV1; +import static org.apache.kafka.server.common.MetadataVersion.IBP_3_3_IV2; +import static org.apache.kafka.server.common.MetadataVersion.IBP_3_3_IV3; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +@Timeout(value = 40) +public class MetadataLoaderTest { + @Test + public void testCreateAndClose() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("testCreateAndClose"); + try (MetadataLoader loader = new MetadataLoader.Builder(). + setFaultHandler(faultHandler). + build()) { + assertEquals(-1L, loader.lastAppliedOffset()); + } + faultHandler.maybeRethrowFirstException(); + } + + static class MockPublisher implements MetadataPublisher { + MetadataDelta latestDelta = null; + MetadataImage latestImage = null; + LogDeltaManifest latestLogDeltaManifest = null; + SnapshotManifest latestSnapshotManifest = null; + MetadataDelta latestPreVersionChangeDelta = null; + MetadataImage latestPreVersionChangeImage = null; + PreVersionChangeManifest latestPreVersionChangeManifest = null; + boolean closed = false; + + @Override + public String name() { + return "MockPublisher"; + } + + @Override + public void publishSnapshot( + MetadataDelta delta, + MetadataImage newImage, + SnapshotManifest manifest + ) { + latestDelta = delta; + latestImage = newImage; + latestSnapshotManifest = manifest; + } + + @Override + public void publishPreVersionChangeImage( + MetadataDelta delta, + MetadataImage preVersionChangeImage, + PreVersionChangeManifest manifest + ) { + latestPreVersionChangeDelta = delta; + latestPreVersionChangeImage = preVersionChangeImage; + latestPreVersionChangeManifest = manifest; + } + + @Override + public void publishLogDelta( + MetadataDelta delta, + MetadataImage newImage, + LogDeltaManifest manifest + ) { + latestDelta = delta; + latestImage = newImage; + latestLogDeltaManifest = manifest; + } + + @Override + public void close() throws Exception { + closed = true; + } + } + + /** + * Install 2 publishers and check that the publishers that were installed are closed when the + * loader is closed. + */ + @Test + public void testInstallPublishers() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("testInstallPublishers"); + List publishers = asList(new MockPublisher(), + new MockPublisher(), + new MockPublisher()); + try (MetadataLoader loader = new MetadataLoader.Builder(). + setFaultHandler(faultHandler). + build()) { + loader.installPublishers(publishers.subList(0, 2)).get(); + } + assertTrue(publishers.get(0).closed); + assertEquals(MetadataImage.EMPTY, publishers.get(0).latestImage); + assertTrue(publishers.get(1).closed); + assertEquals(MetadataImage.EMPTY, publishers.get(1).latestImage); + assertFalse(publishers.get(2).closed); + assertNull(publishers.get(2).latestImage); + faultHandler.maybeRethrowFirstException(); + } + + /** + * Test that a publisher cannot be installed more than once. + */ + @Test + public void testPublisherCannotBeInstalledMoreThanOnce() throws Exception { + MockFaultHandler faultHandler = + new MockFaultHandler("testPublisherCannotBeInstalledMoreThanOnce"); + MockPublisher publisher = new MockPublisher(); + try (MetadataLoader loader = new MetadataLoader.Builder(). + setFaultHandler(faultHandler). + build()) { + loader.installPublishers(asList(publisher)).get(); + assertEquals("testPublisherCannotBeInstalledMoreThanOnce: Attempted to install " + + "publisher MockPublisher, which is already installed.", + assertThrows(ExecutionException.class, + () -> loader.installPublishers(asList(publisher)).get()). + getCause().getMessage()); + } + } + + static class MockSnapshotReader implements SnapshotReader { + private final MetadataProvenance provenance; + private final Iterator> iterator; + private MockTime time = null; + boolean closed = false; + + static MockSnapshotReader fromRecordLists( + MetadataProvenance provenance, + List> lists + ) { + List> batches = new ArrayList<>(); + lists.forEach(records -> batches.add(Batch.data( + provenance.offset(), + provenance.epoch(), + provenance.lastContainedLogTimeMs(), + 0, + records))); + return new MockSnapshotReader(provenance, batches); + } + + MockSnapshotReader( + MetadataProvenance provenance, + List> batches + ) { + this.provenance = provenance; + this.iterator = batches.iterator(); + } + + MockSnapshotReader setTime(MockTime time) { + this.time = time; + return this; + } + + @Override + public OffsetAndEpoch snapshotId() { + return provenance.offsetAndEpoch(); + } + + @Override + public long lastContainedLogOffset() { + return provenance.offset(); + } + + @Override + public int lastContainedLogEpoch() { + return provenance.epoch(); + } + + @Override + public long lastContainedLogTimestamp() { + return provenance.lastContainedLogTimeMs(); + } + + @Override + public void close() { + closed = true; + } + + @Override + public boolean hasNext() { + if (time != null) time.sleep(1); + return iterator.hasNext(); + } + + @Override + public Batch next() { + if (time != null) time.sleep(1); + return iterator.next(); + } + } + + /** + * Install 2 publishers and remove one. + */ + @Test + public void testRemovePublisher() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("testRemovePublisher"); + List publishers = asList(new MockPublisher(), + new MockPublisher(), + new MockPublisher()); + try (MetadataLoader loader = new MetadataLoader.Builder(). + setFaultHandler(faultHandler). + build()) { + loader.installPublishers(publishers.subList(0, 2)).get(); + loader.removeAndClosePublisher(publishers.get(1)).get(); + MockSnapshotReader snapshotReader = MockSnapshotReader.fromRecordLists( + new MetadataProvenance(100, 50, 2000), + asList(asList(new ApiMessageAndVersion( + new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(IBP_3_3_IV2.featureLevel()), (short) 0)))); + assertFalse(snapshotReader.closed); + loader.handleSnapshot(snapshotReader); + loader.waitForAllEventsToBeHandled(); + assertTrue(snapshotReader.closed); + loader.removeAndClosePublisher(publishers.get(0)).get(); + } + assertTrue(publishers.get(0).closed); + assertEquals(IBP_3_3_IV2, + publishers.get(0).latestImage.features().metadataVersion()); + assertTrue(publishers.get(1).closed); + assertEquals(MetadataImage.EMPTY, publishers.get(1).latestImage); + assertFalse(publishers.get(2).closed); + assertNull(publishers.get(2).latestImage); + faultHandler.maybeRethrowFirstException(); + } + + /** + * Test loading a snapshot with 0 records. + */ + @Test + public void testLoadEmptySnapshot() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("testLoadEmptySnapshot"); + MockTime time = new MockTime(); + List publishers = asList(new MockPublisher()); + try (MetadataLoader loader = new MetadataLoader.Builder(). + setFaultHandler(faultHandler). + setTime(time). + build()) { + loader.installPublishers(publishers).get(); + MockSnapshotReader snapshotReader = new MockSnapshotReader( + new MetadataProvenance(200, 100, 4000), + asList(Batch.control(200, 100, 4000, 10, 200))). + setTime(time); + loader.handleSnapshot(snapshotReader); + loader.waitForAllEventsToBeHandled(); + assertEquals(200L, loader.lastAppliedOffset()); + assertEquals(new SnapshotManifest(new MetadataProvenance(200, 100, 4000), 3000000L), + publishers.get(0).latestSnapshotManifest); + } + assertTrue(publishers.get(0).closed); + assertEquals(MetadataVersion.IBP_3_0_IV1, + publishers.get(0).latestImage.features().metadataVersion()); + assertTrue(publishers.get(0).latestImage.isEmpty()); + faultHandler.maybeRethrowFirstException(); + } + + static class MockBatchReader implements BatchReader { + private final long baseOffset; + private final Iterator> iterator; + private boolean closed = false; + private MockTime time = null; + + static Batch newBatch( + long batchBaseOffset, + int epoch, + List records + ) { + return Batch.data(batchBaseOffset, epoch, 0, 0, records); + } + + MockBatchReader( + long baseOffset, + List> batches + ) { + this.baseOffset = baseOffset; + this.iterator = batches.iterator(); + } + + private MockBatchReader setTime(MockTime time) { + this.time = time; + return this; + } + + @Override + public long baseOffset() { + return baseOffset; + } + + @Override + public OptionalLong lastOffset() { + return OptionalLong.empty(); + } + + @Override + public void close() { + this.closed = true; + } + + @Override + public boolean hasNext() { + if (time != null) time.sleep(1); + return iterator.hasNext(); + } + + @Override + public Batch next() { + if (time != null) time.sleep(1); + return iterator.next(); + } + } + + /** + * Test loading a batch with 0 records. + */ + @Test + public void testLoadEmptyBatch() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("testLoadEmptyBatch"); + MockTime time = new MockTime(); + List publishers = asList(new MockPublisher()); + try (MetadataLoader loader = new MetadataLoader.Builder(). + setFaultHandler(faultHandler). + setTime(time). + build()) { + loader.installPublishers(publishers).get(); + MockBatchReader batchReader = new MockBatchReader(300, asList( + Batch.control(300, 100, 4000, 10, 400))). + setTime(time); + loader.handleCommit(batchReader); + loader.waitForAllEventsToBeHandled(); + assertTrue(batchReader.closed); + assertEquals(400L, loader.lastAppliedOffset()); + } + assertTrue(publishers.get(0).closed); + assertEquals(new LogDeltaManifest(new MetadataProvenance(400, 100, 4000), 1, 3000000L, 10), + publishers.get(0).latestLogDeltaManifest); + assertEquals(MetadataVersion.IBP_3_0_IV1, + publishers.get(0).latestImage.features().metadataVersion()); + assertTrue(publishers.get(0).latestImage.isEmpty()); + faultHandler.maybeRethrowFirstException(); + } + + /** + * Test that the lastAppliedOffset moves forward as expected. + */ + @Test + public void testLastAppliedOffset() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("testLastAppliedOffset"); + List publishers = asList(new MockPublisher(), + new MockPublisher()); + try (MetadataLoader loader = new MetadataLoader.Builder(). + setFaultHandler(faultHandler). + build()) { + loader.installPublishers(publishers).get(); + loader.handleSnapshot(MockSnapshotReader.fromRecordLists( + new MetadataProvenance(200, 100, 4000), asList( + asList(new ApiMessageAndVersion(new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(IBP_3_3_IV1.featureLevel()), (short) 0)), + asList(new ApiMessageAndVersion(new TopicRecord(). + setName("foo"). + setTopicId(Uuid.fromString("Uum7sfhHQP-obSvfywmNUA")), (short) 0)) + ))); + loader.waitForAllEventsToBeHandled(); + assertEquals(200L, loader.lastAppliedOffset()); + loader.handleCommit(new MockBatchReader(201, asList( + MockBatchReader.newBatch(201, 100, asList( + new ApiMessageAndVersion(new RemoveTopicRecord(). + setTopicId(Uuid.fromString("Uum7sfhHQP-obSvfywmNUA")), (short) 0)))))); + loader.waitForAllEventsToBeHandled(); + assertEquals(201L, loader.lastAppliedOffset()); + } + for (int i = 0; i < 2; i++) { + assertTrue(publishers.get(i).closed); + assertTrue(publishers.get(i).closed); + assertEquals(IBP_3_3_IV1, + publishers.get(i).latestImage.features().metadataVersion()); + } + faultHandler.maybeRethrowFirstException(); + } + + /** + * Test metadata version changes + */ + @Test + public void testMetadataVersionChange() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("testMetadataVersionChange"); + MockTime time = new MockTime(); + MockPublisher publisher = new MockPublisher(); + try (MetadataLoader loader = new MetadataLoader.Builder(). + setFaultHandler(faultHandler). + setTime(time). + build()) { + // Starting with an empty metadata image and loading a snapshot should not trigger + // the publishPreVersionChangeImage callbacks. + loader.installPublishers(Arrays.asList(publisher)).get(); + loader.handleSnapshot(MockSnapshotReader.fromRecordLists( + new MetadataProvenance(200, 100, 4000), asList( + asList(new ApiMessageAndVersion(new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(IBP_3_3_IV1.featureLevel()), (short) 0)), + asList(new ApiMessageAndVersion(new TopicRecord(). + setName("foo"). + setTopicId(Uuid.fromString("Uum7sfhHQP-obSvfywmNUA")), (short) 0)) + )).setTime(time)); + loader.waitForAllEventsToBeHandled(); + assertEquals(200L, loader.lastAppliedOffset()); + assertNull(publisher.latestPreVersionChangeDelta); + assertNull(publisher.latestPreVersionChangeImage); + assertNull(publisher.latestPreVersionChangeManifest); + + // Loading a snapshot with a new metadata version should trigger the + // publishPreVersionChangeImage callbacks. + loader.handleSnapshot(MockSnapshotReader.fromRecordLists( + new MetadataProvenance(201, 101, 5000), asList( + asList(new ApiMessageAndVersion(new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(IBP_3_3_IV2.featureLevel()), (short) 0)), + asList(new ApiMessageAndVersion(new TopicRecord(). + setName("foo"). + setTopicId(Uuid.fromString("Uum7sfhHQP-obSvfywmNUA")), (short) 0)) + )).setTime(time)); + loader.waitForAllEventsToBeHandled(); + assertEquals(201L, loader.lastAppliedOffset()); + assertNotNull(publisher.latestPreVersionChangeDelta); + assertNotNull(publisher.latestPreVersionChangeImage); + assertEquals(IBP_3_3_IV1, publisher.latestPreVersionChangeImage.features().metadataVersion()); + assertEquals(new PreVersionChangeManifest( + new MetadataProvenance(200, 100, 4000), + new MetadataVersionChange(IBP_3_3_IV1, IBP_3_3_IV2)), + publisher.latestPreVersionChangeManifest); + + // Loading a batch of log messages with a new metadata version should also trigger the + // publishPreVersionChangeImage callbacks. + loader.handleCommit(new MockBatchReader(203, asList( + MockBatchReader.newBatch(203, 102, asList( + new ApiMessageAndVersion(new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(IBP_3_3_IV3.featureLevel()), (short) 0), + new ApiMessageAndVersion(new TopicRecord(). + setName("bar"). + setTopicId(Uuid.fromString("aN0Qt3hRQgiaxkx_raX92Q")), (short) 0))))). + setTime(time)); + loader.waitForAllEventsToBeHandled(); + assertEquals(204L, loader.lastAppliedOffset()); + assertTrue(publisher.latestPreVersionChangeDelta.topicsDelta().changedTopics(). + containsKey(Uuid.fromString("aN0Qt3hRQgiaxkx_raX92Q"))); + assertEquals(IBP_3_3_IV2, publisher.latestPreVersionChangeImage.features().metadataVersion()); + assertEquals(new PreVersionChangeManifest( + new MetadataProvenance(201, 101, 5000), + new MetadataVersionChange(IBP_3_3_IV2, IBP_3_3_IV3)), + publisher.latestPreVersionChangeManifest); + + + // Loading a batch of log messages with a new metadata version should also trigger the + // publishPreVersionChangeImage callbacks. + loader.handleCommit(new MockBatchReader(203, asList( + Batch.data(206, 103, 6000, 20, asList( + new ApiMessageAndVersion(new TopicRecord(). + setName("quux"). + setTopicId(Uuid.fromString("Q1gR0H5AQsmU_Ojh66TnAg")), (short) 0), + new ApiMessageAndVersion(new FeatureLevelRecord(). + setName(MetadataVersion.FEATURE_NAME). + setFeatureLevel(IBP_3_3_IV2.featureLevel()), (short) 0))))). + setTime(time)); + loader.waitForAllEventsToBeHandled(); + assertEquals(207L, loader.lastAppliedOffset()); + assertNotNull(publisher.latestPreVersionChangeDelta); + assertNotNull(publisher.latestPreVersionChangeImage); + assertEquals(IBP_3_3_IV3, publisher.latestPreVersionChangeImage.features().metadataVersion()); + assertEquals(new PreVersionChangeManifest( + new MetadataProvenance(206, 103, 6000), + new MetadataVersionChange(IBP_3_3_IV3, IBP_3_3_IV2)), + publisher.latestPreVersionChangeManifest); + } + faultHandler.maybeRethrowFirstException(); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotEmitterTest.java b/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotEmitterTest.java new file mode 100644 index 0000000000000..9bdcedd87d77d --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotEmitterTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.publisher; + +import org.apache.kafka.image.MetadataImageTest; +import org.apache.kafka.raft.LeaderAndEpoch; +import org.apache.kafka.raft.OffsetAndEpoch; +import org.apache.kafka.raft.RaftClient; +import org.apache.kafka.server.common.ApiMessageAndVersion; +import org.apache.kafka.snapshot.SnapshotWriter; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.OptionalLong; +import java.util.concurrent.CompletableFuture; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +@Timeout(value = 40) +public class SnapshotEmitterTest { + static class MockRaftClient implements RaftClient { + Map writers = new HashMap<>(); + + @Override + public void initialize() { + // nothing to do + } + + @Override + public void register(Listener listener) { + // nothing to do + } + + @Override + public void unregister(Listener listener) { + // nothing to do + } + + @Override + public OptionalLong highWatermark() { + return OptionalLong.empty(); + } + + @Override + public LeaderAndEpoch leaderAndEpoch() { + return LeaderAndEpoch.UNKNOWN; + } + + @Override + public OptionalInt nodeId() { + return OptionalInt.empty(); + } + + @Override + public long scheduleAppend(int epoch, List records) { + return 0; + } + + @Override + public long scheduleAtomicAppend(int epoch, List records) { + return 0; + } + + @Override + public CompletableFuture shutdown(int timeoutMs) { + return CompletableFuture.completedFuture(null); + } + + @Override + public void resign(int epoch) { + // nothing to do + } + + @Override + public Optional> createSnapshot( + long committedOffset, + int committedEpoch, + long lastContainedLogTime + ) { + OffsetAndEpoch snapshotId = new OffsetAndEpoch(committedOffset, committedEpoch); + if (writers.containsKey(snapshotId)) { + return Optional.empty(); + } + MockSnapshotWriter writer = new MockSnapshotWriter(snapshotId); + writers.put(snapshotId, writer); + return Optional.of(writer); + } + + @Override + public void close() throws Exception { + // nothing to do + } + } + + static class MockSnapshotWriter implements SnapshotWriter { + private final OffsetAndEpoch snapshotId; + private boolean frozen = false; + private boolean closed = false; + private final List> batches; + + MockSnapshotWriter(OffsetAndEpoch snapshotId) { + this.snapshotId = snapshotId; + this.batches = new ArrayList<>(); + } + + @Override + public OffsetAndEpoch snapshotId() { + return snapshotId; + } + + @Override + public long lastContainedLogOffset() { + return snapshotId.offset(); + } + + @Override + public int lastContainedLogEpoch() { + return snapshotId.epoch(); + } + + @Override + public boolean isFrozen() { + return frozen; + } + + @Override + public void append(List records) { + batches.add(records); + } + + List> batches() { + List> results = new ArrayList<>(); + batches.forEach(batch -> results.add(new ArrayList<>(batch))); + return results; + } + + @Override + public void freeze() { + frozen = true; + } + + @Override + public void close() { + closed = true; + } + + boolean isClosed() { + return closed; + } + } + + @Test + public void testEmit() throws Exception { + MockRaftClient mockRaftClient = new MockRaftClient(); + SnapshotEmitter emitter = new SnapshotEmitter.Builder(). + setBatchSize(2). + setRaftClient(mockRaftClient). + build(); + emitter.maybeEmit(MetadataImageTest.IMAGE1); + MockSnapshotWriter writer = mockRaftClient.writers.get( + MetadataImageTest.IMAGE1.highestOffsetAndEpoch()); + assertNotNull(writer); + assertEquals(MetadataImageTest.IMAGE1.highestOffsetAndEpoch().offset(), + writer.lastContainedLogOffset()); + assertEquals(MetadataImageTest.IMAGE1.highestOffsetAndEpoch().epoch(), + writer.lastContainedLogEpoch()); + assertTrue(writer.isFrozen()); + assertTrue(writer.isClosed()); + + // Second call to emit does nothing because we already have a snapshot at that offset and epoch. + emitter.maybeEmit(MetadataImageTest.IMAGE1); + assertEquals(1, mockRaftClient.writers.size()); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotGeneratorTest.java b/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotGeneratorTest.java new file mode 100644 index 0000000000000..1c21c06e6f640 --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/image/publisher/SnapshotGeneratorTest.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.publisher; + +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.image.MetadataDelta; +import org.apache.kafka.image.MetadataImage; +import org.apache.kafka.image.MetadataProvenance; +import org.apache.kafka.image.MetadataVersionChange; +import org.apache.kafka.image.loader.LogDeltaManifest; +import org.apache.kafka.image.loader.PreVersionChangeManifest; +import org.apache.kafka.metadata.RecordTestUtils; +import org.apache.kafka.server.fault.FaultHandlerException; +import org.apache.kafka.server.fault.MockFaultHandler; +import org.apache.kafka.test.TestUtils; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.apache.kafka.server.common.MetadataVersion.IBP_3_3_IV2; +import static org.apache.kafka.server.common.MetadataVersion.IBP_3_3_IV3; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +@Timeout(value = 40) +public class SnapshotGeneratorTest { + static class MockEmitter implements SnapshotGenerator.Emitter { + private final CountDownLatch latch = new CountDownLatch(1); + private final List images = new ArrayList<>(); + private RuntimeException problem = null; + + MockEmitter setReady() { + latch.countDown(); + return this; + } + + synchronized MockEmitter setProblem(RuntimeException problem) { + this.problem = problem; + return this; + } + + @Override + public synchronized void maybeEmit(MetadataImage image) { + RuntimeException currentProblem = problem; + if (currentProblem != null) { + throw currentProblem; + } + try { + latch.await(); + } catch (Throwable e) { + throw new RuntimeException(e); + } + images.add(image); + } + + synchronized List images() { + return new ArrayList<>(images); + } + } + + private final static MetadataDelta TEST_DELTA; + + static { + TEST_DELTA = new MetadataDelta.Builder(). + setImage(MetadataImage.EMPTY). + setMetadataVersion(IBP_3_3_IV2). + build(); + TEST_DELTA.replay(RecordTestUtils.testRecord(0).message()); + } + + private final static MetadataImage TEST_IMAGE = TEST_DELTA.apply(MetadataProvenance.EMPTY); + + @Test + public void testCreateSnapshot() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("SnapshotGenerator"); + MockEmitter emitter = new MockEmitter(); + try (SnapshotGenerator generator = new SnapshotGenerator.Builder(emitter). + setFaultHandler(faultHandler). + setMinBytesSinceLastSnapshot(200). + setMaxTimeSinceLastSnapshotNs(TimeUnit.DAYS.toNanos(10)). + build()) { + // Publish a log delta batch. This one will not trigger a snapshot yet. + generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, + new LogDeltaManifest(MetadataProvenance.EMPTY, 1, 100, 100)); + // Publish a log delta batch. This will trigger a snapshot. + generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, + new LogDeltaManifest(MetadataProvenance.EMPTY, 1, 100, 100)); + // Pre-version change images are always queued for writing, even if there is another + // image in progress. + for (int i = 0; i < 2; i++) { + generator.publishPreVersionChangeImage(TEST_DELTA, TEST_IMAGE, + new PreVersionChangeManifest(MetadataProvenance.EMPTY, + new MetadataVersionChange(IBP_3_3_IV2, IBP_3_3_IV3))); + } + // Publish a log delta batch. This one will be ignored because there are other images + // queued for writing. + generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, + new LogDeltaManifest(MetadataProvenance.EMPTY, 1, 100, 2000)); + assertEquals(Collections.emptyList(), emitter.images()); + emitter.setReady(); + } + assertEquals(Arrays.asList(TEST_IMAGE, TEST_IMAGE, TEST_IMAGE), emitter.images()); + faultHandler.maybeRethrowFirstException(); + } + + @Test + public void testSnapshotsDisabled() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("SnapshotGenerator"); + MockEmitter emitter = new MockEmitter().setReady(); + try (SnapshotGenerator generator = new SnapshotGenerator.Builder(emitter). + setFaultHandler(faultHandler). + setMinBytesSinceLastSnapshot(1). + setMaxTimeSinceLastSnapshotNs(0). + build()) { + generator.disable("we are testing disable()"); + // No snapshots are generated because snapshots are disabled. + generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, + new LogDeltaManifest(MetadataProvenance.EMPTY, 1, 100, 100)); + generator.publishPreVersionChangeImage(TEST_DELTA, TEST_IMAGE, + new PreVersionChangeManifest(MetadataProvenance.EMPTY, + new MetadataVersionChange(IBP_3_3_IV2, IBP_3_3_IV3))); + } + assertEquals(Collections.emptyList(), emitter.images()); + faultHandler.maybeRethrowFirstException(); + } + + @Test + public void testTimeBasedSnapshots() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("SnapshotGenerator"); + MockEmitter emitter = new MockEmitter().setReady(); + MockTime mockTime = new MockTime(); + try (SnapshotGenerator generator = new SnapshotGenerator.Builder(emitter). + setTime(mockTime). + setFaultHandler(faultHandler). + setMinBytesSinceLastSnapshot(200). + setMaxTimeSinceLastSnapshotNs(TimeUnit.MINUTES.toNanos(30)). + build()) { + // This image isn't published yet. + generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, + new LogDeltaManifest(MetadataProvenance.EMPTY, 1, 100, 50)); + assertEquals(Collections.emptyList(), emitter.images()); + mockTime.sleep(TimeUnit.MINUTES.toNanos(40)); + // Next image is published because of the time delay. + generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, + new LogDeltaManifest(MetadataProvenance.EMPTY, 1, 100, 50)); + TestUtils.waitForCondition(() -> emitter.images().size() == 1, "images.size == 1"); + // bytesSinceLastSnapshot was reset to 0 by the previous snapshot, + // so this does not trigger a new snapshot. + generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, + new LogDeltaManifest(MetadataProvenance.EMPTY, 1, 100, 150)); + } + assertEquals(Arrays.asList(TEST_IMAGE), emitter.images()); + faultHandler.maybeRethrowFirstException(); + } + + @Test + public void testEmitterProblem() throws Exception { + MockFaultHandler faultHandler = new MockFaultHandler("SnapshotGenerator"); + MockEmitter emitter = new MockEmitter().setProblem(new RuntimeException("oops")); + try (SnapshotGenerator generator = new SnapshotGenerator.Builder(emitter). + setFaultHandler(faultHandler). + setMinBytesSinceLastSnapshot(200). + build()) { + for (int i = 0; i < 2; i++) { + generator.publishLogDelta(TEST_DELTA, TEST_IMAGE, + new LogDeltaManifest(MetadataProvenance.EMPTY, 1, 10000, 50000)); + } + } + assertEquals(Collections.emptyList(), emitter.images()); + assertNotNull(faultHandler.firstException()); + assertEquals(FaultHandlerException.class, faultHandler.firstException().getClass()); + assertEquals("SnapshotGenerator: KRaft snapshot file generation error: oops", + faultHandler.firstException().getMessage()); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/image/writer/ImageReWriterTest.java b/metadata/src/test/java/org/apache/kafka/image/writer/ImageReWriterTest.java new file mode 100644 index 0000000000000..640924fe076be --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/image/writer/ImageReWriterTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.writer; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.metadata.TopicRecord; +import org.apache.kafka.image.MetadataDelta; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import static org.apache.kafka.metadata.RecordTestUtils.testRecord; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + + +@Timeout(value = 40) +public class ImageReWriterTest { + @Test + public void testWrite() { + MetadataDelta delta = new MetadataDelta.Builder().build(); + ImageReWriter writer = new ImageReWriter(delta); + writer.write(testRecord(0)); + writer.write(testRecord(1)); + writer.close(true); + assertEquals(2, delta.getOrCreateTopicsDelta().changedTopics().size()); + assertEquals(2, writer.image().topics().topicsById().size()); + } + + @Test + public void testCloseWithoutFreeze() { + MetadataDelta delta = new MetadataDelta.Builder().build(); + ImageReWriter writer = new ImageReWriter(delta); + writer.close(); + assertNull(writer.image()); + } + + @Test + public void testWriteAfterClose() { + MetadataDelta delta = new MetadataDelta.Builder().build(); + ImageReWriter writer = new ImageReWriter(delta); + writer.close(true); + assertThrows(ImageWriterClosedException.class, () -> + writer.write(0, new TopicRecord(). + setName("foo"). + setTopicId(Uuid.fromString("3B134hrsQgKtz8Sp6QBIfg")))); + } +} diff --git a/metadata/src/test/java/org/apache/kafka/image/writer/ImageWriterOptionsTest.java b/metadata/src/test/java/org/apache/kafka/image/writer/ImageWriterOptionsTest.java new file mode 100644 index 0000000000000..07ddd4ff8cbba --- /dev/null +++ b/metadata/src/test/java/org/apache/kafka/image/writer/ImageWriterOptionsTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.image.writer; + +import org.apache.kafka.server.common.MetadataVersion; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + + +@Timeout(value = 40) +public class ImageWriterOptionsTest { + @Test + public void testDefaultLossHandler() { + ImageWriterOptions options = new ImageWriterOptions.Builder().build(); + assertEquals("stuff", assertThrows(UnwritableMetadataException.class, + () -> options.handleLoss("stuff")).loss()); + } + + @Test + public void testSetMetadataVersion() { + for (int i = MetadataVersion.MINIMUM_KRAFT_VERSION.ordinal(); + i < MetadataVersion.VERSIONS.length; + i++) { + MetadataVersion version = MetadataVersion.VERSIONS[i]; + ImageWriterOptions.Builder options = new ImageWriterOptions.Builder(). + setMetadataVersion(version); + if (i < MetadataVersion.MINIMUM_BOOTSTRAP_VERSION.ordinal()) { + assertEquals(MetadataVersion.MINIMUM_KRAFT_VERSION, options.metadataVersion()); + } else { + assertEquals(version, options.metadataVersion()); + } + } + } +} diff --git a/metadata/src/test/java/org/apache/kafka/metadata/RecordTestUtils.java b/metadata/src/test/java/org/apache/kafka/metadata/RecordTestUtils.java index 7fd2a29d5b13c..04da77b0fc327 100644 --- a/metadata/src/test/java/org/apache/kafka/metadata/RecordTestUtils.java +++ b/metadata/src/test/java/org/apache/kafka/metadata/RecordTestUtils.java @@ -23,7 +23,6 @@ import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.ObjectSerializationCache; import org.apache.kafka.common.utils.ImplicitLinkedHashCollection; -import org.apache.kafka.image.MetadataDelta; import org.apache.kafka.raft.Batch; import org.apache.kafka.raft.BatchReader; import org.apache.kafka.raft.internals.MemoryBatchReader; @@ -57,14 +56,6 @@ public class RecordTestUtils { */ public static void replayAll(Object target, List recordsAndVersions) { - if (target instanceof MetadataDelta) { - MetadataDelta delta = (MetadataDelta) target; - replayAll(delta, - delta.image().highestOffsetAndEpoch().offset(), - delta.image().highestOffsetAndEpoch().epoch(), - recordsAndVersions); - return; - } for (ApiMessageAndVersion recordAndVersion : recordsAndVersions) { ApiMessage record = recordAndVersion.message(); try { @@ -96,26 +87,6 @@ public static void replayAll(Object target, } } - /** - * Replay a list of records to the metadata delta. - * - * @param delta the metadata delta on which to replay the records - * @param highestOffset highest offset from the list of records - * @param highestEpoch highest epoch from the list of records - * @param recordsAndVersions list of records - */ - public static void replayAll( - MetadataDelta delta, - long highestOffset, - int highestEpoch, - List recordsAndVersions - ) { - for (ApiMessageAndVersion recordAndVersion : recordsAndVersions) { - ApiMessage record = recordAndVersion.message(); - delta.replay(highestOffset, highestEpoch, record); - } - } - /** * Replay a list of record batches. * @@ -129,25 +100,6 @@ public static void replayAllBatches(Object target, } } - /** - * Replay a list of record batches to the metadata delta. - * - * @param delta the metadata delta on which to replay the records - * @param highestOffset highest offset from the list of record batches - * @param highestEpoch highest epoch from the list of record batches - * @param batches list of batches of records - */ - public static void replayAllBatches( - MetadataDelta delta, - long highestOffset, - int highestEpoch, - List> batches - ) { - for (List batch : batches) { - replayAll(delta, highestOffset, highestEpoch, batch); - } - } - /** * Materialize the output of an iterator into a set. * diff --git a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java index 6822ea72eb9d6..0edce4d818552 100644 --- a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java +++ b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java @@ -46,6 +46,7 @@ import org.slf4j.LoggerFactory; import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.IdentityHashMap; @@ -318,9 +319,9 @@ synchronized void addSnapshot(RawSnapshotReader newSnapshot) { } /** - * Returns the snapshot whos last offset is the committed offset. + * Returns the snapshot whose last offset is the committed offset. * - * If such snapshot doesn't exists, it waits until it does. + * If such snapshot doesn't exist, it waits until it does. */ synchronized RawSnapshotReader waitForSnapshot(long committedOffset) throws InterruptedException { while (true) { @@ -371,6 +372,20 @@ public SharedLogData setInitialMaxReadOffset(long initialMaxReadOffset) { public long initialMaxReadOffset() { return initialMaxReadOffset; } + + /** + * Return all records in the log as a list. + */ + public synchronized List allRecords() { + List allRecords = new ArrayList<>(); + for (LocalBatch batch : batches.values()) { + if (batch instanceof LocalRecordBatch) { + LocalRecordBatch recordBatch = (LocalRecordBatch) batch; + allRecords.addAll(recordBatch.records); + } + } + return allRecords; + } } private static class MetaLogListenerData { diff --git a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManagerTestEnv.java b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManagerTestEnv.java index 1693b62be1aec..d72b7557b48e0 100644 --- a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManagerTestEnv.java +++ b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManagerTestEnv.java @@ -140,6 +140,13 @@ private LocalLogManagerTestEnv( this.logManagers = newLogManagers; } + /** + * Return all records in the log as a list. + */ + public List allRecords() { + return shared.allRecords(); + } + /** * Append some records to the log. This method is meant to be called before the * controllers are started, to simulate a pre-existing metadata log. diff --git a/server-common/src/main/java/org/apache/kafka/queue/EventQueue.java b/server-common/src/main/java/org/apache/kafka/queue/EventQueue.java index c282917bb6ec0..99a08c3e1c2cc 100644 --- a/server-common/src/main/java/org/apache/kafka/queue/EventQueue.java +++ b/server-common/src/main/java/org/apache/kafka/queue/EventQueue.java @@ -248,6 +248,18 @@ default void beginShutdown(String source, Event cleanupEvent) { */ void beginShutdown(String source, Event cleanupEvent, long timeSpan, TimeUnit timeUnit); + /** + * @return The number of pending events in the queue. If an event is running, it is counted. + */ + int size(); + + /** + * @return True if there are no pending events in the queue. If an event is running, it is counted. + */ + default boolean isEmpty() { + return size() == 0; + } + /** * This method is used during unit tests where MockTime is in use. * It is used to alert the queue that the mock time has changed. diff --git a/server-common/src/main/java/org/apache/kafka/queue/KafkaEventQueue.java b/server-common/src/main/java/org/apache/kafka/queue/KafkaEventQueue.java index 05642f6bac8b5..b53339edfe337 100644 --- a/server-common/src/main/java/org/apache/kafka/queue/KafkaEventQueue.java +++ b/server-common/src/main/java/org/apache/kafka/queue/KafkaEventQueue.java @@ -147,6 +147,8 @@ void completeWithException(Throwable t) { } private class EventHandler implements Runnable { + private int size = 0; + /** * Event contexts indexed by tag. Events without a tag are not included here. */ @@ -195,13 +197,19 @@ private void handleEvents() throws InterruptedException { while (true) { if (toTimeout != null) { toTimeout.completeWithTimeout(); - toTimeout = null; } else if (toRun != null) { toRun.run(log); - toRun = null; } lock.lock(); try { + if (toTimeout != null) { + size--; + toTimeout = null; + } + if (toRun != null) { + size--; + toRun = null; + } long awaitNs = Long.MAX_VALUE; Map.Entry entry = deadlineMap.firstEntry(); if (entry != null) { @@ -272,6 +280,7 @@ Exception enqueue(EventContext eventContext, if (toRemove != null) { existingDeadlineNs = toRemove.deadlineNs; remove(toRemove); + size--; } } OptionalLong deadlineNs = deadlineNsCalculator.apply(existingDeadlineNs); @@ -311,6 +320,7 @@ Exception enqueue(EventContext eventContext, shouldSignal = true; } } + size++; if (shouldSignal) { cond.signal(); } @@ -326,6 +336,7 @@ void cancelDeferred(String tag) { EventContext eventContext = tagToEventContext.get(tag); if (eventContext != null) { remove(eventContext); + size--; } } finally { lock.unlock(); @@ -340,6 +351,15 @@ void wakeUp() { lock.unlock(); } } + + int size() { + lock.lock(); + try { + return size; + } finally { + lock.unlock(); + } + } } private final Time time; @@ -412,6 +432,11 @@ public void beginShutdown(String source, Event newCleanupEvent, } } + @Override + public int size() { + return eventHandler.size(); + } + @Override public void wakeup() { eventHandler.wakeUp(); diff --git a/server-common/src/main/java/org/apache/kafka/server/fault/ProcessExitingFaultHandler.java b/server-common/src/main/java/org/apache/kafka/server/fault/ProcessExitingFaultHandler.java index b7c0d241a2ad3..fd72a8b1c0cda 100644 --- a/server-common/src/main/java/org/apache/kafka/server/fault/ProcessExitingFaultHandler.java +++ b/server-common/src/main/java/org/apache/kafka/server/fault/ProcessExitingFaultHandler.java @@ -28,6 +28,15 @@ */ public class ProcessExitingFaultHandler implements FaultHandler { private static final Logger log = LoggerFactory.getLogger(ProcessExitingFaultHandler.class); + private final Runnable action; + + public ProcessExitingFaultHandler() { + this.action = () -> { }; + } + + public ProcessExitingFaultHandler(Runnable action) { + this.action = action; + } @Override public RuntimeException handleFault(String failureMessage, Throwable cause) { @@ -36,6 +45,11 @@ public RuntimeException handleFault(String failureMessage, Throwable cause) { } else { log.error("Encountered fatal fault: {}", failureMessage, cause); } + try { + action.run(); + } catch (Throwable e) { + log.error("Failed to run ProcessExitingFaultHandler action.", e); + } Exit.exit(1); return null; } diff --git a/server-common/src/test/java/org/apache/kafka/queue/KafkaEventQueueTest.java b/server-common/src/test/java/org/apache/kafka/queue/KafkaEventQueueTest.java index c3ee62a6ac26d..09910200a1f87 100644 --- a/server-common/src/test/java/org/apache/kafka/queue/KafkaEventQueueTest.java +++ b/server-common/src/test/java/org/apache/kafka/queue/KafkaEventQueueTest.java @@ -31,12 +31,14 @@ import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.test.TestUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; @Timeout(value = 60) @@ -240,4 +242,32 @@ public void handleException(Throwable e) { assertEquals(RejectedExecutionException.class, assertThrows( ExecutionException.class, () -> future.get()).getCause().getClass()); } -} + + @Test + public void testSize() throws Exception { + KafkaEventQueue queue = new KafkaEventQueue(Time.SYSTEM, new LogContext(), + "testEmpty"); + assertTrue(queue.isEmpty()); + CompletableFuture future = new CompletableFuture<>(); + queue.append(() -> future.get()); + assertFalse(queue.isEmpty()); + assertEquals(1, queue.size()); + queue.append(() -> future.get()); + assertEquals(2, queue.size()); + future.complete(null); + TestUtils.waitForCondition(() -> queue.isEmpty(), "Failed to see the queue become empty."); + queue.scheduleDeferred("later", + __ -> OptionalLong.of(Time.SYSTEM.nanoseconds() + TimeUnit.HOURS.toNanos(1)), + () -> { }); + assertFalse(queue.isEmpty()); + queue.scheduleDeferred("soon", + __ -> OptionalLong.of(Time.SYSTEM.nanoseconds() + TimeUnit.MILLISECONDS.toNanos(1)), + () -> { }); + assertFalse(queue.isEmpty()); + queue.cancelDeferred("later"); + queue.cancelDeferred("soon"); + assertTrue(queue.isEmpty()); + queue.close(); + assertTrue(queue.isEmpty()); + } +} \ No newline at end of file