From e894d84f27ece5680768c251d8ffbf9183269981 Mon Sep 17 00:00:00 2001 From: Justine Date: Wed, 15 Nov 2023 17:45:02 -0800 Subject: [PATCH 01/24] Redo verification path --- .../group/GroupMetadataManager.scala | 40 ++- .../transaction/TransactionStateManager.scala | 24 +- .../main/scala/kafka/server/KafkaApis.scala | 28 +- .../kafka/server/KafkaRequestHandler.scala | 8 +- .../scala/kafka/server/ReplicaManager.scala | 313 +++++++++--------- .../AbstractCoordinatorConcurrencyTest.scala | 18 +- .../group/GroupCoordinatorTest.scala | 16 +- .../group/GroupMetadataManagerTest.scala | 33 +- .../unit/kafka/server/KafkaApisTest.scala | 42 ++- .../kafka/server/ReplicaManagerTest.scala | 99 ++++-- 10 files changed, 397 insertions(+), 224 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala index 276602836fc21..b5b3ae78bcddd 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala @@ -26,7 +26,7 @@ import java.util.concurrent.locks.ReentrantLock import java.util.concurrent.ConcurrentHashMap import com.yammer.metrics.core.Gauge import kafka.common.OffsetAndMetadata -import kafka.server.{ReplicaManager, RequestLocal} +import kafka.server.{LogAppendResult, ReplicaManager, RequestLocal} import kafka.utils.CoreUtils.inLock import kafka.utils.Implicits._ import kafka.utils._ @@ -328,7 +328,7 @@ class GroupMetadataManager(brokerId: Int, records: Map[TopicPartition, MemoryRecords], requestLocal: RequestLocal, callback: Map[TopicPartition, PartitionResponse] => Unit, - transactionalId: String = null): Unit = { + preAppendErrors: Map[TopicPartition, LogAppendResult] = Map.empty): Unit = { // call replica manager to append the group message replicaManager.appendRecords( timeout = config.offsetCommitTimeoutMs.toLong, @@ -339,7 +339,7 @@ class GroupMetadataManager(brokerId: Int, delayedProduceLock = Some(group.lock), responseCallback = callback, requestLocal = requestLocal, - transactionalId = transactionalId) + preAppendErrors = preAppendErrors) } /** @@ -469,19 +469,35 @@ class GroupMetadataManager(brokerId: Int, responseCallback(commitStatus) } - if (isTxnOffsetCommit) { - group.inLock { - addProducerGroup(producerId, group.groupId) - group.prepareTxnOffsetCommit(producerId, offsetMetadata) + val transactionVerificationEntries = new ReplicaManager.TransactionVerificationEntries + def postVerificationCallback(newRequestLocal: RequestLocal) + (newlyVerifiedEntries: Map[TopicPartition, MemoryRecords], errorResults: Map[TopicPartition, LogAppendResult]): Unit = { + + if (isTxnOffsetCommit) { + group.inLock { + addProducerGroup(producerId, group.groupId) + group.prepareTxnOffsetCommit(producerId, offsetMetadata) + } + } else { + group.inLock { + group.prepareOffsetCommit(offsetMetadata) + } } + + appendForGroup(group, newlyVerifiedEntries ++ transactionVerificationEntries.verified, newRequestLocal, putCacheCallback, errorResults) + } + if (transactionalId != null ) { + replicaManager.appendRecordsWithVerification( + entriesPerPartition = entries, + transactionVerificationEntries = transactionVerificationEntries, + transactionalId = transactionalId, + requestLocal = requestLocal, + postVerificationCallback = postVerificationCallback + ) } else { - group.inLock { - group.prepareOffsetCommit(offsetMetadata) - } + postVerificationCallback(requestLocal)(entries, Map.empty) } - appendForGroup(group, entries, requestLocal, putCacheCallback, transactionalId) - case None => val commitStatus = offsetMetadata.map { case (topicIdPartition, _) => (topicIdPartition, Errors.NOT_COORDINATOR) diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala index d460edb029dda..10445dcb99443 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala @@ -281,12 +281,12 @@ class TransactionStateManager(brokerId: Int, inReadLock(stateLock) { replicaManager.appendRecords( - config.requestTimeoutMs, - TransactionLog.EnforcedRequiredAcks, + timeout = config.requestTimeoutMs, + requiredAcks = TransactionLog.EnforcedRequiredAcks, internalTopicsAllowed = true, origin = AppendOrigin.COORDINATOR, entriesPerPartition = Map(transactionPartition -> tombstoneRecords), - removeFromCacheCallback, + responseCallback = removeFromCacheCallback, requestLocal = RequestLocal.NoCaching) } } @@ -767,15 +767,15 @@ class TransactionStateManager(brokerId: Int, } if (append) { replicaManager.appendRecords( - newMetadata.txnTimeoutMs.toLong, - TransactionLog.EnforcedRequiredAcks, - internalTopicsAllowed = true, - origin = AppendOrigin.COORDINATOR, - recordsPerPartition, - updateCacheCallback, - requestLocal = requestLocal) - - trace(s"Appending new metadata $newMetadata for transaction id $transactionalId with coordinator epoch $coordinatorEpoch to the local transaction log") + timeout = newMetadata.txnTimeoutMs.toLong, + requiredAcks = TransactionLog.EnforcedRequiredAcks, + internalTopicsAllowed = true, + origin = AppendOrigin.COORDINATOR, + entriesPerPartition = recordsPerPartition, + responseCallback = updateCacheCallback, + requestLocal = requestLocal) + + trace(s"Appending new metadata $newMetadata for transaction id $transactionalId with coordinator epoch $coordinatorEpoch to the local transaction log") } } } diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index bd0959d40d2f8..3b5ead4e3e27a 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -675,22 +675,34 @@ class KafkaApis(val requestChannel: RequestChannel, } } - if (authorizedRequestInfo.isEmpty) - sendResponseCallback(Map.empty) - else { - val internalTopicsAllowed = request.header.clientId == AdminUtils.ADMIN_CLIENT_ID + val internalTopicsAllowed = request.header.clientId == AdminUtils.ADMIN_CLIENT_ID + val transactionVerificationEntries = new ReplicaManager.TransactionVerificationEntries - // call the replica manager to append messages to the replicas + def postVerificationCallback(newRequestLocal: RequestLocal) + (newlyVerifiedEntries: Map[TopicPartition, MemoryRecords], errorResults: Map[TopicPartition, LogAppendResult]): Unit = { replicaManager.appendRecords( timeout = produceRequest.timeout.toLong, requiredAcks = produceRequest.acks, internalTopicsAllowed = internalTopicsAllowed, origin = AppendOrigin.CLIENT, - entriesPerPartition = authorizedRequestInfo, - requestLocal = requestLocal, + entriesPerPartition = newlyVerifiedEntries ++ transactionVerificationEntries.verified, responseCallback = sendResponseCallback, recordConversionStatsCallback = processingStatsCallback, - transactionalId = produceRequest.transactionalId() + requestLocal = newRequestLocal, + preAppendErrors = errorResults + ) + } + + if (authorizedRequestInfo.isEmpty) + sendResponseCallback(Map.empty) + else { + // call the replica manager to append messages to the replicas + replicaManager.appendRecordsWithVerification( + entriesPerPartition = authorizedRequestInfo, + transactionVerificationEntries = transactionVerificationEntries, + transactionalId = produceRequest.transactionalId, + requestLocal = requestLocal, + postVerificationCallback = postVerificationCallback ) // if the request is put into the purgatory, it will have a held reference and hence cannot be garbage collected; diff --git a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala index f5974b1e93f6e..225c5425bb616 100755 --- a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala +++ b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala @@ -67,17 +67,17 @@ object KafkaRequestHandler { if (requestChannel == null || currentRequest == null) { if (!bypassThreadCheck) throw new IllegalStateException("Attempted to reschedule to request handler thread from non-request handler thread.") - T => asyncCompletionCallback(requestLocal, T) + t => asyncCompletionCallback(requestLocal, t) } else { - T => { + t => { if (threadCurrentRequest.get() == currentRequest) { // If the callback is actually executed on the same request thread, we can directly execute // it without re-scheduling it. - asyncCompletionCallback(requestLocal, T) + asyncCompletionCallback(requestLocal, t) } else { // The requestChannel and request are captured in this lambda, so when it's executed on the callback thread // we can re-schedule the original callback on a request thread and update the metrics accordingly. - requestChannel.sendCallbackRequest(RequestChannel.CallbackRequest(newRequestLocal => asyncCompletionCallback(newRequestLocal, T), currentRequest)) + requestChannel.sendCallbackRequest(RequestChannel.CallbackRequest(newRequestLocal => asyncCompletionCallback(newRequestLocal, t), currentRequest)) } } } diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index 3eaef6bb44426..435a7820140d6 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -24,8 +24,7 @@ import kafka.log.remote.RemoteLogManager import kafka.log.{LogManager, UnifiedLog} import kafka.server.HostedPartition.Online import kafka.server.QuotaFactory.QuotaManagers -import kafka.server.ReplicaManager.{AtMinIsrPartitionCountMetricName, FailedIsrUpdatesPerSecMetricName, IsrExpandsPerSecMetricName, IsrShrinksPerSecMetricName, LeaderCountMetricName, OfflineReplicaCountMetricName, PartitionCountMetricName, PartitionsWithLateTransactionsCountMetricName, ProducerIdCountMetricName, ReassigningPartitionsMetricName, UnderMinIsrPartitionCountMetricName, UnderReplicatedPartitionsMetricName} -import kafka.server.ReplicaManager.createLogReadResult +import kafka.server.ReplicaManager.{AtMinIsrPartitionCountMetricName, FailedIsrUpdatesPerSecMetricName, IsrExpandsPerSecMetricName, IsrShrinksPerSecMetricName, LeaderCountMetricName, OfflineReplicaCountMetricName, PartitionCountMetricName, PartitionsWithLateTransactionsCountMetricName, ProducerIdCountMetricName, ReassigningPartitionsMetricName, TransactionVerificationEntries, UnderMinIsrPartitionCountMetricName, UnderReplicatedPartitionsMetricName, createLogReadResult} import kafka.server.checkpoints.{LazyOffsetCheckpoints, OffsetCheckpointFile, OffsetCheckpoints} import kafka.server.metadata.ZkMetadataCache import kafka.utils.Implicits._ @@ -247,6 +246,14 @@ object ReplicaManager { lastStableOffset = None, exception = Some(e)) } + + class TransactionVerificationEntries { + + val verified = mutable.Map[TopicPartition, MemoryRecords]() + val unverified = mutable.Map[TopicPartition, MemoryRecords]() + val errors = mutable.Map[TopicPartition, Errors]() + val verificationGuards = mutable.Map[TopicPartition, VerificationGuard]() + } } class ReplicaManager(val config: KafkaConfig, @@ -744,27 +751,6 @@ class ReplicaManager(val config: KafkaConfig, def tryCompleteActions(): Unit = defaultActionQueue.tryCompleteActions() - /** - * Append messages to leader replicas of the partition, and wait for them to be replicated to other replicas; - * the callback function will be triggered either when timeout or the required acks are satisfied; - * if the callback function itself is already synchronized on some object then pass this object to avoid deadlock. - * - * Noted that all pending delayed check operations are stored in a queue. All callers to ReplicaManager.appendRecords() - * are expected to call ActionQueue.tryCompleteActions for all affected partitions, without holding any conflicting - * locks. - * - * @param timeout maximum time we will wait to append before returning - * @param requiredAcks number of replicas who must acknowledge the append before sending the response - * @param internalTopicsAllowed boolean indicating whether internal topics can be appended to - * @param origin source of the append request (ie, client, replication, coordinator) - * @param entriesPerPartition the records per partition to be appended - * @param responseCallback callback for sending the response - * @param delayedProduceLock lock for the delayed actions - * @param recordConversionStatsCallback callback for updating stats on record conversions - * @param requestLocal container for the stateful instances scoped to this request - * @param transactionalId transactional ID if the request is from a producer and the producer is transactional - * @param actionQueue the action queue to use. ReplicaManager#defaultActionQueue is used by default. - */ def appendRecords(timeout: Long, requiredAcks: Short, internalTopicsAllowed: Boolean, @@ -774,116 +760,20 @@ class ReplicaManager(val config: KafkaConfig, delayedProduceLock: Option[Lock] = None, recordConversionStatsCallback: Map[TopicPartition, RecordConversionStats] => Unit = _ => (), requestLocal: RequestLocal = RequestLocal.NoCaching, - transactionalId: String = null, - actionQueue: ActionQueue = this.defaultActionQueue): Unit = { - if (isValidRequiredAcks(requiredAcks)) { - - val verificationGuards: mutable.Map[TopicPartition, VerificationGuard] = mutable.Map[TopicPartition, VerificationGuard]() - val (verifiedEntriesPerPartition, notYetVerifiedEntriesPerPartition, errorsPerPartition) = - if (transactionalId == null || !config.transactionPartitionVerificationEnable) - (entriesPerPartition, Map.empty[TopicPartition, MemoryRecords], Map.empty[TopicPartition, Errors]) - else { - val verifiedEntries = mutable.Map[TopicPartition, MemoryRecords]() - val unverifiedEntries = mutable.Map[TopicPartition, MemoryRecords]() - val errorEntries = mutable.Map[TopicPartition, Errors]() - partitionEntriesForVerification(verificationGuards, entriesPerPartition, verifiedEntries, unverifiedEntries, errorEntries) - (verifiedEntries.toMap, unverifiedEntries.toMap, errorEntries.toMap) - } - - if (notYetVerifiedEntriesPerPartition.isEmpty || addPartitionsToTxnManager.isEmpty) { - appendEntries(verifiedEntriesPerPartition, internalTopicsAllowed, origin, requiredAcks, verificationGuards.toMap, - errorsPerPartition, recordConversionStatsCallback, timeout, responseCallback, delayedProduceLock, actionQueue)(requestLocal, Map.empty) - } else { - // For unverified entries, send a request to verify. When verified, the append process will proceed via the callback. - // We verify above that all partitions use the same producer ID. - val batchInfo = notYetVerifiedEntriesPerPartition.head._2.firstBatch() - addPartitionsToTxnManager.foreach(_.verifyTransaction( - transactionalId = transactionalId, - producerId = batchInfo.producerId, - producerEpoch = batchInfo.producerEpoch, - topicPartitions = notYetVerifiedEntriesPerPartition.keySet.toSeq, - callback = KafkaRequestHandler.wrapAsyncCallback( - appendEntries( - entriesPerPartition, - internalTopicsAllowed, - origin, - requiredAcks, - verificationGuards.toMap, - errorsPerPartition, - recordConversionStatsCallback, - timeout, - responseCallback, - delayedProduceLock, - actionQueue - ), - requestLocal) - )) - } - } else { - // If required.acks is outside accepted range, something is wrong with the client - // Just return an error and don't handle the request at all - val responseStatus = entriesPerPartition.map { case (topicPartition, _) => - topicPartition -> new PartitionResponse( - Errors.INVALID_REQUIRED_ACKS, - LogAppendInfo.UNKNOWN_LOG_APPEND_INFO.firstOffset, - RecordBatch.NO_TIMESTAMP, - LogAppendInfo.UNKNOWN_LOG_APPEND_INFO.logStartOffset - ) - } - responseCallback(responseStatus) + actionQueue: ActionQueue = this.defaultActionQueue, + verificationGuards: Map[TopicPartition, VerificationGuard] = Map.empty, + preAppendErrors: Map[TopicPartition, LogAppendResult] = Map.empty): Unit = { + if (!isValidRequiredAcks(requiredAcks)) { + sendInvalidRequiredAcksResponse(entriesPerPartition, responseCallback) + return } - } - /* - * Note: This method can be used as a callback in a different request thread. Ensure that correct RequestLocal - * is passed when executing this method. Accessing non-thread-safe data structures should be avoided if possible. - */ - private def appendEntries(allEntries: Map[TopicPartition, MemoryRecords], - internalTopicsAllowed: Boolean, - origin: AppendOrigin, - requiredAcks: Short, - verificationGuards: Map[TopicPartition, VerificationGuard], - errorsPerPartition: Map[TopicPartition, Errors], - recordConversionStatsCallback: Map[TopicPartition, RecordConversionStats] => Unit, - timeout: Long, - responseCallback: Map[TopicPartition, PartitionResponse] => Unit, - delayedProduceLock: Option[Lock], - actionQueue: ActionQueue) - (requestLocal: RequestLocal, unverifiedEntries: Map[TopicPartition, Errors]): Unit = { val sTime = time.milliseconds - val verifiedEntries = - if (unverifiedEntries.isEmpty) - allEntries - else - allEntries.filter { case (tp, _) => - !unverifiedEntries.contains(tp) - } - val localProduceResults = appendToLocalLog(internalTopicsAllowed = internalTopicsAllowed, - origin, verifiedEntries, requiredAcks, requestLocal, verificationGuards.toMap) + origin, entriesPerPartition, requiredAcks, requestLocal, verificationGuards.toMap) debug("Produce to local log in %d ms".format(time.milliseconds - sTime)) - val errorResults = (unverifiedEntries ++ errorsPerPartition).map { - case (topicPartition, error) => - // translate transaction coordinator errors to known producer response errors - val customException = - error match { - case Errors.INVALID_TXN_STATE => Some(error.exception("Partition was not added to the transaction")) - case Errors.CONCURRENT_TRANSACTIONS | - Errors.COORDINATOR_LOAD_IN_PROGRESS | - Errors.COORDINATOR_NOT_AVAILABLE | - Errors.NOT_COORDINATOR => Some(new NotEnoughReplicasException( - s"Unable to verify the partition has been added to the transaction. Underlying error: ${error.toString}")) - case _ => None - } - topicPartition -> LogAppendResult( - LogAppendInfo.UNKNOWN_LOG_APPEND_INFO, - Some(customException.getOrElse(error.exception)), - hasCustomErrorMessage = customException.isDefined - ) - } - - val allResults = localProduceResults ++ errorResults + val allResults = localProduceResults ++ preAppendErrors val produceStatus = allResults.map { case (topicPartition, result) => topicPartition -> ProducePartitionStatus( result.info.lastOffset + 1, // required offset @@ -900,18 +790,19 @@ class ReplicaManager(val config: KafkaConfig, } actionQueue.add { - () => allResults.foreach { case (topicPartition, result) => - val requestKey = TopicPartitionOperationKey(topicPartition) - result.info.leaderHwChange match { - case LeaderHwChange.INCREASED => - // some delayed operations may be unblocked after HW changed - delayedProducePurgatory.checkAndComplete(requestKey) - delayedFetchPurgatory.checkAndComplete(requestKey) - delayedDeleteRecordsPurgatory.checkAndComplete(requestKey) - case LeaderHwChange.SAME => - // probably unblock some follower fetch requests since log end offset has been updated - delayedFetchPurgatory.checkAndComplete(requestKey) - case LeaderHwChange.NONE => + () => + allResults.foreach { case (topicPartition, result) => + val requestKey = TopicPartitionOperationKey(topicPartition) + result.info.leaderHwChange match { + case LeaderHwChange.INCREASED => + // some delayed operations may be unblocked after HW changed + delayedProducePurgatory.checkAndComplete(requestKey) + delayedFetchPurgatory.checkAndComplete(requestKey) + delayedDeleteRecordsPurgatory.checkAndComplete(requestKey) + case LeaderHwChange.SAME => + // probably unblock some follower fetch requests since log end offset has been updated + delayedFetchPurgatory.checkAndComplete(requestKey) + case LeaderHwChange.NONE => // nothing } } @@ -919,13 +810,13 @@ class ReplicaManager(val config: KafkaConfig, recordConversionStatsCallback(localProduceResults.map { case (k, v) => k -> v.info.recordConversionStats }) - if (delayedProduceRequestRequired(requiredAcks, allEntries, allResults)) { + if (delayedProduceRequestRequired(requiredAcks, entriesPerPartition, allResults)) { // create delayed produce operation val produceMetadata = ProduceMetadata(requiredAcks, produceStatus) val delayedProduce = new DelayedProduce(timeout, produceMetadata, this, responseCallback, delayedProduceLock) // create a list of (topic, partition) pairs to use as keys for this delayed produce operation - val producerRequestKeys = allEntries.keys.map(TopicPartitionOperationKey(_)).toSeq + val producerRequestKeys = entriesPerPartition.keys.map(TopicPartitionOperationKey(_)).toSeq // try to complete the request immediately, otherwise put it into the purgatory // this is because while the delayed produce operation is being created, new @@ -938,11 +829,130 @@ class ReplicaManager(val config: KafkaConfig, } } - private def partitionEntriesForVerification(verificationGuards: mutable.Map[TopicPartition, VerificationGuard], - entriesPerPartition: Map[TopicPartition, MemoryRecords], - verifiedEntries: mutable.Map[TopicPartition, MemoryRecords], - unverifiedEntries: mutable.Map[TopicPartition, MemoryRecords], - errorEntries: mutable.Map[TopicPartition, Errors]): Unit= { + private def sendInvalidRequiredAcksResponse(entries: Map[TopicPartition, MemoryRecords], + responseCallback: Map[TopicPartition, PartitionResponse] => Unit): Unit = { + // If required.acks is outside accepted range, something is wrong with the client + // Just return an error and don't handle the request at all + val responseStatus = entries.map { case (topicPartition, _) => + topicPartition -> new PartitionResponse( + Errors.INVALID_REQUIRED_ACKS, + LogAppendInfo.UNKNOWN_LOG_APPEND_INFO.firstOffset, + RecordBatch.NO_TIMESTAMP, + LogAppendInfo.UNKNOWN_LOG_APPEND_INFO.logStartOffset + ) + } + responseCallback(responseStatus) + } + + /** + * Append messages to leader replicas of the partition, and wait for them to be replicated to other replicas; + * the callback function will be triggered either when timeout or the required acks are satisfied; + * if the callback function itself is already synchronized on some object then pass this object to avoid deadlock. + * + * Noted that all pending delayed check operations are stored in a queue. All callers to ReplicaManager.appendRecords() + * are expected to call ActionQueue.tryCompleteActions for all affected partitions, without holding any conflicting + * locks. + * + * @param timeout maximum time we will wait to append before returning + * @param requiredAcks number of replicas who must acknowledge the append before sending the response + * @param internalTopicsAllowed boolean indicating whether internal topics can be appended to + * @param origin source of the append request (ie, client, replication, coordinator) + * @param entriesPerPartition the records per partition to be appended + * @param responseCallback callback for sending the response + * @param delayedProduceLock lock for the delayed actions + * @param recordConversionStatsCallback callback for updating stats on record conversions + * @param requestLocal container for the stateful instances scoped to this request + * @param transactionalId transactional ID if the request is from a producer and the producer is transactional + * @param actionQueue the action queue to use. ReplicaManager#defaultActionQueue is used by default. + */ + def appendRecordsWithVerification(entriesPerPartition: Map[TopicPartition, MemoryRecords], + transactionVerificationEntries: TransactionVerificationEntries, + transactionalId: String, + requestLocal: RequestLocal, + postVerificationCallback: RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit): Unit = { + if (transactionalId != null && config.transactionPartitionVerificationEnable && addPartitionsToTxnManager.isDefined) + partitionEntriesForVerification(transactionVerificationEntries, entriesPerPartition) + + val onVerificationComplete: (RequestLocal, Map[TopicPartition, Errors]) => Unit = appendRecordsAfterVerification( + entriesPerPartition, + transactionVerificationEntries, + postVerificationCallback, + ) + + if (transactionVerificationEntries.unverified.isEmpty) { + onVerificationComplete(requestLocal, transactionVerificationEntries.errors.toMap) + } else { + // For unverified entries, send a request to verify. When verified, the append process will proceed via the callback. + // We verify above that all partitions use the same producer ID. + val batchInfo = transactionVerificationEntries.unverified.head._2.firstBatch() + addPartitionsToTxnManager.foreach(_.verifyTransaction( + transactionalId = transactionalId, + producerId = batchInfo.producerId, + producerEpoch = batchInfo.producerEpoch, + topicPartitions = transactionVerificationEntries.unverified.keySet.toSeq, + callback = KafkaRequestHandler.wrapAsyncCallback(onVerificationComplete, requestLocal) + )) + } + } + + /** + * Append messages to leader replicas of the partition, and wait for them to be replicated to other replicas; + * the callback function will be triggered either when timeout or the required acks are satisfied; + * if the callback function itself is already synchronized on some object then pass this object to avoid deadlock. + * + * Noted that all pending delayed check operations are stored in a queue. All callers to ReplicaManager.appendRecords() + * are expected to call ActionQueue.tryCompleteActions for all affected partitions, without holding any conflicting + * locks. + * + * @param allEntries the records per partition for all partitions in the request + * @param internalTopicsAllowed boolean indicating whether internal topics can be appended to + * @param origin source of the append request (ie, client, replication, coordinator) + * @param requiredAcks number of replicas who must acknowledge the append before sending the response + * @param verificationGuards verificationGuards for ensuring a partition has been added to the transaction + * @param errorsPerPartition the mapping from partition to errors we have already seen + * @param recordConversionStatsCallback callback for updating stats on record conversions + * @param timeout maximum time we will wait to append before returning + * @param responseCallback callback for sending the response + * @param delayedProduceLock lock for the delayed actions + * @param actionQueue the action queue to use. ReplicaManager#defaultActionQueue is used by default. + * @param requestLocal container for the stateful instances scoped to this request + * @param unverifiedEntries the records per partition for topic partitions that were not verified by the transaction coordinator + */ + def appendRecordsAfterVerification(allEntries: Map[TopicPartition, MemoryRecords], + transactionVerificationEntries: TransactionVerificationEntries, + postVerificationCallback: RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit) + (requestLocal: RequestLocal, unverifiedEntries: Map[TopicPartition, Errors] = Map.empty): Unit = { + val verifiedEntries = + if (unverifiedEntries.isEmpty) + allEntries + else + allEntries.filter { case (tp, _) => + !unverifiedEntries.contains(tp) + } + + val errorResults = (unverifiedEntries ++ transactionVerificationEntries.errors).map { + case (topicPartition, error) => + // translate transaction coordinator errors to known producer response errors + val customException = + error match { + case Errors.INVALID_TXN_STATE => Some(error.exception("Partition was not added to the transaction")) + case Errors.CONCURRENT_TRANSACTIONS | + Errors.COORDINATOR_LOAD_IN_PROGRESS | + Errors.COORDINATOR_NOT_AVAILABLE | + Errors.NOT_COORDINATOR => Some(new NotEnoughReplicasException( + s"Unable to verify the partition has been added to the transaction. Underlying error: ${error.toString}")) + case _ => None + } + topicPartition -> LogAppendResult( + LogAppendInfo.UNKNOWN_LOG_APPEND_INFO, + Some(customException.getOrElse(error.exception)), + hasCustomErrorMessage = customException.isDefined + ) + } + postVerificationCallback(requestLocal)(verifiedEntries, errorResults) + } + + private def partitionEntriesForVerification(transactionVerificationEntries: TransactionVerificationEntries, entriesPerPartition: Map[TopicPartition, MemoryRecords]): TransactionVerificationEntries = { val transactionalProducerIds = mutable.HashSet[Long]() entriesPerPartition.foreach { case (topicPartition, records) => try { @@ -955,22 +965,23 @@ class ReplicaManager(val config: KafkaConfig, val firstBatch = records.firstBatch val verificationGuard = getPartitionOrException(topicPartition).maybeStartTransactionVerification(firstBatch.producerId, firstBatch.baseSequence, firstBatch.producerEpoch) if (verificationGuard != VerificationGuard.SENTINEL) { - verificationGuards.put(topicPartition, verificationGuard) - unverifiedEntries.put(topicPartition, records) + transactionVerificationEntries.verificationGuards.put(topicPartition, verificationGuard) + transactionVerificationEntries.unverified.put(topicPartition, records) } else - verifiedEntries.put(topicPartition, records) + transactionVerificationEntries.verified.put(topicPartition, records) } else { // If there is no producer ID or transactional records in the batches, no need to verify. - verifiedEntries.put(topicPartition, records) + transactionVerificationEntries.verified.put(topicPartition, records) } } catch { - case e: Exception => errorEntries.put(topicPartition, Errors.forException(e)) + case e: Exception => transactionVerificationEntries.errors.put(topicPartition, Errors.forException(e)) } } // We should have exactly one producer ID for transactional records if (transactionalProducerIds.size > 1) { throw new InvalidPidMappingException("Transactional records contained more than one producer ID") } + transactionVerificationEntries } /** diff --git a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala index 9b8c02249aa27..463faf7a9d7b9 100644 --- a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala @@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.locks.Lock import kafka.coordinator.AbstractCoordinatorConcurrencyTest._ import kafka.log.UnifiedLog +import kafka.server.ReplicaManager.TransactionVerificationEntries import kafka.server._ import kafka.utils._ import kafka.zk.KafkaZkClient @@ -32,7 +33,7 @@ import org.apache.kafka.common.record.{MemoryRecords, RecordBatch, RecordConvers import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse import org.apache.kafka.server.util.timer.MockTimer import org.apache.kafka.server.util.{MockScheduler, MockTime} -import org.apache.kafka.storage.internals.log.{AppendOrigin, LogConfig} +import org.apache.kafka.storage.internals.log.{AppendOrigin, LogConfig, VerificationGuard} import org.junit.jupiter.api.{AfterEach, BeforeEach} import org.mockito.Mockito.{CALLS_REAL_METHODS, mock, withSettings} @@ -179,8 +180,9 @@ object AbstractCoordinatorConcurrencyTest { delayedProduceLock: Option[Lock] = None, processingStatsCallback: Map[TopicPartition, RecordConversionStats] => Unit = _ => (), requestLocal: RequestLocal = RequestLocal.NoCaching, - transactionalId: String = null, - actionQueue: ActionQueue = null): Unit = { + actionQueue: ActionQueue = null, + verificationGuards: Map[TopicPartition, VerificationGuard] = Map.empty, + preAppendErrors: Map[TopicPartition, LogAppendResult] = Map.empty): Unit = { if (entriesPerPartition.isEmpty) return @@ -209,6 +211,16 @@ object AbstractCoordinatorConcurrencyTest { producePurgatory.tryCompleteElseWatch(delayedProduce, producerRequestKeys) } + override def appendRecordsWithVerification(entriesPerPartition: Map[TopicPartition, MemoryRecords], + transactionVerificationEntries: TransactionVerificationEntries, + transactionalId: String, + requestLocal: RequestLocal, + postVerificationCallback: RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit): Unit = { + + postVerificationCallback(requestLocal)(entriesPerPartition, Map.empty) + } + + override def getMagic(topicPartition: TopicPartition): Option[Byte] = { Some(RecordBatch.MAGIC_VALUE_V2) } diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala index 04462d7ae88bb..adf7f12869466 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala @@ -19,7 +19,7 @@ package kafka.coordinator.group import java.util.{Optional, OptionalInt} import kafka.common.OffsetAndMetadata -import kafka.server.{DelayedOperationPurgatory, HostedPartition, KafkaConfig, ReplicaManager, RequestLocal} +import kafka.server.{DelayedOperationPurgatory, HostedPartition, KafkaConfig, LogAppendResult, ReplicaManager, RequestLocal} import kafka.utils._ import org.apache.kafka.common.{TopicIdPartition, TopicPartition, Uuid} import org.apache.kafka.common.protocol.Errors @@ -48,7 +48,7 @@ import org.mockito.ArgumentMatchers.{any, anyLong, anyShort} import org.mockito.Mockito.{mock, when} import scala.jdk.CollectionConverters._ -import scala.collection.{Seq, mutable} +import scala.collection.{Seq, mutable, Map => SMap} import scala.collection.mutable.ArrayBuffer import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future, Promise, TimeoutException} @@ -3865,6 +3865,7 @@ class GroupCoordinatorTest { any(), any(), any(), + any(), any() )).thenAnswer(_ => { capturedArgument.getValue.apply( @@ -3901,6 +3902,7 @@ class GroupCoordinatorTest { any(), any(), any(), + any(), any())).thenAnswer(_ => { capturedArgument.getValue.apply( Map(new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId) -> @@ -4047,6 +4049,7 @@ class GroupCoordinatorTest { any(), any(), any(), + any(), any() )).thenAnswer(_ => { capturedArgument.getValue.apply( @@ -4074,6 +4077,12 @@ class GroupCoordinatorTest { // Since transactional ID is only used in appendRecords, we can use a dummy value. Ensure it passes through. val transactionalId = "dummy-txn-id" + + val postVerificationCallback: ArgumentCaptor[RequestLocal => (SMap[TopicPartition, MemoryRecords], SMap[TopicPartition, LogAppendResult]) => Unit] = ArgumentCaptor.forClass( + classOf[RequestLocal => (SMap[TopicPartition, MemoryRecords], SMap[TopicPartition, LogAppendResult]) => Unit]) + when(replicaManager.appendRecordsWithVerification(any(), any(), ArgumentMatchers.eq(transactionalId), any(), postVerificationCallback.capture())).thenAnswer( + _ => postVerificationCallback.getValue()(RequestLocal.NoCaching)(Map.empty, Map.empty) + ) when(replicaManager.appendRecords(anyLong, anyShort(), internalTopicsAllowed = ArgumentMatchers.eq(true), @@ -4083,7 +4092,8 @@ class GroupCoordinatorTest { any[Option[ReentrantLock]], any(), any(), - ArgumentMatchers.eq(transactionalId), + any(), + any(), any() )).thenAnswer(_ => { capturedArgument.getValue.apply( diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala index c7736fd7bb918..7896fcb4ad613 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala @@ -27,7 +27,7 @@ import javax.management.ObjectName import kafka.cluster.Partition import kafka.common.OffsetAndMetadata import kafka.log.UnifiedLog -import kafka.server.{HostedPartition, KafkaConfig, ReplicaManager, RequestLocal} +import kafka.server.{HostedPartition, KafkaConfig, LogAppendResult, ReplicaManager, RequestLocal} import kafka.utils.TestUtils import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription @@ -1187,6 +1187,7 @@ class GroupMetadataManagerTest { any(), any(), any(), + any(), any()) verify(replicaManager).getMagic(any()) } @@ -1225,6 +1226,7 @@ class GroupMetadataManagerTest { any(), any(), any(), + any(), any()) verify(replicaManager).getMagic(any()) } @@ -1301,6 +1303,7 @@ class GroupMetadataManagerTest { any(), any(), any(), + any(), any()) // Will update sensor after commit assertEquals(1, TestUtils.totalMetricValue(metrics, "offset-commit-count")) @@ -1330,6 +1333,7 @@ class GroupMetadataManagerTest { commitErrors = Some(errors) } + setUpTransactionVerification(replicaManager, transactionalId) groupMetadataManager.storeOffsets(group, memberId, offsets, callback, transactionalId, producerId, producerEpoch) assertTrue(group.hasOffsets) assertTrue(group.allOffsets.isEmpty) @@ -1343,7 +1347,8 @@ class GroupMetadataManagerTest { any[Option[ReentrantLock]], any(), any(), - ArgumentMatchers.eq(transactionalId), + any(), + any(), any()) verify(replicaManager).getMagic(any()) capturedResponseCallback.getValue.apply(Map(groupTopicPartition -> @@ -1381,6 +1386,7 @@ class GroupMetadataManagerTest { commitErrors = Some(errors) } + setUpTransactionVerification(replicaManager, transactionalId) groupMetadataManager.storeOffsets(group, memberId, offsets, callback, transactionalId, producerId, producerEpoch) assertTrue(group.hasOffsets) assertTrue(group.allOffsets.isEmpty) @@ -1404,7 +1410,8 @@ class GroupMetadataManagerTest { any[Option[ReentrantLock]], any(), any(), - ArgumentMatchers.eq(transactionalId), + any(), + any(), any()) verify(replicaManager).getMagic(any()) } @@ -1432,6 +1439,7 @@ class GroupMetadataManagerTest { commitErrors = Some(errors) } + setUpTransactionVerification(replicaManager, transactionalId) groupMetadataManager.storeOffsets(group, memberId, offsets, callback, transactionalId, producerId, producerEpoch) assertTrue(group.hasOffsets) assertTrue(group.allOffsets.isEmpty) @@ -1455,7 +1463,8 @@ class GroupMetadataManagerTest { any[Option[ReentrantLock]], any(), any(), - ArgumentMatchers.eq(transactionalId), + any(), + any(), any()) verify(replicaManager).getMagic(any()) } @@ -1484,6 +1493,7 @@ class GroupMetadataManagerTest { commitErrors = Some(errors) } + setUpTransactionVerification(replicaManager, transactionalId) groupMetadataManager.storeOffsets(group, memberId, offsets, callback, transactionalId, producerId, producerEpoch) assertTrue(group.hasOffsets) assertTrue(group.allOffsets.isEmpty) @@ -1509,7 +1519,8 @@ class GroupMetadataManagerTest { any[Option[ReentrantLock]], any(), any(), - ArgumentMatchers.eq(transactionalId), + any(), + any(), any()) verify(replicaManager).getMagic(any()) } @@ -1662,6 +1673,7 @@ class GroupMetadataManagerTest { any(), any(), any(), + any(), any()) verify(replicaManager).getMagic(any()) assertEquals(1, TestUtils.totalMetricValue(metrics, "offset-commit-count")) @@ -1769,6 +1781,7 @@ class GroupMetadataManagerTest { any(), any(), any(), + any(), any()) verify(replicaManager, times(2)).getMagic(any()) } @@ -2876,6 +2889,7 @@ class GroupMetadataManagerTest { any(), any(), any(), + any(), any()) capturedArgument } @@ -2893,6 +2907,7 @@ class GroupMetadataManagerTest { any(), any(), any(), + any(), any() )).thenAnswer(_ => { capturedCallback.getValue.apply( @@ -3112,4 +3127,12 @@ class GroupMetadataManagerTest { assertTrue(group.offset(topicPartition).map(_.expireTimestamp).contains(None)) } } + + def setUpTransactionVerification(replicaManager: ReplicaManager, transactionalId: String): Unit = { + val postVerificationCallback: ArgumentCaptor[RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit] = ArgumentCaptor.forClass( + classOf[RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit]) + when(replicaManager.appendRecordsWithVerification(any(), any(), ArgumentMatchers.eq(transactionalId), any(), postVerificationCallback.capture())).thenAnswer( + _ => postVerificationCallback.getValue()(RequestLocal.NoCaching)(Map.empty, Map.empty) + ) + } } diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 17abdb0472d24..a43c29fbca931 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -2445,6 +2445,12 @@ class KafkaApisTest { .build(version.toShort) val request = buildRequest(produceRequest) + val postVerificationCallback: ArgumentCaptor[RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit] = ArgumentCaptor.forClass( + classOf[RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit]) + when(replicaManager.appendRecordsWithVerification(any(), any(), any(), any(), postVerificationCallback.capture())).thenAnswer( + _ => postVerificationCallback.getValue()(RequestLocal.NoCaching)(Map.empty, Map.empty) + ) + when(replicaManager.appendRecords(anyLong, anyShort, ArgumentMatchers.eq(false), @@ -2455,7 +2461,8 @@ class KafkaApisTest { any(), any(), any(), - any() + any(), + any(), )).thenAnswer(_ => responseCallback.getValue.apply(Map(tp -> new PartitionResponse(Errors.INVALID_PRODUCER_EPOCH)))) when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), @@ -2487,6 +2494,8 @@ class KafkaApisTest { reset(replicaManager, clientQuotaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) val responseCallback: ArgumentCaptor[Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[Map[TopicPartition, PartitionResponse] => Unit]) + val postVerificationCallback: ArgumentCaptor[RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit] = ArgumentCaptor.forClass( + classOf[RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit]) val tp = new TopicPartition("topic", 0) @@ -2505,8 +2514,31 @@ class KafkaApisTest { val request = buildRequest(produceRequest) val kafkaApis = createKafkaApis() + val requestLocal = RequestLocal.withThreadConfinedCaching + val newRequestLocal = RequestLocal.NoCaching + + when(replicaManager.appendRecordsWithVerification(any(), any(), any(), any(), postVerificationCallback.capture())).thenAnswer( + arg => replicaManager.appendRecordsAfterVerification(arg.getArgument(0), arg.getArgument(1), postVerificationCallback.getValue)(newRequestLocal, Map.empty) + ) + + when(replicaManager.appendRecordsAfterVerification(any(), any(), postVerificationCallback.capture())(any(), any())).thenAnswer( + _ => postVerificationCallback.getValue()(newRequestLocal)(Map.empty, Map.empty) + ) - kafkaApis.handleProduceRequest(request, RequestLocal.withThreadConfinedCaching) + kafkaApis.handleProduceRequest(request, requestLocal) + + verify(replicaManager).appendRecordsWithVerification( + any(), + any(), + ArgumentMatchers.eq(transactionalId), + ArgumentMatchers.eq(requestLocal), + ArgumentMatchers.eq(postVerificationCallback.getValue) + ) + + verify(replicaManager).appendRecordsAfterVerification( + any(), + any(), + ArgumentMatchers.eq(postVerificationCallback.getValue))(ArgumentMatchers.eq(newRequestLocal), any()) verify(replicaManager).appendRecords(anyLong, anyShort, @@ -2516,8 +2548,9 @@ class KafkaApisTest { responseCallback.capture(), any(), any(), + ArgumentMatchers.eq(newRequestLocal), + any(), any(), - ArgumentMatchers.eq(transactionalId), any()) } } @@ -2645,6 +2678,7 @@ class KafkaApisTest { any(), ArgumentMatchers.eq(requestLocal), any(), + any(), any() )).thenAnswer(_ => responseCallback.getValue.apply(Map(tp2 -> new PartitionResponse(Errors.NONE)))) @@ -2777,6 +2811,7 @@ class KafkaApisTest { any(), ArgumentMatchers.eq(requestLocal), any(), + any(), any() )).thenAnswer(_ => responseCallback.getValue.apply(Map(tp2 -> new PartitionResponse(Errors.NONE)))) @@ -2811,6 +2846,7 @@ class KafkaApisTest { any(), ArgumentMatchers.eq(requestLocal), any(), + any(), any()) } diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index ec61da1e9c70f..3100ac1a2f06b 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -2175,7 +2175,7 @@ class ReplicaManagerTest { val idempotentRecords2 = MemoryRecords.withIdempotentRecords(CompressionType.NONE, producerId, producerEpoch, sequence, new SimpleRecord("message".getBytes)) - appendRecordsToMultipleTopics(replicaManager, Map(tp0 -> transactionalRecords, tp1 -> idempotentRecords2), transactionalId) + appendRecordsToMultipleTopicsWithVerification(replicaManager, Map(tp0 -> transactionalRecords, tp1 -> idempotentRecords2), transactionalId) verify(addPartitionsToTxnManager, times(1)).verifyTransaction( ArgumentMatchers.eq(transactionalId), ArgumentMatchers.eq(producerId), @@ -2210,7 +2210,7 @@ class ReplicaManagerTest { new SimpleRecord("message".getBytes)) // We should add these partitions to the manager to verify. - val result = appendRecords(replicaManager, tp0, transactionalRecords, origin = appendOrigin, transactionalId = transactionalId) + val result = appendRecordsWithVerification(replicaManager, tp0, transactionalRecords, origin = appendOrigin, transactionalId = transactionalId) val appendCallback = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(1)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2229,7 +2229,7 @@ class ReplicaManagerTest { assertEquals(verificationGuard, getVerificationGuard(replicaManager, tp0, producerId)) // This time verification is successful. - appendRecords(replicaManager, tp0, transactionalRecords, origin = appendOrigin, transactionalId = transactionalId) + appendRecordsWithVerification(replicaManager, tp0, transactionalRecords, origin = appendOrigin, transactionalId = transactionalId) val appendCallback2 = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(2)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2268,7 +2268,7 @@ class ReplicaManagerTest { new SimpleRecord("message".getBytes)) // We should add these partitions to the manager to verify. - val result = appendRecords(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) + val result = appendRecordsWithVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) val appendCallback = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(1)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2290,7 +2290,7 @@ class ReplicaManagerTest { val transactionalRecords2 = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, producerEpoch, sequence + 1, new SimpleRecord("message".getBytes)) - val result2 = appendRecords(replicaManager, tp0, transactionalRecords2, transactionalId = transactionalId) + val result2 = appendRecordsWithVerification(replicaManager, tp0, transactionalRecords2, transactionalId = transactionalId) val appendCallback2 = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(2)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2333,7 +2333,7 @@ class ReplicaManagerTest { val transactionalRecords = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, producerEpoch, sequence, new SimpleRecord(s"message $sequence".getBytes)) - appendRecordsToMultipleTopics(replicaManager, Map(tp0 -> transactionalRecords, tp1 -> transactionalRecords), transactionalId).onFire { responses => + appendRecordsToMultipleTopicsWithVerification(replicaManager, Map(tp0 -> transactionalRecords, tp1 -> transactionalRecords), transactionalId).onFire { responses => responses.foreach { entry => assertEquals(Errors.NONE, entry._2.error) } @@ -2373,7 +2373,7 @@ class ReplicaManagerTest { new SimpleRecord(s"message $sequence".getBytes))) assertThrows(classOf[InvalidPidMappingException], - () => appendRecordsToMultipleTopics(replicaManager, transactionalRecords, transactionalId = transactionalId)) + () => appendRecordsToMultipleTopicsWithVerification(replicaManager, transactionalRecords, transactionalId = transactionalId)) // We should not add these partitions to the manager to verify. verify(addPartitionsToTxnManager, times(0)).verifyTransaction(any(), any(), any(), any(), any()) } finally { @@ -2472,7 +2472,7 @@ class ReplicaManagerTest { new SimpleRecord("message".getBytes)) // We should add these partitions to the manager to verify. - val result = appendRecords(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) + val result = appendRecordsWithVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) val appendCallback = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(1)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2498,7 +2498,7 @@ class ReplicaManagerTest { assertEquals(verificationGuard, getVerificationGuard(replicaManager, tp0, producerId)) // This time we do not verify - appendRecords(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) + appendRecordsWithVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) verify(addPartitionsToTxnManager, times(1)).verifyTransaction(any(), any(), any(), any(), any()) assertEquals(VerificationGuard.SENTINEL, getVerificationGuard(replicaManager, tp0, producerId)) assertTrue(replicaManager.localLog(tp0).get.hasOngoingTransaction(producerId)) @@ -2527,7 +2527,7 @@ class ReplicaManagerTest { // Start verification and return the coordinator related errors. val expectedMessage = s"Unable to verify the partition has been added to the transaction. Underlying error: ${error.toString}" - val result = appendRecords(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) + val result = appendRecordsWithVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) val appendCallback = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(1)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2931,31 +2931,84 @@ class ReplicaManagerTest { origin = origin, entriesPerPartition = Map(partition -> records), responseCallback = appendCallback, - transactionalId = transactionalId, ) result } - private def appendRecordsToMultipleTopics(replicaManager: ReplicaManager, - entriesToAppend: Map[TopicPartition, MemoryRecords], - transactionalId: String, - origin: AppendOrigin = AppendOrigin.CLIENT, - requiredAcks: Short = -1): CallbackResult[Map[TopicPartition, PartitionResponse]] = { + private def appendRecordsToMultipleTopicsWithVerification(replicaManager: ReplicaManager, + entriesToAppend: Map[TopicPartition, MemoryRecords], + transactionalId: String, + origin: AppendOrigin = AppendOrigin.CLIENT, + requiredAcks: Short = -1): CallbackResult[Map[TopicPartition, PartitionResponse]] = { val result = new CallbackResult[Map[TopicPartition, PartitionResponse]]() def appendCallback(responses: Map[TopicPartition, PartitionResponse]): Unit = { responses.foreach( response => assertTrue(responses.get(response._1).isDefined)) result.fire(responses) } - replicaManager.appendRecords( - timeout = 1000, - requiredAcks = requiredAcks, - internalTopicsAllowed = false, - origin = origin, + val transactionVerificationEntries = new ReplicaManager.TransactionVerificationEntries + + def postVerificationCallback(newRequestLocal: RequestLocal) + (newlyVerifiedEntries: Map[TopicPartition, MemoryRecords], errorResults: Map[TopicPartition, LogAppendResult]): Unit = { + replicaManager.appendRecords( + timeout = 1000, + requiredAcks = requiredAcks, + internalTopicsAllowed = false, + origin = origin, + entriesPerPartition = newlyVerifiedEntries ++ transactionVerificationEntries.verified, + responseCallback = appendCallback, + requestLocal = newRequestLocal, + preAppendErrors = errorResults + ) + } + + replicaManager.appendRecordsWithVerification( entriesPerPartition = entriesToAppend, - responseCallback = appendCallback, - transactionalId = transactionalId, + transactionVerificationEntries, + transactionalId, + RequestLocal.NoCaching, + postVerificationCallback + ) + + result + } + + private def appendRecordsWithVerification(replicaManager: ReplicaManager, + partition: TopicPartition, + records: MemoryRecords, + origin: AppendOrigin = AppendOrigin.CLIENT, + requiredAcks: Short = -1, + transactionalId: String): CallbackResult[PartitionResponse] = { + val result = new CallbackResult[PartitionResponse]() + + def appendCallback(responses: Map[TopicPartition, PartitionResponse]): Unit = { + val response = responses.get(partition) + assertTrue(response.isDefined) + result.fire(response.get) + } + + val transactionVerificationEntries = new ReplicaManager.TransactionVerificationEntries + def postVerificationCallback(newRequestLocal: RequestLocal) + (newlyVerifiedEntries: Map[TopicPartition, MemoryRecords], errorResults: Map[TopicPartition, LogAppendResult]): Unit = { + replicaManager.appendRecords( + timeout = 1000, + requiredAcks = requiredAcks, + internalTopicsAllowed = false, + origin = origin, + entriesPerPartition = newlyVerifiedEntries ++ transactionVerificationEntries.verified, + responseCallback = appendCallback, + requestLocal = newRequestLocal, + preAppendErrors = errorResults + ) + } + + replicaManager.appendRecordsWithVerification( + entriesPerPartition = Map(partition -> records), + transactionVerificationEntries, + transactionalId, + RequestLocal.NoCaching, + postVerificationCallback ) result From 2a5231880e940d2c9c6d07b705384efdf92883d9 Mon Sep 17 00:00:00 2001 From: Justine Date: Thu, 16 Nov 2023 16:59:31 -0800 Subject: [PATCH 02/24] Fix build issues --- .../group/GroupMetadataManager.scala | 23 ++++++++++--------- .../main/scala/kafka/server/KafkaApis.scala | 1 + .../unit/kafka/server/KafkaApisTest.scala | 12 +++++++--- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala index b5b3ae78bcddd..76020476fb8c1 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala @@ -47,7 +47,7 @@ import org.apache.kafka.server.common.MetadataVersion import org.apache.kafka.server.common.MetadataVersion.{IBP_0_10_1_IV0, IBP_2_1_IV0, IBP_2_1_IV1, IBP_2_3_IV0} import org.apache.kafka.server.metrics.KafkaMetricsGroup import org.apache.kafka.server.util.KafkaScheduler -import org.apache.kafka.storage.internals.log.{AppendOrigin, FetchIsolation} +import org.apache.kafka.storage.internals.log.{AppendOrigin, FetchIsolation, VerificationGuard} import scala.collection._ import scala.collection.mutable.ArrayBuffer @@ -328,6 +328,7 @@ class GroupMetadataManager(brokerId: Int, records: Map[TopicPartition, MemoryRecords], requestLocal: RequestLocal, callback: Map[TopicPartition, PartitionResponse] => Unit, + verificationGuards: Map[TopicPartition, VerificationGuard] = Map.empty, preAppendErrors: Map[TopicPartition, LogAppendResult] = Map.empty): Unit = { // call replica manager to append the group message replicaManager.appendRecords( @@ -339,6 +340,7 @@ class GroupMetadataManager(brokerId: Int, delayedProduceLock = Some(group.lock), responseCallback = callback, requestLocal = requestLocal, + verificationGuards = verificationGuards, preAppendErrors = preAppendErrors) } @@ -473,18 +475,17 @@ class GroupMetadataManager(brokerId: Int, def postVerificationCallback(newRequestLocal: RequestLocal) (newlyVerifiedEntries: Map[TopicPartition, MemoryRecords], errorResults: Map[TopicPartition, LogAppendResult]): Unit = { - if (isTxnOffsetCommit) { - group.inLock { - addProducerGroup(producerId, group.groupId) - group.prepareTxnOffsetCommit(producerId, offsetMetadata) - } - } else { - group.inLock { - group.prepareOffsetCommit(offsetMetadata) + group.inLock { + if (isTxnOffsetCommit) { + addProducerGroup(producerId, group.groupId) + group.prepareTxnOffsetCommit(producerId, offsetMetadata) + } else { + group.prepareOffsetCommit(offsetMetadata) } - } - appendForGroup(group, newlyVerifiedEntries ++ transactionVerificationEntries.verified, newRequestLocal, putCacheCallback, errorResults) + appendForGroup(group, newlyVerifiedEntries ++ transactionVerificationEntries.verified, newRequestLocal, + putCacheCallback, transactionVerificationEntries.verificationGuards.toMap, errorResults) + } } if (transactionalId != null ) { replicaManager.appendRecordsWithVerification( diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 3b5ead4e3e27a..c9b10ce800baa 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -689,6 +689,7 @@ class KafkaApis(val requestChannel: RequestChannel, responseCallback = sendResponseCallback, recordConversionStatsCallback = processingStatsCallback, requestLocal = newRequestLocal, + verificationGuards = transactionVerificationEntries.verificationGuards.toMap, preAppendErrors = errorResults ) } diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index a43c29fbca931..d4331038abb76 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -2448,7 +2448,10 @@ class KafkaApisTest { val postVerificationCallback: ArgumentCaptor[RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit] = ArgumentCaptor.forClass( classOf[RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit]) when(replicaManager.appendRecordsWithVerification(any(), any(), any(), any(), postVerificationCallback.capture())).thenAnswer( - _ => postVerificationCallback.getValue()(RequestLocal.NoCaching)(Map.empty, Map.empty) + _ => { + val callback = postVerificationCallback.getValue + callback(RequestLocal.NoCaching)(Map.empty, Map.empty) + } ) when(replicaManager.appendRecords(anyLong, @@ -2518,11 +2521,14 @@ class KafkaApisTest { val newRequestLocal = RequestLocal.NoCaching when(replicaManager.appendRecordsWithVerification(any(), any(), any(), any(), postVerificationCallback.capture())).thenAnswer( - arg => replicaManager.appendRecordsAfterVerification(arg.getArgument(0), arg.getArgument(1), postVerificationCallback.getValue)(newRequestLocal, Map.empty) + arg => replicaManager.appendRecordsAfterVerification(arg.getArgument(0), arg.getArgument(1), postVerificationCallback.getValue())(newRequestLocal, Map.empty) ) when(replicaManager.appendRecordsAfterVerification(any(), any(), postVerificationCallback.capture())(any(), any())).thenAnswer( - _ => postVerificationCallback.getValue()(newRequestLocal)(Map.empty, Map.empty) + _ => { + val callback = postVerificationCallback.getValue + callback(RequestLocal.NoCaching)(Map.empty, Map.empty) + } ) kafkaApis.handleProduceRequest(request, requestLocal) From 865078bdf310a3ebb9278a291452feb494da2a36 Mon Sep 17 00:00:00 2001 From: Justine Date: Fri, 17 Nov 2023 10:47:48 -0800 Subject: [PATCH 03/24] Fix tests --- .../transaction/TransactionStateManagerTest.scala | 6 ++++++ core/src/test/scala/unit/kafka/server/KafkaApisTest.scala | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala index 94ffe7a979557..d870900d01893 100644 --- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala @@ -658,6 +658,7 @@ class TransactionStateManagerTest { any(), any(), any(), + any(), any() ) @@ -703,6 +704,7 @@ class TransactionStateManagerTest { any(), any(), any(), + any(), any() ) @@ -745,6 +747,7 @@ class TransactionStateManagerTest { any(), any(), any(), + any(), any()) assertEquals(Set.empty, listExpirableTransactionalIds()) @@ -803,6 +806,7 @@ class TransactionStateManagerTest { any(), any(), any(), + any(), any() ) @@ -953,6 +957,7 @@ class TransactionStateManagerTest { any(), any(), any(), + any(), any() )).thenAnswer(_ => callbackCapture.getValue.apply( recordsCapture.getValue.map { case (topicPartition, records) => @@ -1105,6 +1110,7 @@ class TransactionStateManagerTest { any(), any(), any(), + any(), any() )).thenAnswer(_ => capturedArgument.getValue.apply( Map(new TopicPartition(TRANSACTION_STATE_TOPIC_NAME, partitionId) -> diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index d4331038abb76..202f473175a72 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -2449,7 +2449,7 @@ class KafkaApisTest { classOf[RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit]) when(replicaManager.appendRecordsWithVerification(any(), any(), any(), any(), postVerificationCallback.capture())).thenAnswer( _ => { - val callback = postVerificationCallback.getValue + val callback = postVerificationCallback.getValue() callback(RequestLocal.NoCaching)(Map.empty, Map.empty) } ) @@ -2526,7 +2526,7 @@ class KafkaApisTest { when(replicaManager.appendRecordsAfterVerification(any(), any(), postVerificationCallback.capture())(any(), any())).thenAnswer( _ => { - val callback = postVerificationCallback.getValue + val callback = postVerificationCallback.getValue() callback(RequestLocal.NoCaching)(Map.empty, Map.empty) } ) @@ -2538,13 +2538,13 @@ class KafkaApisTest { any(), ArgumentMatchers.eq(transactionalId), ArgumentMatchers.eq(requestLocal), - ArgumentMatchers.eq(postVerificationCallback.getValue) + ArgumentMatchers.eq(postVerificationCallback.getValue()) ) verify(replicaManager).appendRecordsAfterVerification( any(), any(), - ArgumentMatchers.eq(postVerificationCallback.getValue))(ArgumentMatchers.eq(newRequestLocal), any()) + ArgumentMatchers.eq(postVerificationCallback.getValue()))(ArgumentMatchers.eq(newRequestLocal), any()) verify(replicaManager).appendRecords(anyLong, anyShort, From 9bac9a9a5ff8899ef78cc8b77748b4da9c74982e Mon Sep 17 00:00:00 2001 From: Justine Date: Fri, 17 Nov 2023 13:16:32 -0800 Subject: [PATCH 04/24] Fix test failures --- .../main/scala/kafka/server/KafkaApis.scala | 18 +++++++++++------- .../unit/kafka/server/KafkaApisTest.scala | 12 +++--------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 129d4b279ad8f..89461d513806c 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -730,13 +730,17 @@ class KafkaApis(val requestChannel: RequestChannel, sendResponseCallback(Map.empty) else { // call the replica manager to append messages to the replicas - replicaManager.appendRecordsWithVerification( - entriesPerPartition = authorizedRequestInfo, - transactionVerificationEntries = transactionVerificationEntries, - transactionalId = produceRequest.transactionalId, - requestLocal = requestLocal, - postVerificationCallback = postVerificationCallback - ) + if (produceRequest.transactionalId == null){ + postVerificationCallback(requestLocal)(Map.empty, Map.empty) + } else { + replicaManager.appendRecordsWithVerification( + entriesPerPartition = authorizedRequestInfo, + transactionVerificationEntries = transactionVerificationEntries, + transactionalId = produceRequest.transactionalId, + requestLocal = requestLocal, + postVerificationCallback = postVerificationCallback + ) + } // if the request is put into the purgatory, it will have a held reference and hence cannot be garbage collected; // hence we clear its data here in order to let GC reclaim its memory since it is already appended to log diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 1ad9886bd5a5a..a264b29c47c42 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -2446,15 +2446,6 @@ class KafkaApisTest { .build(version.toShort) val request = buildRequest(produceRequest) - val postVerificationCallback: ArgumentCaptor[RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit] = ArgumentCaptor.forClass( - classOf[RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit]) - when(replicaManager.appendRecordsWithVerification(any(), any(), any(), any(), postVerificationCallback.capture())).thenAnswer( - _ => { - val callback = postVerificationCallback.getValue() - callback(RequestLocal.NoCaching)(Map.empty, Map.empty) - } - ) - when(replicaManager.appendRecords(anyLong, anyShort, ArgumentMatchers.eq(false), @@ -2525,6 +2516,7 @@ class KafkaApisTest { any(), any(), any(), + any(), any()) ).thenAnswer(_ => responseCallback.getValue.apply(Map(tp -> new PartitionResponse(Errors.NOT_LEADER_OR_FOLLOWER)))) @@ -2591,6 +2583,7 @@ class KafkaApisTest { any(), any(), any(), + any(), any()) ).thenAnswer(_ => responseCallback.getValue.apply(Map(tp -> new PartitionResponse(Errors.NOT_LEADER_OR_FOLLOWER)))) @@ -2656,6 +2649,7 @@ class KafkaApisTest { any(), any(), any(), + any(), any()) ).thenAnswer(_ => responseCallback.getValue.apply(Map(tp -> new PartitionResponse(Errors.NOT_LEADER_OR_FOLLOWER)))) From b4a920b49c12ff2d7c9d35f07d4af13f4713c20f Mon Sep 17 00:00:00 2001 From: Justine Date: Wed, 29 Nov 2023 15:04:26 -0800 Subject: [PATCH 05/24] Rewrite GroupCoordinator and GroupMetadataManager to handle checks for txnOffsetCommits --- .../coordinator/group/GroupCoordinator.scala | 93 ++++-- .../group/GroupMetadataManager.scala | 295 ++++++++++-------- .../main/scala/kafka/server/KafkaApis.scala | 6 +- .../scala/kafka/server/ReplicaManager.scala | 42 +-- .../AbstractCoordinatorConcurrencyTest.scala | 4 +- .../group/GroupCoordinatorTest.scala | 57 +++- .../group/GroupMetadataManagerTest.scala | 81 +++-- .../unit/kafka/server/KafkaApisTest.scala | 11 +- .../kafka/server/ReplicaManagerTest.scala | 13 +- 9 files changed, 346 insertions(+), 256 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala index 0f546a4744129..5f23915565dd4 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala @@ -19,6 +19,7 @@ package kafka.coordinator.group import java.util.{OptionalInt, Properties} import java.util.concurrent.atomic.AtomicBoolean import kafka.common.OffsetAndMetadata +import kafka.server.ReplicaManager.TransactionVerificationEntries import kafka.server._ import kafka.utils.Logging import org.apache.kafka.common.{TopicIdPartition, TopicPartition} @@ -909,8 +910,68 @@ private[group] class GroupCoordinator( val group = groupManager.getGroup(groupId).getOrElse { groupManager.addGroup(new GroupMetadata(groupId, Empty, time)) } - doTxnCommitOffsets(group, transactionalId, memberId, groupInstanceId, generationId, producerId, producerEpoch, - offsetMetadata, requestLocal, responseCallback) + + val filteredOffsetMetadata = offsetMetadata.filter { case (_, offsetAndMetadata) => + groupManager.validateOffsetMetadataLength(offsetAndMetadata.metadata) + } + if (filteredOffsetMetadata.isEmpty) { + // compute the final error codes for the commit response + val commitStatus = offsetMetadata.map { case (k, _) => k -> Errors.OFFSET_METADATA_TOO_LARGE } + responseCallback(commitStatus) + return + } + + val magicOpt = groupManager.getMagic(partitionFor(group.groupId)) + if (magicOpt.isEmpty) { + val commitStatus = offsetMetadata.map { case (topicIdPartition, _) => + (topicIdPartition, Errors.NOT_COORDINATOR) + } + responseCallback(commitStatus) + return + } + + val records = groupManager.generateOffsetRecords(magicOpt.get, true, group.groupId, filteredOffsetMetadata, producerId, producerEpoch) + val transactionVerificationEntries = new TransactionVerificationEntries + + def postVerificationCallback(newRequestLocal: RequestLocal) + (errorResults: Map[TopicPartition, LogAppendResult]): Unit = { + group.inLock { + val validationErrorOpt = validateOffsetCommit( + group, + generationId, + memberId, + groupInstanceId, + isTransactional = true + ) + + val verifiedOffsets = offsetMetadata.filter { + case (tp, _) => + !errorResults.contains(tp.topicPartition) + } + + val verifiedRecords = records.filter { + case (tp, _) => + !errorResults.contains(tp) + } + + if (validationErrorOpt.isDefined) { + responseCallback(offsetMetadata.map { case (k, _) => k -> validationErrorOpt.get }) + } else if (verifiedOffsets.isEmpty) { + responseCallback(offsetMetadata.map { case (k, _) => k -> errorResults(k.topicPartition).error }) + } else { + val putCacheCallback = groupManager.createPutCacheCallback(true, group, memberId, offsetMetadata, verifiedOffsets, responseCallback, producerId, verifiedRecords, errorResults) + groupManager.storeOffsetsAfterVerification(group, verifiedOffsets, records, putCacheCallback, producerId, errorResults, newRequestLocal) + } + } + } + + groupManager.replicaManager.appendRecordsWithVerification( + entriesPerPartition = records, + transactionVerificationEntries = transactionVerificationEntries, + transactionalId = transactionalId, + requestLocal = requestLocal, + postVerificationCallback = postVerificationCallback + ) } } @@ -951,34 +1012,6 @@ private[group] class GroupCoordinator( groupManager.scheduleHandleTxnCompletion(producerId, offsetsPartitions.map(_.partition).toSet, isCommit) } - private def doTxnCommitOffsets(group: GroupMetadata, - transactionalId: String, - memberId: String, - groupInstanceId: Option[String], - generationId: Int, - producerId: Long, - producerEpoch: Short, - offsetMetadata: immutable.Map[TopicIdPartition, OffsetAndMetadata], - requestLocal: RequestLocal, - responseCallback: immutable.Map[TopicIdPartition, Errors] => Unit): Unit = { - group.inLock { - val validationErrorOpt = validateOffsetCommit( - group, - generationId, - memberId, - groupInstanceId, - isTransactional = true - ) - - if (validationErrorOpt.isDefined) { - responseCallback(offsetMetadata.map { case (k, _) => k -> validationErrorOpt.get }) - } else { - groupManager.storeOffsets(group, memberId, offsetMetadata, responseCallback, transactionalId, producerId, - producerEpoch, requestLocal) - } - } - } - private def validateOffsetCommit( group: GroupMetadata, generationId: Int, diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala index 76020476fb8c1..cae2a399afbb3 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala @@ -344,6 +344,122 @@ class GroupMetadataManager(brokerId: Int, preAppendErrors = preAppendErrors) } + def generateOffsetRecords(magicValue: Byte, + isTxnOffsetCommit: Boolean, + groupId: String, + filteredOffsetMetadata: Map[TopicIdPartition, OffsetAndMetadata], + producerId: Long, + producerEpoch: Short): Map[TopicPartition, MemoryRecords] = { + // We always use CREATE_TIME, like the producer. The conversion to LOG_APPEND_TIME (if necessary) happens automatically. + val timestampType = TimestampType.CREATE_TIME + val timestamp = time.milliseconds() + + val records = filteredOffsetMetadata.map { case (topicIdPartition, offsetAndMetadata) => + val key = GroupMetadataManager.offsetCommitKey(groupId, topicIdPartition.topicPartition) + val value = GroupMetadataManager.offsetCommitValue(offsetAndMetadata, interBrokerProtocolVersion) + new SimpleRecord(timestamp, key, value) + } + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, partitionFor(groupId)) + val buffer = ByteBuffer.allocate(AbstractRecords.estimateSizeInBytes(magicValue, compressionType, records.asJava)) + + if (isTxnOffsetCommit && magicValue < RecordBatch.MAGIC_VALUE_V2) + throw Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT.exception("Attempting to make a transaction offset commit with an invalid magic: " + magicValue) + + val builder = MemoryRecords.builder(buffer, magicValue, compressionType, timestampType, 0L, time.milliseconds(), + producerId, producerEpoch, 0, isTxnOffsetCommit, RecordBatch.NO_PARTITION_LEADER_EPOCH) + + records.foreach(builder.append) + Map(offsetTopicPartition -> builder.build()) + } + + def createPutCacheCallback(isTxnOffsetCommit: Boolean, + group: GroupMetadata, + consumerId: String, + offsetMetadata: immutable.Map[TopicIdPartition, OffsetAndMetadata], + filteredOffsetMetadata: Map[TopicIdPartition, OffsetAndMetadata], + responseCallback: immutable.Map[TopicIdPartition, Errors] => Unit, + producerId: Long = RecordBatch.NO_PRODUCER_ID, + records: Map[TopicPartition, MemoryRecords], + preAppendErrors: Map[TopicPartition, LogAppendResult] = Map.empty): Map[TopicPartition, PartitionResponse] => Unit = { + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, partitionFor(group.groupId)) + // set the callback function to insert offsets into cache after log append completed + def putCacheCallback(responseStatus: Map[TopicPartition, PartitionResponse]): Unit = { + // the append response should only contain the topics partition + if (responseStatus.size != 1 || !responseStatus.contains(offsetTopicPartition)) + throw new IllegalStateException("Append status %s should only have one partition %s" + .format(responseStatus, offsetTopicPartition)) + + // construct the commit response status and insert + // the offset and metadata to cache if the append status has no error + val status = responseStatus(offsetTopicPartition) + + val responseError = group.inLock { + if (status.error == Errors.NONE) { + if (!group.is(Dead)) { + filteredOffsetMetadata.forKeyValue { (topicIdPartition, offsetAndMetadata) => + if (isTxnOffsetCommit) + group.onTxnOffsetCommitAppend(producerId, topicIdPartition, CommitRecordMetadataAndOffset(Some(status.baseOffset), offsetAndMetadata)) + else + group.onOffsetCommitAppend(topicIdPartition, CommitRecordMetadataAndOffset(Some(status.baseOffset), offsetAndMetadata)) + } + } + + // Record the number of offsets committed to the log + offsetCommitsSensor.record(records.size) + + Errors.NONE + } else { + if (!group.is(Dead)) { + if (!group.hasPendingOffsetCommitsFromProducer(producerId)) + removeProducerGroup(producerId, group.groupId) + filteredOffsetMetadata.forKeyValue { (topicIdPartition, offsetAndMetadata) => + if (isTxnOffsetCommit) + group.failPendingTxnOffsetCommit(producerId, topicIdPartition) + else + group.failPendingOffsetWrite(topicIdPartition, offsetAndMetadata) + } + } + + debug(s"Offset commit $filteredOffsetMetadata from group ${group.groupId}, consumer $consumerId " + + s"with generation ${group.generationId} failed when appending to log due to ${status.error.exceptionName}") + + // transform the log append error code to the corresponding the commit status error code + status.error match { + case Errors.UNKNOWN_TOPIC_OR_PARTITION + | Errors.NOT_ENOUGH_REPLICAS + | Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND => + Errors.COORDINATOR_NOT_AVAILABLE + + case Errors.NOT_LEADER_OR_FOLLOWER + | Errors.KAFKA_STORAGE_ERROR => + Errors.NOT_COORDINATOR + + case Errors.MESSAGE_TOO_LARGE + | Errors.RECORD_LIST_TOO_LARGE + | Errors.INVALID_FETCH_SIZE => + Errors.INVALID_COMMIT_OFFSET_SIZE + + case other => other + } + } + } + + // compute the final error codes for the commit response + val commitStatus = offsetMetadata.map { case (topicIdPartition, offsetAndMetadata) => + if (!validateOffsetMetadataLength(offsetAndMetadata.metadata)) + (topicIdPartition, Errors.OFFSET_METADATA_TOO_LARGE) + else if (preAppendErrors.contains(topicIdPartition.topicPartition)) + (topicIdPartition, preAppendErrors(topicIdPartition.topicPartition).error) + else + (topicIdPartition, responseError) + } + + // finally trigger the callback logic passed from the API layer + responseCallback(commitStatus) + } + putCacheCallback + } + /** * Store offsets by appending it to the replicated log and then inserting to cache */ @@ -351,15 +467,9 @@ class GroupMetadataManager(brokerId: Int, consumerId: String, offsetMetadata: immutable.Map[TopicIdPartition, OffsetAndMetadata], responseCallback: immutable.Map[TopicIdPartition, Errors] => Unit, - transactionalId: String = null, producerId: Long = RecordBatch.NO_PRODUCER_ID, producerEpoch: Short = RecordBatch.NO_PRODUCER_EPOCH, requestLocal: RequestLocal = RequestLocal.NoCaching): Unit = { - // first filter out partitions with offset metadata size exceeding limit - val filteredOffsetMetadata = offsetMetadata.filter { case (_, offsetAndMetadata) => - validateOffsetMetadataLength(offsetAndMetadata.metadata) - } - group.inLock { if (!group.hasReceivedConsistentOffsetCommits) warn(s"group: ${group.groupId} with leader: ${group.leaderOrNull} has received offset commits from consumers as well " + @@ -367,145 +477,58 @@ class GroupMetadataManager(brokerId: Int, s"should be avoided.") } - val isTxnOffsetCommit = producerId != RecordBatch.NO_PRODUCER_ID - // construct the message set to append + val filteredOffsetMetadata = offsetMetadata.filter { case (_, offsetAndMetadata) => + validateOffsetMetadataLength(offsetAndMetadata.metadata) + } if (filteredOffsetMetadata.isEmpty) { // compute the final error codes for the commit response val commitStatus = offsetMetadata.map { case (k, _) => k -> Errors.OFFSET_METADATA_TOO_LARGE } responseCallback(commitStatus) - } else { - getMagic(partitionFor(group.groupId)) match { - case Some(magicValue) => - // We always use CREATE_TIME, like the producer. The conversion to LOG_APPEND_TIME (if necessary) happens automatically. - val timestampType = TimestampType.CREATE_TIME - val timestamp = time.milliseconds() - - val records = filteredOffsetMetadata.map { case (topicIdPartition, offsetAndMetadata) => - val key = GroupMetadataManager.offsetCommitKey(group.groupId, topicIdPartition.topicPartition) - val value = GroupMetadataManager.offsetCommitValue(offsetAndMetadata, interBrokerProtocolVersion) - new SimpleRecord(timestamp, key, value) - } - val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, partitionFor(group.groupId)) - val buffer = ByteBuffer.allocate(AbstractRecords.estimateSizeInBytes(magicValue, compressionType, records.asJava)) - - if (isTxnOffsetCommit && magicValue < RecordBatch.MAGIC_VALUE_V2) - throw Errors.UNSUPPORTED_FOR_MESSAGE_FORMAT.exception("Attempting to make a transaction offset commit with an invalid magic: " + magicValue) - - val builder = MemoryRecords.builder(buffer, magicValue, compressionType, timestampType, 0L, time.milliseconds(), - producerId, producerEpoch, 0, isTxnOffsetCommit, RecordBatch.NO_PARTITION_LEADER_EPOCH) - - records.foreach(builder.append) - val entries = Map(offsetTopicPartition -> builder.build()) - - // set the callback function to insert offsets into cache after log append completed - def putCacheCallback(responseStatus: Map[TopicPartition, PartitionResponse]): Unit = { - // the append response should only contain the topics partition - if (responseStatus.size != 1 || !responseStatus.contains(offsetTopicPartition)) - throw new IllegalStateException("Append status %s should only have one partition %s" - .format(responseStatus, offsetTopicPartition)) - - // construct the commit response status and insert - // the offset and metadata to cache if the append status has no error - val status = responseStatus(offsetTopicPartition) - - val responseError = group.inLock { - if (status.error == Errors.NONE) { - if (!group.is(Dead)) { - filteredOffsetMetadata.forKeyValue { (topicIdPartition, offsetAndMetadata) => - if (isTxnOffsetCommit) - group.onTxnOffsetCommitAppend(producerId, topicIdPartition, CommitRecordMetadataAndOffset(Some(status.baseOffset), offsetAndMetadata)) - else - group.onOffsetCommitAppend(topicIdPartition, CommitRecordMetadataAndOffset(Some(status.baseOffset), offsetAndMetadata)) - } - } - - // Record the number of offsets committed to the log - offsetCommitsSensor.record(records.size) - - Errors.NONE - } else { - if (!group.is(Dead)) { - if (!group.hasPendingOffsetCommitsFromProducer(producerId)) - removeProducerGroup(producerId, group.groupId) - filteredOffsetMetadata.forKeyValue { (topicIdPartition, offsetAndMetadata) => - if (isTxnOffsetCommit) - group.failPendingTxnOffsetCommit(producerId, topicIdPartition) - else - group.failPendingOffsetWrite(topicIdPartition, offsetAndMetadata) - } - } - - debug(s"Offset commit $filteredOffsetMetadata from group ${group.groupId}, consumer $consumerId " + - s"with generation ${group.generationId} failed when appending to log due to ${status.error.exceptionName}") - - // transform the log append error code to the corresponding the commit status error code - status.error match { - case Errors.UNKNOWN_TOPIC_OR_PARTITION - | Errors.NOT_ENOUGH_REPLICAS - | Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND => - Errors.COORDINATOR_NOT_AVAILABLE - - case Errors.NOT_LEADER_OR_FOLLOWER - | Errors.KAFKA_STORAGE_ERROR => - Errors.NOT_COORDINATOR - - case Errors.MESSAGE_TOO_LARGE - | Errors.RECORD_LIST_TOO_LARGE - | Errors.INVALID_FETCH_SIZE => - Errors.INVALID_COMMIT_OFFSET_SIZE - - case other => other - } - } - } + return + } - // compute the final error codes for the commit response - val commitStatus = offsetMetadata.map { case (topicIdPartition, offsetAndMetadata) => - if (validateOffsetMetadataLength(offsetAndMetadata.metadata)) - (topicIdPartition, responseError) - else - (topicIdPartition, Errors.OFFSET_METADATA_TOO_LARGE) - } + val magicOpt = getMagic(partitionFor(group.groupId)) + if (magicOpt.isEmpty) { + val commitStatus = offsetMetadata.map { case (topicIdPartition, _) => + (topicIdPartition, Errors.NOT_COORDINATOR) + } + responseCallback(commitStatus) + return + } - // finally trigger the callback logic passed from the API layer - responseCallback(commitStatus) - } + val isTxnOffsetCommit = producerId != RecordBatch.NO_PRODUCER_ID + val records = generateOffsetRecords(magicOpt.get, isTxnOffsetCommit, group.groupId, filteredOffsetMetadata, producerId, producerEpoch) + val putCacheCallback = createPutCacheCallback(isTxnOffsetCommit, group, consumerId, offsetMetadata, filteredOffsetMetadata, responseCallback, producerId, records) - val transactionVerificationEntries = new ReplicaManager.TransactionVerificationEntries - def postVerificationCallback(newRequestLocal: RequestLocal) - (newlyVerifiedEntries: Map[TopicPartition, MemoryRecords], errorResults: Map[TopicPartition, LogAppendResult]): Unit = { + group.inLock { + group.prepareOffsetCommit(offsetMetadata) + } - group.inLock { - if (isTxnOffsetCommit) { - addProducerGroup(producerId, group.groupId) - group.prepareTxnOffsetCommit(producerId, offsetMetadata) - } else { - group.prepareOffsetCommit(offsetMetadata) - } + appendForGroup(group, records, requestLocal, putCacheCallback) + } - appendForGroup(group, newlyVerifiedEntries ++ transactionVerificationEntries.verified, newRequestLocal, - putCacheCallback, transactionVerificationEntries.verificationGuards.toMap, errorResults) - } - } - if (transactionalId != null ) { - replicaManager.appendRecordsWithVerification( - entriesPerPartition = entries, - transactionVerificationEntries = transactionVerificationEntries, - transactionalId = transactionalId, - requestLocal = requestLocal, - postVerificationCallback = postVerificationCallback - ) - } else { - postVerificationCallback(requestLocal)(entries, Map.empty) - } + def storeOffsetsAfterVerification(group: GroupMetadata, + verifiedOffsetMetadata: immutable.Map[TopicIdPartition, OffsetAndMetadata], + records: Map[TopicPartition, MemoryRecords], + putCacheCallback: Map[TopicPartition, PartitionResponse] => Unit, + producerId: Long, + errorResults: Map[TopicPartition, LogAppendResult], + requestLocal: RequestLocal = RequestLocal.NoCaching): Unit = { + group.inLock { + if (!group.hasReceivedConsistentOffsetCommits) + warn(s"group: ${group.groupId} with leader: ${group.leaderOrNull} has received offset commits from consumers as well " + + s"as transactional producers. Mixing both types of offset commits will generally result in surprises and " + + s"should be avoided.") + } - case None => - val commitStatus = offsetMetadata.map { case (topicIdPartition, _) => - (topicIdPartition, Errors.NOT_COORDINATOR) - } - responseCallback(commitStatus) - } + val transactionVerificationEntries = new ReplicaManager.TransactionVerificationEntries + group.inLock { + addProducerGroup(producerId, group.groupId) + group.prepareTxnOffsetCommit(producerId, verifiedOffsetMetadata) } + + appendForGroup(group, records, requestLocal, + putCacheCallback, transactionVerificationEntries.verificationGuards.toMap, errorResults) } /** @@ -994,7 +1017,7 @@ class GroupMetadataManager(brokerId: Int, /* * Check if the offset metadata length is valid */ - private def validateOffsetMetadataLength(metadata: String) : Boolean = { + protected[coordinator] def validateOffsetMetadataLength(metadata: String) : Boolean = { metadata == null || metadata.length() <= config.maxMetadataSize } @@ -1015,7 +1038,7 @@ class GroupMetadataManager(brokerId: Int, * @param partition Partition of GroupMetadataTopic * @return Some(MessageFormatVersion) if replica is local, None otherwise */ - private def getMagic(partition: Int): Option[Byte] = + protected[coordinator] def getMagic(partition: Int): Option[Byte] = replicaManager.getMagic(new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, partition)) /** diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 89461d513806c..9427bf745d28c 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -711,13 +711,13 @@ class KafkaApis(val requestChannel: RequestChannel, val transactionVerificationEntries = new ReplicaManager.TransactionVerificationEntries def postVerificationCallback(newRequestLocal: RequestLocal) - (newlyVerifiedEntries: Map[TopicPartition, MemoryRecords], errorResults: Map[TopicPartition, LogAppendResult]): Unit = { + (errorResults: Map[TopicPartition, LogAppendResult]): Unit = { replicaManager.appendRecords( timeout = produceRequest.timeout.toLong, requiredAcks = produceRequest.acks, internalTopicsAllowed = internalTopicsAllowed, origin = AppendOrigin.CLIENT, - entriesPerPartition = newlyVerifiedEntries ++ transactionVerificationEntries.verified, + entriesPerPartition = authorizedRequestInfo, responseCallback = sendResponseCallback, recordConversionStatsCallback = processingStatsCallback, requestLocal = newRequestLocal, @@ -731,7 +731,7 @@ class KafkaApis(val requestChannel: RequestChannel, else { // call the replica manager to append messages to the replicas if (produceRequest.transactionalId == null){ - postVerificationCallback(requestLocal)(Map.empty, Map.empty) + postVerificationCallback(requestLocal)(Map.empty) } else { replicaManager.appendRecordsWithVerification( entriesPerPartition = authorizedRequestInfo, diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index 6315be62c1c8d..a32b6ef0bc943 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -250,8 +250,6 @@ object ReplicaManager { } class TransactionVerificationEntries { - - val verified = mutable.Map[TopicPartition, MemoryRecords]() val unverified = mutable.Map[TopicPartition, MemoryRecords]() val errors = mutable.Map[TopicPartition, Errors]() val verificationGuards = mutable.Map[TopicPartition, VerificationGuard]() @@ -771,9 +769,13 @@ class ReplicaManager(val config: KafkaConfig, return } + val nonErrorEntriesPerPartition = entriesPerPartition.filter { + case (tp, _) => !preAppendErrors.contains(tp) + } + val sTime = time.milliseconds val localProduceResults = appendToLocalLog(internalTopicsAllowed = internalTopicsAllowed, - origin, entriesPerPartition, requiredAcks, requestLocal, verificationGuards.toMap) + origin, nonErrorEntriesPerPartition, requiredAcks, requestLocal, verificationGuards.toMap) debug("Produce to local log in %d ms".format(time.milliseconds - sTime)) val allResults = localProduceResults ++ preAppendErrors @@ -872,15 +874,15 @@ class ReplicaManager(val config: KafkaConfig, transactionVerificationEntries: TransactionVerificationEntries, transactionalId: String, requestLocal: RequestLocal, - postVerificationCallback: RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit): Unit = { + postVerificationCallback: RequestLocal => Map[TopicPartition, LogAppendResult] => Unit): Unit = { if (transactionalId != null && config.transactionPartitionVerificationEnable && addPartitionsToTxnManager.isDefined) partitionEntriesForVerification(transactionVerificationEntries, entriesPerPartition) - val onVerificationComplete: (RequestLocal, Map[TopicPartition, Errors]) => Unit = appendRecordsAfterVerification( - entriesPerPartition, - transactionVerificationEntries, - postVerificationCallback, - ) + val onVerificationComplete: (RequestLocal, Map[TopicPartition, Errors]) => Unit = + appendRecordsAfterVerification( + transactionVerificationEntries, + postVerificationCallback, + ) if (transactionVerificationEntries.unverified.isEmpty) { onVerificationComplete(requestLocal, transactionVerificationEntries.errors.toMap) @@ -921,18 +923,9 @@ class ReplicaManager(val config: KafkaConfig, * @param requestLocal container for the stateful instances scoped to this request * @param unverifiedEntries the records per partition for topic partitions that were not verified by the transaction coordinator */ - def appendRecordsAfterVerification(allEntries: Map[TopicPartition, MemoryRecords], - transactionVerificationEntries: TransactionVerificationEntries, - postVerificationCallback: RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit) + def appendRecordsAfterVerification(transactionVerificationEntries: TransactionVerificationEntries, + postVerificationCallback: RequestLocal => Map[TopicPartition, LogAppendResult] => Unit) (requestLocal: RequestLocal, unverifiedEntries: Map[TopicPartition, Errors] = Map.empty): Unit = { - val verifiedEntries = - if (unverifiedEntries.isEmpty) - allEntries - else - allEntries.filter { case (tp, _) => - !unverifiedEntries.contains(tp) - } - val errorResults = (unverifiedEntries ++ transactionVerificationEntries.errors).map { case (topicPartition, error) => // translate transaction coordinator errors to known producer response errors @@ -952,7 +945,7 @@ class ReplicaManager(val config: KafkaConfig, hasCustomErrorMessage = customException.isDefined ) } - postVerificationCallback(requestLocal)(verifiedEntries, errorResults) + postVerificationCallback(requestLocal)(errorResults) } private def partitionEntriesForVerification(transactionVerificationEntries: TransactionVerificationEntries, entriesPerPartition: Map[TopicPartition, MemoryRecords]): TransactionVerificationEntries = { @@ -963,6 +956,7 @@ class ReplicaManager(val config: KafkaConfig, val transactionalBatches = records.batches.asScala.filter(batch => batch.hasProducerId && batch.isTransactional) transactionalBatches.foreach(batch => transactionalProducerIds.add(batch.producerId)) + // If there is no producer ID or transactional records in the batches, no need to verify. if (transactionalBatches.nonEmpty) { // We return VerificationGuard if the partition needs to be verified. If no state is present, no need to verify. val firstBatch = records.firstBatch @@ -970,11 +964,7 @@ class ReplicaManager(val config: KafkaConfig, if (verificationGuard != VerificationGuard.SENTINEL) { transactionVerificationEntries.verificationGuards.put(topicPartition, verificationGuard) transactionVerificationEntries.unverified.put(topicPartition, records) - } else - transactionVerificationEntries.verified.put(topicPartition, records) - } else { - // If there is no producer ID or transactional records in the batches, no need to verify. - transactionVerificationEntries.verified.put(topicPartition, records) + } } } catch { case e: Exception => transactionVerificationEntries.errors.put(topicPartition, Errors.forException(e)) diff --git a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala index 463faf7a9d7b9..e0efdc4ef99f6 100644 --- a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala @@ -215,9 +215,9 @@ object AbstractCoordinatorConcurrencyTest { transactionVerificationEntries: TransactionVerificationEntries, transactionalId: String, requestLocal: RequestLocal, - postVerificationCallback: RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit): Unit = { + postVerificationCallback: RequestLocal => Map[TopicPartition, LogAppendResult] => Unit): Unit = { - postVerificationCallback(requestLocal)(entriesPerPartition, Map.empty) + postVerificationCallback(requestLocal)(Map.empty) } diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala index adf7f12869466..ac2d0c99834ba 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala @@ -38,7 +38,7 @@ import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity import org.apache.kafka.server.util.timer.MockTimer import org.apache.kafka.server.util.{KafkaScheduler, MockTime} -import org.apache.kafka.storage.internals.log.AppendOrigin +import org.apache.kafka.storage.internals.log.{AppendOrigin, LogAppendInfo} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} import org.junit.jupiter.params.ParameterizedTest @@ -3782,6 +3782,45 @@ class GroupCoordinatorTest { assertTrue(groupCoordinator.tryCompleteHeartbeat(group, leaderMemberId, false, () => true)) } + @Test + def testVerificationErrorsForTxnOffsetCommits(): Unit = { + val tip1 = new TopicIdPartition(Uuid.randomUuid(), 0, "topic-1") + val offset1 = offsetAndMetadata(0) + val tip2 = new TopicIdPartition(Uuid.randomUuid(), 0, "topic-2") + val offset2 = offsetAndMetadata(0) + val producerId = 1000L + val producerEpoch: Short = 2 + + def verifyErrors(errors: Map[TopicIdPartition, Errors]): Unit = { + val commitOffsetResult = commitTransactionalOffsets(groupId, + producerId, + producerEpoch, + Map(tip1 -> offset1, tip2 -> offset2), + verificationErrors = errors) + errors.foreach { + case (tip, error) => assertEquals(error, commitOffsetResult(tip)) + } + } + + verifyErrors(Map(tip1 -> Errors.INVALID_TXN_STATE, tip2 -> Errors.INVALID_PRODUCER_ID_MAPPING)) + verifyErrors(Map(tip2 -> Errors.INVALID_TXN_STATE)) + } + + @Test + def testTxnOffsetMetadataTooLarge(): Unit = { + val tip = new TopicIdPartition(Uuid.randomUuid(), 0, "foo") + val offset = 37 + val producerId = 100L + val producerEpoch: Short = 3 + + val offsets = Map( + tip -> OffsetAndMetadata(offset, "s" * (OffsetConfig.DefaultMaxMetadataSize + 1), 0) + ) + + val commitOffsetResult = commitTransactionalOffsets(groupId, producerId, producerEpoch, offsets) + assertEquals(Map(tip -> Errors.OFFSET_METADATA_TOO_LARGE), commitOffsetResult) + } + private def getGroup(groupId: String): GroupMetadata = { val groupOpt = groupCoordinator.groupManager.getGroup(groupId) assertTrue(groupOpt.isDefined) @@ -4070,7 +4109,8 @@ class GroupCoordinatorTest { offsets: Map[TopicIdPartition, OffsetAndMetadata], memberId: String = JoinGroupRequest.UNKNOWN_MEMBER_ID, groupInstanceId: Option[String] = Option.empty, - generationId: Int = JoinGroupRequest.UNKNOWN_GENERATION_ID) : CommitOffsetCallbackParams = { + generationId: Int = JoinGroupRequest.UNKNOWN_GENERATION_ID, + verificationErrors: Map[TopicIdPartition, Errors] = Map.empty[TopicIdPartition, Errors]) : CommitOffsetCallbackParams = { val (responseFuture, responseCallback) = setupCommitOffsetsCallback val capturedArgument: ArgumentCaptor[scala.collection.Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[scala.collection.Map[TopicPartition, PartitionResponse] => Unit]) @@ -4078,10 +4118,17 @@ class GroupCoordinatorTest { // Since transactional ID is only used in appendRecords, we can use a dummy value. Ensure it passes through. val transactionalId = "dummy-txn-id" - val postVerificationCallback: ArgumentCaptor[RequestLocal => (SMap[TopicPartition, MemoryRecords], SMap[TopicPartition, LogAppendResult]) => Unit] = ArgumentCaptor.forClass( - classOf[RequestLocal => (SMap[TopicPartition, MemoryRecords], SMap[TopicPartition, LogAppendResult]) => Unit]) + val preAppendErrors = verificationErrors.map { case (tp, error) => + tp.topicPartition -> LogAppendResult( + LogAppendInfo.UNKNOWN_LOG_APPEND_INFO, + Some(error.exception()), + hasCustomErrorMessage = false) + } + + val postVerificationCallback: ArgumentCaptor[RequestLocal => SMap[TopicPartition, LogAppendResult] => Unit] = ArgumentCaptor.forClass( + classOf[RequestLocal => SMap[TopicPartition, LogAppendResult] => Unit]) when(replicaManager.appendRecordsWithVerification(any(), any(), ArgumentMatchers.eq(transactionalId), any(), postVerificationCallback.capture())).thenAnswer( - _ => postVerificationCallback.getValue()(RequestLocal.NoCaching)(Map.empty, Map.empty) + _ => postVerificationCallback.getValue()(RequestLocal.NoCaching)(preAppendErrors) ) when(replicaManager.appendRecords(anyLong, anyShort(), diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala index 7896fcb4ad613..b03c17475865f 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala @@ -57,7 +57,7 @@ import org.mockito.ArgumentMatchers.{any, anyInt, anyLong, anyShort} import org.mockito.Mockito.{mock, reset, times, verify, when} import scala.jdk.CollectionConverters._ -import scala.collection._ +import scala.collection.{immutable, _} class GroupMetadataManagerTest { @@ -1316,7 +1316,6 @@ class GroupMetadataManagerTest { val offset = 37 val producerId = 232L val producerEpoch = 0.toShort - val transactionalId = "txnId" groupMetadataManager.addOwnedPartition(groupPartitionId) @@ -1327,14 +1326,12 @@ class GroupMetadataManagerTest { val offsets = immutable.Map(topicIdPartition -> offsetAndMetadata) val capturedResponseCallback: ArgumentCaptor[Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[Map[TopicPartition, PartitionResponse] => Unit]) - when(replicaManager.getMagic(any())).thenReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE)) var commitErrors: Option[immutable.Map[TopicIdPartition, Errors]] = None def callback(errors: immutable.Map[TopicIdPartition, Errors]): Unit = { commitErrors = Some(errors) } - setUpTransactionVerification(replicaManager, transactionalId) - groupMetadataManager.storeOffsets(group, memberId, offsets, callback, transactionalId, producerId, producerEpoch) + storeOffsetsWithVerification(group, memberId, offsets, callback, producerId, producerEpoch) assertTrue(group.hasOffsets) assertTrue(group.allOffsets.isEmpty) @@ -1350,7 +1347,6 @@ class GroupMetadataManagerTest { any(), any(), any()) - verify(replicaManager).getMagic(any()) capturedResponseCallback.getValue.apply(Map(groupTopicPartition -> new PartitionResponse(Errors.NONE, 0L, RecordBatch.NO_TIMESTAMP, 0L))) @@ -1370,7 +1366,6 @@ class GroupMetadataManagerTest { val offset = 37 val producerId = 232L val producerEpoch = 0.toShort - val transactionalId = "txnId" groupMetadataManager.addOwnedPartition(groupPartitionId) @@ -1379,15 +1374,12 @@ class GroupMetadataManagerTest { val offsets = immutable.Map(topicIdPartition -> OffsetAndMetadata(offset, "", time.milliseconds())) - - when(replicaManager.getMagic(any())).thenReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE)) var commitErrors: Option[immutable.Map[TopicIdPartition, Errors]] = None def callback(errors: immutable.Map[TopicIdPartition, Errors]): Unit = { commitErrors = Some(errors) } - setUpTransactionVerification(replicaManager, transactionalId) - groupMetadataManager.storeOffsets(group, memberId, offsets, callback, transactionalId, producerId, producerEpoch) + storeOffsetsWithVerification(group, memberId, offsets, callback, producerId, producerEpoch) assertTrue(group.hasOffsets) assertTrue(group.allOffsets.isEmpty) val capturedResponseCallback = verifyAppendAndCaptureCallback() @@ -1413,7 +1405,6 @@ class GroupMetadataManagerTest { any(), any(), any()) - verify(replicaManager).getMagic(any()) } @Test @@ -1423,7 +1414,6 @@ class GroupMetadataManagerTest { val offset = 37 val producerId = 232L val producerEpoch = 0.toShort - val transactionalId = "txnId" groupMetadataManager.addOwnedPartition(groupPartitionId) @@ -1432,15 +1422,12 @@ class GroupMetadataManagerTest { val offsets = immutable.Map(topicIdPartition -> OffsetAndMetadata(offset, "", time.milliseconds())) - when(replicaManager.getMagic(any())).thenReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE)) - var commitErrors: Option[immutable.Map[TopicIdPartition, Errors]] = None def callback(errors: immutable.Map[TopicIdPartition, Errors]): Unit = { commitErrors = Some(errors) } - setUpTransactionVerification(replicaManager, transactionalId) - groupMetadataManager.storeOffsets(group, memberId, offsets, callback, transactionalId, producerId, producerEpoch) + storeOffsetsWithVerification(group, memberId, offsets, callback, producerId, producerEpoch) assertTrue(group.hasOffsets) assertTrue(group.allOffsets.isEmpty) val capturedResponseCallback = verifyAppendAndCaptureCallback() @@ -1466,49 +1453,44 @@ class GroupMetadataManagerTest { any(), any(), any()) - verify(replicaManager).getMagic(any()) } @ParameterizedTest @EnumSource(value = classOf[Errors], names = Array("INVALID_TXN_STATE", "INVALID_PRODUCER_ID_MAPPING")) def testTransactionalCommitOffsetTransactionalErrors(error: Errors): Unit = { val memberId = "" - val topicIdPartition = new TopicIdPartition(Uuid.randomUuid(), 0, "foo") + val topicId = Uuid.randomUuid() + val topicIdPartition1 = new TopicIdPartition(topicId, 0, "foo") + val topicIdPartition2 = new TopicIdPartition(topicId, 1, "foo") val offset = 37 val producerId = 232L val producerEpoch = 0.toShort - val transactionalId = "txnId" groupMetadataManager.addOwnedPartition(groupPartitionId) val group = new GroupMetadata(groupId, Empty, time) groupMetadataManager.addGroup(group) - val offsets = immutable.Map(topicIdPartition -> OffsetAndMetadata(offset, "", time.milliseconds())) - - when(replicaManager.getMagic(any())).thenReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE)) + val offsets = immutable.Map(topicIdPartition1 -> OffsetAndMetadata(offset, "", time.milliseconds()), + topicIdPartition2 -> OffsetAndMetadata(offset, "", time.milliseconds())) var commitErrors: Option[immutable.Map[TopicIdPartition, Errors]] = None def callback(errors: immutable.Map[TopicIdPartition, Errors]): Unit = { commitErrors = Some(errors) } - setUpTransactionVerification(replicaManager, transactionalId) - groupMetadataManager.storeOffsets(group, memberId, offsets, callback, transactionalId, producerId, producerEpoch) + storeOffsetsWithVerification(group, memberId, offsets, callback, producerId, producerEpoch, immutable.Map(topicIdPartition1 -> error)) assertTrue(group.hasOffsets) - assertTrue(group.allOffsets.isEmpty) val capturedResponseCallback = verifyAppendAndCaptureCallback() capturedResponseCallback.getValue.apply(Map(groupTopicPartition -> - new PartitionResponse(error, 0L, RecordBatch.NO_TIMESTAMP, 0L))) - - assertFalse(group.hasOffsets) - assertTrue(group.allOffsets.isEmpty) + new PartitionResponse(Errors.NONE, 0L, RecordBatch.NO_TIMESTAMP, 0L))) - group.completePendingTxnOffsetCommit(producerId, isCommit = false) - assertFalse(group.hasOffsets) - assertTrue(group.allOffsets.isEmpty) - assertFalse(commitErrors.contains(topicIdPartition)) - assertEquals(error, commitErrors.get(topicIdPartition)) + group.completePendingTxnOffsetCommit(producerId, isCommit = true) + assertTrue(commitErrors.isDefined) + assertEquals(error, commitErrors.get(topicIdPartition1)) + assertEquals(Errors.NONE, commitErrors.get(topicIdPartition2)) + assertTrue(group.offset(topicIdPartition1.topicPartition).isEmpty) + assertTrue(group.offset(topicIdPartition2.topicPartition).isDefined) verify(replicaManager).appendRecords(anyLong(), anyShort(), @@ -1522,7 +1504,6 @@ class GroupMetadataManagerTest { any(), any(), any()) - verify(replicaManager).getMagic(any()) } @Test @@ -3128,11 +3109,27 @@ class GroupMetadataManagerTest { } } - def setUpTransactionVerification(replicaManager: ReplicaManager, transactionalId: String): Unit = { - val postVerificationCallback: ArgumentCaptor[RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit] = ArgumentCaptor.forClass( - classOf[RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit]) - when(replicaManager.appendRecordsWithVerification(any(), any(), ArgumentMatchers.eq(transactionalId), any(), postVerificationCallback.capture())).thenAnswer( - _ => postVerificationCallback.getValue()(RequestLocal.NoCaching)(Map.empty, Map.empty) - ) + def storeOffsetsWithVerification(group: GroupMetadata, + memberId: String, + offsets: immutable.Map[TopicIdPartition, OffsetAndMetadata], + callback: immutable.Map[TopicIdPartition, Errors] => Unit, + producerId: Long, + producerEpoch: Short, + errors: immutable.Map[TopicIdPartition, Errors] = immutable.Map.empty[TopicIdPartition, Errors]): Unit = { + val verifiedOffsets = offsets.filter { case (tp, _) => + !errors.contains(tp) + } + val preAppendErrors = errors.map { case (tp, error) => + tp.topicPartition -> LogAppendResult( + LogAppendInfo.UNKNOWN_LOG_APPEND_INFO, + Some(error.exception()), + hasCustomErrorMessage = false) + } + + val records = groupMetadataManager.generateOffsetRecords(RecordBatch.CURRENT_MAGIC_VALUE, true, group.groupId, verifiedOffsets, producerId, producerEpoch) + val putCacheCallback = groupMetadataManager.createPutCacheCallback(true, group, memberId, offsets, verifiedOffsets, callback, producerId, records, preAppendErrors) + + + groupMetadataManager.storeOffsetsAfterVerification(group, verifiedOffsets, records, putCacheCallback, producerId, preAppendErrors) } } diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index a264b29c47c42..d9924d9f509d9 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -2690,8 +2690,8 @@ class KafkaApisTest { reset(replicaManager, clientQuotaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) val responseCallback: ArgumentCaptor[Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[Map[TopicPartition, PartitionResponse] => Unit]) - val postVerificationCallback: ArgumentCaptor[RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit] = ArgumentCaptor.forClass( - classOf[RequestLocal => (Map[TopicPartition, MemoryRecords], Map[TopicPartition, LogAppendResult]) => Unit]) + val postVerificationCallback: ArgumentCaptor[RequestLocal => Map[TopicPartition, LogAppendResult] => Unit] = ArgumentCaptor.forClass( + classOf[RequestLocal => Map[TopicPartition, LogAppendResult] => Unit]) val tp = new TopicPartition("topic", 0) @@ -2714,13 +2714,13 @@ class KafkaApisTest { val newRequestLocal = RequestLocal.NoCaching when(replicaManager.appendRecordsWithVerification(any(), any(), any(), any(), postVerificationCallback.capture())).thenAnswer( - arg => replicaManager.appendRecordsAfterVerification(arg.getArgument(0), arg.getArgument(1), postVerificationCallback.getValue())(newRequestLocal, Map.empty) + arg => replicaManager.appendRecordsAfterVerification(arg.getArgument(1), postVerificationCallback.getValue())(newRequestLocal, Map.empty) ) - when(replicaManager.appendRecordsAfterVerification(any(), any(), postVerificationCallback.capture())(any(), any())).thenAnswer( + when(replicaManager.appendRecordsAfterVerification(any(), postVerificationCallback.capture())(any(), any())).thenAnswer( _ => { val callback = postVerificationCallback.getValue() - callback(RequestLocal.NoCaching)(Map.empty, Map.empty) + callback(RequestLocal.NoCaching)(Map.empty) } ) @@ -2735,7 +2735,6 @@ class KafkaApisTest { ) verify(replicaManager).appendRecordsAfterVerification( - any(), any(), ArgumentMatchers.eq(postVerificationCallback.getValue()))(ArgumentMatchers.eq(newRequestLocal), any()) diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index 27b08efc02854..e45d300e113d2 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -2950,13 +2950,13 @@ class ReplicaManagerTest { val transactionVerificationEntries = new ReplicaManager.TransactionVerificationEntries def postVerificationCallback(newRequestLocal: RequestLocal) - (newlyVerifiedEntries: Map[TopicPartition, MemoryRecords], errorResults: Map[TopicPartition, LogAppendResult]): Unit = { + (errorResults: Map[TopicPartition, LogAppendResult]): Unit = { replicaManager.appendRecords( timeout = 1000, requiredAcks = requiredAcks, internalTopicsAllowed = false, origin = origin, - entriesPerPartition = newlyVerifiedEntries ++ transactionVerificationEntries.verified, + entriesPerPartition = entriesToAppend, responseCallback = appendCallback, requestLocal = newRequestLocal, preAppendErrors = errorResults @@ -2964,7 +2964,7 @@ class ReplicaManagerTest { } replicaManager.appendRecordsWithVerification( - entriesPerPartition = entriesToAppend, + entriesToAppend, transactionVerificationEntries, transactionalId, RequestLocal.NoCaching, @@ -2988,15 +2988,16 @@ class ReplicaManagerTest { result.fire(response.get) } + val entriesPerPartition = Map(partition -> records) val transactionVerificationEntries = new ReplicaManager.TransactionVerificationEntries def postVerificationCallback(newRequestLocal: RequestLocal) - (newlyVerifiedEntries: Map[TopicPartition, MemoryRecords], errorResults: Map[TopicPartition, LogAppendResult]): Unit = { + (errorResults: Map[TopicPartition, LogAppendResult]): Unit = { replicaManager.appendRecords( timeout = 1000, requiredAcks = requiredAcks, internalTopicsAllowed = false, origin = origin, - entriesPerPartition = newlyVerifiedEntries ++ transactionVerificationEntries.verified, + entriesPerPartition = entriesPerPartition, responseCallback = appendCallback, requestLocal = newRequestLocal, preAppendErrors = errorResults @@ -3004,7 +3005,7 @@ class ReplicaManagerTest { } replicaManager.appendRecordsWithVerification( - entriesPerPartition = Map(partition -> records), + entriesPerPartition, transactionVerificationEntries, transactionalId, RequestLocal.NoCaching, From 7bc6d06d017ec0d6a716466aac321e4063bede9a Mon Sep 17 00:00:00 2001 From: Justine Date: Wed, 29 Nov 2023 16:22:44 -0800 Subject: [PATCH 06/24] Update comments and method names --- .../coordinator/group/GroupCoordinator.scala | 2 +- .../main/scala/kafka/server/KafkaApis.scala | 2 +- .../scala/kafka/server/ReplicaManager.scala | 64 ++++++++++--------- .../AbstractCoordinatorConcurrencyTest.scala | 10 +-- .../group/GroupCoordinatorTest.scala | 2 +- .../unit/kafka/server/KafkaApisTest.scala | 26 ++++---- .../kafka/server/ReplicaManagerTest.scala | 4 +- 7 files changed, 57 insertions(+), 53 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala index 5f23915565dd4..da695920e583b 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala @@ -965,7 +965,7 @@ private[group] class GroupCoordinator( } } - groupManager.replicaManager.appendRecordsWithVerification( + groupManager.replicaManager.appendRecordsWithTransactionVerification( entriesPerPartition = records, transactionVerificationEntries = transactionVerificationEntries, transactionalId = transactionalId, diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 273b8cd6087cc..f9bf6e9fc453a 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -734,7 +734,7 @@ class KafkaApis(val requestChannel: RequestChannel, if (produceRequest.transactionalId == null){ postVerificationCallback(requestLocal)(Map.empty) } else { - replicaManager.appendRecordsWithVerification( + replicaManager.appendRecordsWithTransactionVerification( entriesPerPartition = authorizedRequestInfo, transactionVerificationEntries = transactionVerificationEntries, transactionalId = produceRequest.transactionalId, diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index 8bae664b21f9a..65cebcd8bd911 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -769,9 +769,11 @@ class ReplicaManager(val config: KafkaConfig, * @param responseCallback callback for sending the response * @param delayedProduceLock lock for the delayed actions * @param recordValidationStatsCallback callback for updating stats on record conversions - * @param requestLocal container for the stateful instances scoped to this request - * @param transactionalId transactional ID if the request is from a producer and the producer is transactional + * @param requestLocal container for the stateful instances scoped to this request -- this must correspond to the + * thread calling this method * @param actionQueue the action queue to use. ReplicaManager#defaultActionQueue is used by default. + * @param verificationGuards the mapping from topic partition to verification guards if transaction verification is used + * @param preAppendErrors the mapping from topic partition to LogAppendResult for errors that occurred before appending */ def appendRecords(timeout: Long, requiredAcks: Short, @@ -870,16 +872,32 @@ class ReplicaManager(val config: KafkaConfig, responseCallback(responseStatus) } - def appendRecordsWithVerification(entriesPerPartition: Map[TopicPartition, MemoryRecords], - transactionVerificationEntries: TransactionVerificationEntries, - transactionalId: String, - requestLocal: RequestLocal, - postVerificationCallback: RequestLocal => Map[TopicPartition, LogAppendResult] => Unit): Unit = { + /** + * Apply the postVerificationCallback asynchronously only after verifying the partitions have been added to the transaction. + * The postVerificationCallback takes the arguments of the requestLocal for the thread that will be doing the append as + * well as a mapping of topic partitions to LogAppendResult for the partitions that saw errors when verifying. + * + * This method will start the verification process for all the topicPartitions in entriesPerPartition and supply the + * postVerificationCallback to be run on a request handler thread when the response is received. + * + * @param entriesPerPartition the records per partition to be appended and therefore need verification + * @param transactionVerificationEntries the object that will store the entries to verify, the errors, and the verification guards + * @param transactionalId the id for the transaction + * @param requestLocal container for the stateful instances scoped to this request -- this must correspond to the + * thread calling this method + * @param postVerificationCallback the method to be called when verification completes and the verification errors + * and the thread's RequestLocal are supplied + */ + def appendRecordsWithTransactionVerification(entriesPerPartition: Map[TopicPartition, MemoryRecords], + transactionVerificationEntries: TransactionVerificationEntries, + transactionalId: String, + requestLocal: RequestLocal, + postVerificationCallback: RequestLocal => Map[TopicPartition, LogAppendResult] => Unit): Unit = { if (transactionalId != null && config.transactionPartitionVerificationEnable && addPartitionsToTxnManager.isDefined) partitionEntriesForVerification(transactionVerificationEntries, entriesPerPartition) val onVerificationComplete: (RequestLocal, Map[TopicPartition, Errors]) => Unit = - appendRecordsAfterVerification( + executePostVerificationCallback( transactionVerificationEntries, postVerificationCallback, ) @@ -901,30 +919,18 @@ class ReplicaManager(val config: KafkaConfig, } /** - * Append messages to leader replicas of the partition, and wait for them to be replicated to other replicas; - * the callback function will be triggered either when timeout or the required acks are satisfied; - * if the callback function itself is already synchronized on some object then pass this object to avoid deadlock. + * A helper method to compile the results from the transaction verification and call the postVerificationCallback. * - * Noted that all pending delayed check operations are stored in a queue. All callers to ReplicaManager.appendRecords() - * are expected to call ActionQueue.tryCompleteActions for all affected partitions, without holding any conflicting - * locks. + * @param transactionVerificationEntries the object that will store the entries to verify, the errors, and the verification guards + * @param postVerificationCallback the method to be called when verification completes and the verification errors + * and the thread's RequestLocal are supplied + * @param requestLocal container for the stateful instances scoped to this request -- this must correspond to the + * thread calling this method * - * @param allEntries the records per partition for all partitions in the request - * @param internalTopicsAllowed boolean indicating whether internal topics can be appended to - * @param origin source of the append request (ie, client, replication, coordinator) - * @param requiredAcks number of replicas who must acknowledge the append before sending the response - * @param verificationGuards verificationGuards for ensuring a partition has been added to the transaction - * @param errorsPerPartition the mapping from partition to errors we have already seen - * @param timeout maximum time we will wait to append before returning - * @param responseCallback callback for sending the response - * @param delayedProduceLock lock for the delayed actions - * @param actionQueue the action queue to use. ReplicaManager#defaultActionQueue is used by default. - * @param requestLocal container for the stateful instances scoped to this request - * @param unverifiedEntries the records per partition for topic partitions that were not verified by the transaction coordinator */ - def appendRecordsAfterVerification(transactionVerificationEntries: TransactionVerificationEntries, - postVerificationCallback: RequestLocal => Map[TopicPartition, LogAppendResult] => Unit) - (requestLocal: RequestLocal, unverifiedEntries: Map[TopicPartition, Errors] = Map.empty): Unit = { + private def executePostVerificationCallback(transactionVerificationEntries: TransactionVerificationEntries, + postVerificationCallback: RequestLocal => Map[TopicPartition, LogAppendResult] => Unit) + (requestLocal: RequestLocal, unverifiedEntries: Map[TopicPartition, Errors]): Unit = { val errorResults = (unverifiedEntries ++ transactionVerificationEntries.errors).map { case (topicPartition, error) => // translate transaction coordinator errors to known producer response errors diff --git a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala index ce68d05c0bfd8..13cf4d0cf3d30 100644 --- a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala @@ -211,11 +211,11 @@ object AbstractCoordinatorConcurrencyTest { producePurgatory.tryCompleteElseWatch(delayedProduce, producerRequestKeys) } - override def appendRecordsWithVerification(entriesPerPartition: Map[TopicPartition, MemoryRecords], - transactionVerificationEntries: TransactionVerificationEntries, - transactionalId: String, - requestLocal: RequestLocal, - postVerificationCallback: RequestLocal => Map[TopicPartition, LogAppendResult] => Unit): Unit = { + override def appendRecordsWithTransactionVerification(entriesPerPartition: Map[TopicPartition, MemoryRecords], + transactionVerificationEntries: TransactionVerificationEntries, + transactionalId: String, + requestLocal: RequestLocal, + postVerificationCallback: RequestLocal => Map[TopicPartition, LogAppendResult] => Unit): Unit = { postVerificationCallback(requestLocal)(Map.empty) } diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala index ac2d0c99834ba..9de43311abdbb 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala @@ -4127,7 +4127,7 @@ class GroupCoordinatorTest { val postVerificationCallback: ArgumentCaptor[RequestLocal => SMap[TopicPartition, LogAppendResult] => Unit] = ArgumentCaptor.forClass( classOf[RequestLocal => SMap[TopicPartition, LogAppendResult] => Unit]) - when(replicaManager.appendRecordsWithVerification(any(), any(), ArgumentMatchers.eq(transactionalId), any(), postVerificationCallback.capture())).thenAnswer( + when(replicaManager.appendRecordsWithTransactionVerification(any(), any(), ArgumentMatchers.eq(transactionalId), any(), postVerificationCallback.capture())).thenAnswer( _ => postVerificationCallback.getValue()(RequestLocal.NoCaching)(preAppendErrors) ) when(replicaManager.appendRecords(anyLong, diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 4b954215957e7..2b922c5074d1a 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -30,6 +30,7 @@ import kafka.coordinator.transaction.{InitProducerIdResult, TransactionCoordinat import kafka.log.UnifiedLog import kafka.network.{RequestChannel, RequestMetrics} import kafka.server.QuotaFactory.QuotaManagers +import kafka.server.ReplicaManager.TransactionVerificationEntries import kafka.server.metadata.{ConfigRepository, KRaftMetadataCache, MockConfigRepository, ZkMetadataCache} import kafka.utils.{Log4jController, TestUtils} import kafka.zk.KafkaZkClient @@ -99,7 +100,7 @@ import org.apache.kafka.server.common.{Features, MetadataVersion} import org.apache.kafka.server.common.MetadataVersion.{IBP_0_10_2_IV0, IBP_2_2_IV1} import org.apache.kafka.server.metrics.ClientMetricsTestUtils import org.apache.kafka.server.util.MockTime -import org.apache.kafka.storage.internals.log.{AppendOrigin, FetchParams, FetchPartitionData, LogConfig} +import org.apache.kafka.storage.internals.log.{AppendOrigin, FetchParams, FetchPartitionData, LogAppendInfo, LogConfig} class KafkaApisTest { private val requestChannel: RequestChannel = mock(classOf[RequestChannel]) @@ -2692,6 +2693,7 @@ class KafkaApisTest { val responseCallback: ArgumentCaptor[Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[Map[TopicPartition, PartitionResponse] => Unit]) val postVerificationCallback: ArgumentCaptor[RequestLocal => Map[TopicPartition, LogAppendResult] => Unit] = ArgumentCaptor.forClass( classOf[RequestLocal => Map[TopicPartition, LogAppendResult] => Unit]) + val transactionVerificationEntries: ArgumentCaptor[TransactionVerificationEntries] = ArgumentCaptor.forClass(classOf[TransactionVerificationEntries]) val tp = new TopicPartition("topic", 0) @@ -2712,31 +2714,27 @@ class KafkaApisTest { val kafkaApis = createKafkaApis() val requestLocal = RequestLocal.withThreadConfinedCaching val newRequestLocal = RequestLocal.NoCaching + val preAppendErrors = Map(tp -> LogAppendResult( + LogAppendInfo.UNKNOWN_LOG_APPEND_INFO, + Some(Errors.INVALID_TXN_STATE.exception()), + hasCustomErrorMessage = false)) - when(replicaManager.appendRecordsWithVerification(any(), any(), any(), any(), postVerificationCallback.capture())).thenAnswer( - arg => replicaManager.appendRecordsAfterVerification(arg.getArgument(1), postVerificationCallback.getValue())(newRequestLocal, Map.empty) - ) - - when(replicaManager.appendRecordsAfterVerification(any(), postVerificationCallback.capture())(any(), any())).thenAnswer( + when(replicaManager.appendRecordsWithTransactionVerification(any(), transactionVerificationEntries.capture(), any(), any(), postVerificationCallback.capture())).thenAnswer( _ => { val callback = postVerificationCallback.getValue() - callback(RequestLocal.NoCaching)(Map.empty) + callback(newRequestLocal)(preAppendErrors) } ) kafkaApis.handleProduceRequest(request, requestLocal) - verify(replicaManager).appendRecordsWithVerification( + verify(replicaManager).appendRecordsWithTransactionVerification( any(), any(), ArgumentMatchers.eq(transactionalId), ArgumentMatchers.eq(requestLocal), ArgumentMatchers.eq(postVerificationCallback.getValue()) ) - - verify(replicaManager).appendRecordsAfterVerification( - any(), - ArgumentMatchers.eq(postVerificationCallback.getValue()))(ArgumentMatchers.eq(newRequestLocal), any()) verify(replicaManager).appendRecords(anyLong, anyShort, @@ -2748,8 +2746,8 @@ class KafkaApisTest { any(), ArgumentMatchers.eq(newRequestLocal), any(), - any(), - any()) + ArgumentMatchers.eq(transactionVerificationEntries.getValue().verificationGuards), + ArgumentMatchers.eq(preAppendErrors)) } } diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index b3f75af652fee..047a17a293d91 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -2963,7 +2963,7 @@ class ReplicaManagerTest { ) } - replicaManager.appendRecordsWithVerification( + replicaManager.appendRecordsWithTransactionVerification( entriesToAppend, transactionVerificationEntries, transactionalId, @@ -3004,7 +3004,7 @@ class ReplicaManagerTest { ) } - replicaManager.appendRecordsWithVerification( + replicaManager.appendRecordsWithTransactionVerification( entriesPerPartition, transactionVerificationEntries, transactionalId, From a471f04c454562d9e2e729470b57380bd5913c1e Mon Sep 17 00:00:00 2001 From: Justine Date: Mon, 4 Dec 2023 11:40:36 -0800 Subject: [PATCH 07/24] Fix style issues and passing verification guards --- .../scala/kafka/coordinator/group/GroupCoordinator.scala | 2 +- .../kafka/coordinator/group/GroupMetadataManager.scala | 7 ++++--- .../coordinator/AbstractCoordinatorConcurrencyTest.scala | 2 -- .../kafka/coordinator/group/GroupMetadataManagerTest.scala | 6 ++++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala index da695920e583b..c8dc149ff5366 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala @@ -960,7 +960,7 @@ private[group] class GroupCoordinator( responseCallback(offsetMetadata.map { case (k, _) => k -> errorResults(k.topicPartition).error }) } else { val putCacheCallback = groupManager.createPutCacheCallback(true, group, memberId, offsetMetadata, verifiedOffsets, responseCallback, producerId, verifiedRecords, errorResults) - groupManager.storeOffsetsAfterVerification(group, verifiedOffsets, records, putCacheCallback, producerId, errorResults, newRequestLocal) + groupManager.storeOffsetsAfterVerification(group, verifiedOffsets, records, putCacheCallback, producerId, transactionVerificationEntries, errorResults, newRequestLocal) } } } diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala index cae2a399afbb3..85c7b11ced503 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala @@ -26,6 +26,7 @@ import java.util.concurrent.locks.ReentrantLock import java.util.concurrent.ConcurrentHashMap import com.yammer.metrics.core.Gauge import kafka.common.OffsetAndMetadata +import kafka.server.ReplicaManager.TransactionVerificationEntries import kafka.server.{LogAppendResult, ReplicaManager, RequestLocal} import kafka.utils.CoreUtils.inLock import kafka.utils.Implicits._ @@ -512,6 +513,7 @@ class GroupMetadataManager(brokerId: Int, records: Map[TopicPartition, MemoryRecords], putCacheCallback: Map[TopicPartition, PartitionResponse] => Unit, producerId: Long, + transactionVerificationEntries: TransactionVerificationEntries, errorResults: Map[TopicPartition, LogAppendResult], requestLocal: RequestLocal = RequestLocal.NoCaching): Unit = { group.inLock { @@ -521,7 +523,6 @@ class GroupMetadataManager(brokerId: Int, s"should be avoided.") } - val transactionVerificationEntries = new ReplicaManager.TransactionVerificationEntries group.inLock { addProducerGroup(producerId, group.groupId) group.prepareTxnOffsetCommit(producerId, verifiedOffsetMetadata) @@ -1017,7 +1018,7 @@ class GroupMetadataManager(brokerId: Int, /* * Check if the offset metadata length is valid */ - protected[coordinator] def validateOffsetMetadataLength(metadata: String) : Boolean = { + private[coordinator] def validateOffsetMetadataLength(metadata: String) : Boolean = { metadata == null || metadata.length() <= config.maxMetadataSize } @@ -1038,7 +1039,7 @@ class GroupMetadataManager(brokerId: Int, * @param partition Partition of GroupMetadataTopic * @return Some(MessageFormatVersion) if replica is local, None otherwise */ - protected[coordinator] def getMagic(partition: Int): Option[Byte] = + private[coordinator] def getMagic(partition: Int): Option[Byte] = replicaManager.getMagic(new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, partition)) /** diff --git a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala index 13cf4d0cf3d30..9fb31882fa91c 100644 --- a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala @@ -216,11 +216,9 @@ object AbstractCoordinatorConcurrencyTest { transactionalId: String, requestLocal: RequestLocal, postVerificationCallback: RequestLocal => Map[TopicPartition, LogAppendResult] => Unit): Unit = { - postVerificationCallback(requestLocal)(Map.empty) } - override def getMagic(topicPartition: TopicPartition): Option[Byte] = { Some(RecordBatch.MAGIC_VALUE_V2) } diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala index b03c17475865f..cfe9659272d76 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala @@ -27,6 +27,7 @@ import javax.management.ObjectName import kafka.cluster.Partition import kafka.common.OffsetAndMetadata import kafka.log.UnifiedLog +import kafka.server.ReplicaManager.TransactionVerificationEntries import kafka.server.{HostedPartition, KafkaConfig, LogAppendResult, ReplicaManager, RequestLocal} import kafka.utils.TestUtils import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor @@ -3116,6 +3117,7 @@ class GroupMetadataManagerTest { producerId: Long, producerEpoch: Short, errors: immutable.Map[TopicIdPartition, Errors] = immutable.Map.empty[TopicIdPartition, Errors]): Unit = { + // Consider any non-error topic partition as verified. val verifiedOffsets = offsets.filter { case (tp, _) => !errors.contains(tp) } @@ -3129,7 +3131,7 @@ class GroupMetadataManagerTest { val records = groupMetadataManager.generateOffsetRecords(RecordBatch.CURRENT_MAGIC_VALUE, true, group.groupId, verifiedOffsets, producerId, producerEpoch) val putCacheCallback = groupMetadataManager.createPutCacheCallback(true, group, memberId, offsets, verifiedOffsets, callback, producerId, records, preAppendErrors) - - groupMetadataManager.storeOffsetsAfterVerification(group, verifiedOffsets, records, putCacheCallback, producerId, preAppendErrors) + // Pass in new verification guard since the Log/ReplicaManager code is mocked. + groupMetadataManager.storeOffsetsAfterVerification(group, verifiedOffsets, records, putCacheCallback, producerId, new TransactionVerificationEntries, preAppendErrors) } } From 7c0168291dd8246add6e0d0be7118be67686f919 Mon Sep 17 00:00:00 2001 From: Justine Date: Wed, 6 Dec 2023 15:00:19 -0800 Subject: [PATCH 08/24] Clean up GroupCoordinator, GroupMetadataManger, and ReplicaManager code --- .../coordinator/group/GroupCoordinator.scala | 115 ++++--- .../group/GroupMetadataManager.scala | 43 +-- .../main/scala/kafka/server/KafkaApis.scala | 31 +- .../scala/kafka/server/ReplicaManager.scala | 288 +++++++++++------- .../AbstractCoordinatorConcurrencyTest.scala | 22 +- .../CoordinatorPartitionWriterTest.scala | 3 + .../group/GroupCoordinatorTest.scala | 38 +-- .../group/GroupMetadataManagerTest.scala | 139 +++------ .../unit/kafka/server/KafkaApisTest.scala | 64 +--- .../kafka/server/ReplicaManagerTest.scala | 108 +++---- 10 files changed, 385 insertions(+), 466 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala index c8dc149ff5366..3ba981fd2b372 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala @@ -19,7 +19,6 @@ package kafka.coordinator.group import java.util.{OptionalInt, Properties} import java.util.concurrent.atomic.AtomicBoolean import kafka.common.OffsetAndMetadata -import kafka.server.ReplicaManager.TransactionVerificationEntries import kafka.server._ import kafka.utils.Logging import org.apache.kafka.common.{TopicIdPartition, TopicPartition} @@ -30,9 +29,11 @@ import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.metrics.stats.Meter import org.apache.kafka.common.protocol.{ApiKeys, Errors} +import org.apache.kafka.common.record.RecordBatch import org.apache.kafka.common.requests._ import org.apache.kafka.common.utils.Time import org.apache.kafka.server.record.BrokerCompressionType +import org.apache.kafka.storage.internals.log.VerificationGuard import scala.collection.{Map, Seq, Set, immutable, mutable} import scala.math.max @@ -911,66 +912,29 @@ private[group] class GroupCoordinator( groupManager.addGroup(new GroupMetadata(groupId, Empty, time)) } - val filteredOffsetMetadata = offsetMetadata.filter { case (_, offsetAndMetadata) => - groupManager.validateOffsetMetadataLength(offsetAndMetadata.metadata) - } - if (filteredOffsetMetadata.isEmpty) { - // compute the final error codes for the commit response - val commitStatus = offsetMetadata.map { case (k, _) => k -> Errors.OFFSET_METADATA_TOO_LARGE } - responseCallback(commitStatus) - return - } - - val magicOpt = groupManager.getMagic(partitionFor(group.groupId)) - if (magicOpt.isEmpty) { - val commitStatus = offsetMetadata.map { case (topicIdPartition, _) => - (topicIdPartition, Errors.NOT_COORDINATOR) - } - responseCallback(commitStatus) - return - } - - val records = groupManager.generateOffsetRecords(magicOpt.get, true, group.groupId, filteredOffsetMetadata, producerId, producerEpoch) - val transactionVerificationEntries = new TransactionVerificationEntries - - def postVerificationCallback(newRequestLocal: RequestLocal) - (errorResults: Map[TopicPartition, LogAppendResult]): Unit = { - group.inLock { - val validationErrorOpt = validateOffsetCommit( - group, - generationId, - memberId, - groupInstanceId, - isTransactional = true - ) - - val verifiedOffsets = offsetMetadata.filter { - case (tp, _) => - !errorResults.contains(tp.topicPartition) - } - - val verifiedRecords = records.filter { - case (tp, _) => - !errorResults.contains(tp) - } - - if (validationErrorOpt.isDefined) { - responseCallback(offsetMetadata.map { case (k, _) => k -> validationErrorOpt.get }) - } else if (verifiedOffsets.isEmpty) { - responseCallback(offsetMetadata.map { case (k, _) => k -> errorResults(k.topicPartition).error }) - } else { - val putCacheCallback = groupManager.createPutCacheCallback(true, group, memberId, offsetMetadata, verifiedOffsets, responseCallback, producerId, verifiedRecords, errorResults) - groupManager.storeOffsetsAfterVerification(group, verifiedOffsets, records, putCacheCallback, producerId, transactionVerificationEntries, errorResults, newRequestLocal) - } + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, partitionFor(group.groupId)) + + def postVerificationCallback( + error: Errors, + newRequestLocal: RequestLocal, + verificationGuard: VerificationGuard + ): Unit = { + if (error != Errors.NONE) { + responseCallback(offsetMetadata.map { case (k, _) => k -> error }) + } else { + doTxnCommitOffsets(group, memberId, groupInstanceId, generationId, producerId, producerEpoch, + offsetTopicPartition, offsetMetadata, newRequestLocal, responseCallback, Some(verificationGuard)) } } - groupManager.replicaManager.appendRecordsWithTransactionVerification( - entriesPerPartition = records, - transactionVerificationEntries = transactionVerificationEntries, - transactionalId = transactionalId, - requestLocal = requestLocal, - postVerificationCallback = postVerificationCallback + groupManager.replicaManager.maybeStartTransactionVerificationForPartition( + topicPartition = offsetTopicPartition, + transactionalId, + producerId, + producerEpoch, + RecordBatch.NO_SEQUENCE, + requestLocal, + postVerificationCallback ) } } @@ -1012,6 +976,35 @@ private[group] class GroupCoordinator( groupManager.scheduleHandleTxnCompletion(producerId, offsetsPartitions.map(_.partition).toSet, isCommit) } + private def doTxnCommitOffsets(group: GroupMetadata, + memberId: String, + groupInstanceId: Option[String], + generationId: Int, + producerId: Long, + producerEpoch: Short, + offsetTopicPartition: TopicPartition, + offsetMetadata: immutable.Map[TopicIdPartition, OffsetAndMetadata], + requestLocal: RequestLocal, + responseCallback: immutable.Map[TopicIdPartition, Errors] => Unit, + verificationGuard: Option[VerificationGuard]): Unit = { + group.inLock { + val validationErrorOpt = validateOffsetCommit( + group, + generationId, + memberId, + groupInstanceId, + isTransactional = true + ) + + if (validationErrorOpt.isDefined) { + responseCallback(offsetMetadata.map { case (k, _) => k -> validationErrorOpt.get }) + } else { + groupManager.storeOffsets(group, memberId, offsetTopicPartition, offsetMetadata, responseCallback, producerId, + producerEpoch, requestLocal, verificationGuard) + } + } + } + private def validateOffsetCommit( group: GroupMetadata, generationId: Int, @@ -1066,19 +1059,21 @@ private[group] class GroupCoordinator( isTransactional = false ) + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, partitionFor(group.groupId)) + if (validationErrorOpt.isDefined) { responseCallback(offsetMetadata.map { case (k, _) => k -> validationErrorOpt.get }) } else { group.currentState match { case Empty => - groupManager.storeOffsets(group, memberId, offsetMetadata, responseCallback) + groupManager.storeOffsets(group, memberId, offsetTopicPartition, offsetMetadata, responseCallback, verificationGuard = None) case Stable | PreparingRebalance => // During PreparingRebalance phase, we still allow a commit request since we rely // on heartbeat response to eventually notify the rebalance in progress signal to the consumer val member = group.get(memberId) completeAndScheduleNextHeartbeatExpiration(group, member) - groupManager.storeOffsets(group, memberId, offsetMetadata, responseCallback, requestLocal = requestLocal) + groupManager.storeOffsets(group, memberId, offsetTopicPartition, offsetMetadata, responseCallback, requestLocal = requestLocal, verificationGuard = None) case CompletingRebalance => // We should not receive a commit request if the group has not completed rebalance; diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala index 85c7b11ced503..4c5622ebe9d12 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala @@ -26,7 +26,6 @@ import java.util.concurrent.locks.ReentrantLock import java.util.concurrent.ConcurrentHashMap import com.yammer.metrics.core.Gauge import kafka.common.OffsetAndMetadata -import kafka.server.ReplicaManager.TransactionVerificationEntries import kafka.server.{LogAppendResult, ReplicaManager, RequestLocal} import kafka.utils.CoreUtils.inLock import kafka.utils.Implicits._ @@ -348,6 +347,7 @@ class GroupMetadataManager(brokerId: Int, def generateOffsetRecords(magicValue: Byte, isTxnOffsetCommit: Boolean, groupId: String, + offsetTopicPartition: TopicPartition, filteredOffsetMetadata: Map[TopicIdPartition, OffsetAndMetadata], producerId: Long, producerEpoch: Short): Map[TopicPartition, MemoryRecords] = { @@ -360,7 +360,6 @@ class GroupMetadataManager(brokerId: Int, val value = GroupMetadataManager.offsetCommitValue(offsetAndMetadata, interBrokerProtocolVersion) new SimpleRecord(timestamp, key, value) } - val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, partitionFor(groupId)) val buffer = ByteBuffer.allocate(AbstractRecords.estimateSizeInBytes(magicValue, compressionType, records.asJava)) if (isTxnOffsetCommit && magicValue < RecordBatch.MAGIC_VALUE_V2) @@ -466,11 +465,13 @@ class GroupMetadataManager(brokerId: Int, */ def storeOffsets(group: GroupMetadata, consumerId: String, + offsetTopicPartition: TopicPartition, offsetMetadata: immutable.Map[TopicIdPartition, OffsetAndMetadata], responseCallback: immutable.Map[TopicIdPartition, Errors] => Unit, producerId: Long = RecordBatch.NO_PRODUCER_ID, producerEpoch: Short = RecordBatch.NO_PRODUCER_EPOCH, - requestLocal: RequestLocal = RequestLocal.NoCaching): Unit = { + requestLocal: RequestLocal = RequestLocal.NoCaching, + verificationGuard: Option[VerificationGuard]): Unit = { group.inLock { if (!group.hasReceivedConsistentOffsetCommits) warn(s"group: ${group.groupId} with leader: ${group.leaderOrNull} has received offset commits from consumers as well " + @@ -498,38 +499,24 @@ class GroupMetadataManager(brokerId: Int, } val isTxnOffsetCommit = producerId != RecordBatch.NO_PRODUCER_ID - val records = generateOffsetRecords(magicOpt.get, isTxnOffsetCommit, group.groupId, filteredOffsetMetadata, producerId, producerEpoch) + val records = generateOffsetRecords(magicOpt.get, isTxnOffsetCommit, group.groupId, offsetTopicPartition, filteredOffsetMetadata, producerId, producerEpoch) val putCacheCallback = createPutCacheCallback(isTxnOffsetCommit, group, consumerId, offsetMetadata, filteredOffsetMetadata, responseCallback, producerId, records) - group.inLock { - group.prepareOffsetCommit(offsetMetadata) - } - - appendForGroup(group, records, requestLocal, putCacheCallback) - } - - def storeOffsetsAfterVerification(group: GroupMetadata, - verifiedOffsetMetadata: immutable.Map[TopicIdPartition, OffsetAndMetadata], - records: Map[TopicPartition, MemoryRecords], - putCacheCallback: Map[TopicPartition, PartitionResponse] => Unit, - producerId: Long, - transactionVerificationEntries: TransactionVerificationEntries, - errorResults: Map[TopicPartition, LogAppendResult], - requestLocal: RequestLocal = RequestLocal.NoCaching): Unit = { - group.inLock { - if (!group.hasReceivedConsistentOffsetCommits) - warn(s"group: ${group.groupId} with leader: ${group.leaderOrNull} has received offset commits from consumers as well " + - s"as transactional producers. Mixing both types of offset commits will generally result in surprises and " + - s"should be avoided.") + val verificationGuards = verificationGuard match { + case Some(guard) => Map(offsetTopicPartition -> guard) + case None => Map.empty[TopicPartition, VerificationGuard] } group.inLock { - addProducerGroup(producerId, group.groupId) - group.prepareTxnOffsetCommit(producerId, verifiedOffsetMetadata) + if (isTxnOffsetCommit) { + addProducerGroup(producerId, group.groupId) + group.prepareTxnOffsetCommit(producerId, offsetMetadata) + } else { + group.prepareOffsetCommit(offsetMetadata) + } } - appendForGroup(group, records, requestLocal, - putCacheCallback, transactionVerificationEntries.verificationGuards.toMap, errorResults) + appendForGroup(group, records, requestLocal, putCacheCallback, verificationGuards) } /** diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index f9bf6e9fc453a..6c6a4fab301d1 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -709,39 +709,20 @@ class KafkaApis(val requestChannel: RequestChannel, } val internalTopicsAllowed = request.header.clientId == AdminUtils.ADMIN_CLIENT_ID - val transactionVerificationEntries = new ReplicaManager.TransactionVerificationEntries - def postVerificationCallback(newRequestLocal: RequestLocal) - (errorResults: Map[TopicPartition, LogAppendResult]): Unit = { - replicaManager.appendRecords( + if (authorizedRequestInfo.isEmpty) + sendResponseCallback(Map.empty) + else { + replicaManager.handleProduceAppend( timeout = produceRequest.timeout.toLong, requiredAcks = produceRequest.acks, internalTopicsAllowed = internalTopicsAllowed, origin = AppendOrigin.CLIENT, + transactionalId = produceRequest.transactionalId, entriesPerPartition = authorizedRequestInfo, responseCallback = sendResponseCallback, recordValidationStatsCallback = processingStatsCallback, - requestLocal = newRequestLocal, - verificationGuards = transactionVerificationEntries.verificationGuards.toMap, - preAppendErrors = errorResults - ) - } - - if (authorizedRequestInfo.isEmpty) - sendResponseCallback(Map.empty) - else { - // call the replica manager to append messages to the replicas - if (produceRequest.transactionalId == null){ - postVerificationCallback(requestLocal)(Map.empty) - } else { - replicaManager.appendRecordsWithTransactionVerification( - entriesPerPartition = authorizedRequestInfo, - transactionVerificationEntries = transactionVerificationEntries, - transactionalId = produceRequest.transactionalId, - requestLocal = requestLocal, - postVerificationCallback = postVerificationCallback - ) - } + requestLocal = requestLocal) // if the request is put into the purgatory, it will have a held reference and hence cannot be garbage collected; // hence we clear its data here in order to let GC reclaim its memory since it is already appended to log diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index 65cebcd8bd911..0b517f721502f 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -24,7 +24,7 @@ import kafka.log.remote.RemoteLogManager import kafka.log.{LogManager, UnifiedLog} import kafka.server.HostedPartition.Online import kafka.server.QuotaFactory.QuotaManagers -import kafka.server.ReplicaManager.{AtMinIsrPartitionCountMetricName, FailedIsrUpdatesPerSecMetricName, IsrExpandsPerSecMetricName, IsrShrinksPerSecMetricName, LeaderCountMetricName, OfflineReplicaCountMetricName, PartitionCountMetricName, PartitionsWithLateTransactionsCountMetricName, ProducerIdCountMetricName, ReassigningPartitionsMetricName, TransactionVerificationEntries, UnderMinIsrPartitionCountMetricName, UnderReplicatedPartitionsMetricName, createLogReadResult} +import kafka.server.ReplicaManager.{AtMinIsrPartitionCountMetricName, FailedIsrUpdatesPerSecMetricName, IsrExpandsPerSecMetricName, IsrShrinksPerSecMetricName, LeaderCountMetricName, OfflineReplicaCountMetricName, PartitionCountMetricName, PartitionsWithLateTransactionsCountMetricName, ProducerIdCountMetricName, ReassigningPartitionsMetricName, UnderMinIsrPartitionCountMetricName, UnderReplicatedPartitionsMetricName, createLogReadResult} import kafka.server.checkpoints.{LazyOffsetCheckpoints, OffsetCheckpointFile, OffsetCheckpoints} import kafka.server.metadata.{KRaftMetadataCache, ZkMetadataCache} import kafka.utils.Implicits._ @@ -248,12 +248,6 @@ object ReplicaManager { lastStableOffset = None, exception = Some(e)) } - - class TransactionVerificationEntries { - val unverified = mutable.Map[TopicPartition, MemoryRecords]() - val errors = mutable.Map[TopicPartition, Errors]() - val verificationGuards = mutable.Map[TopicPartition, VerificationGuard]() - } } class ReplicaManager(val config: KafkaConfig, @@ -857,6 +851,90 @@ class ReplicaManager(val config: KafkaConfig, } } + /** + * Handles the produce request by starting any transactional verification before appending. + * + * @param timeout maximum time we will wait to append before returning + * @param requiredAcks number of replicas who must acknowledge the append before sending the response + * @param internalTopicsAllowed boolean indicating whether internal topics can be appended to + * @param origin source of the append request (ie, client, replication, coordinator) + * @param entriesPerPartition the records per partition to be appended + * @param responseCallback callback for sending the response + * @param delayedProduceLock lock for the delayed actions + * @param recordValidationStatsCallback callback for updating stats on record conversions + * @param requestLocal container for the stateful instances scoped to this request -- this must correspond to the + * thread calling this method + * @param actionQueue the action queue to use. ReplicaManager#defaultActionQueue is used by default. + * @param verificationGuards the mapping from topic partition to verification guards if transaction verification is used + */ + def handleProduceAppend(timeout: Long, + requiredAcks: Short, + internalTopicsAllowed: Boolean, + origin: AppendOrigin, + transactionalId: String, + entriesPerPartition: Map[TopicPartition, MemoryRecords], + responseCallback: Map[TopicPartition, PartitionResponse] => Unit, + recordValidationStatsCallback: Map[TopicPartition, RecordValidationStats] => Unit = _ => (), + requestLocal: RequestLocal = RequestLocal.NoCaching, + actionQueue: ActionQueue = this.defaultActionQueue): Unit = { + + val transactionalProducerInfo = mutable.HashSet[(Long, Short)]() + val topicPartitionBatchInfo = mutable.Map[TopicPartition, Int]() + entriesPerPartition.foreach { case (topicPartition, records) => + // Produce requests (only requests that require verification) should only have one batch per partition in "batches" but check all just to be safe. + val transactionalBatches = records.batches.asScala.filter(batch => batch.hasProducerId && batch.isTransactional) + transactionalBatches.foreach(batch => transactionalProducerInfo.add(batch.producerId, batch.producerEpoch)) + if (!transactionalBatches.isEmpty) topicPartitionBatchInfo.put(topicPartition, records.firstBatch.baseSequence) + } + if (transactionalProducerInfo.size > 1) { + throw new InvalidPidMappingException("Transactional records contained more than one producer ID") + } + + def postVerificationCallback(preAppendErrors: Map[TopicPartition, Errors], + newRequestLocal: RequestLocal, + verificationGuards: Map[TopicPartition, VerificationGuard]): Unit = { + val errorResults = preAppendErrors.map { + case (topicPartition, error) => + // translate transaction coordinator errors to known producer response errors + val customException = + error match { + case Errors.INVALID_TXN_STATE => Some(error.exception("Partition was not added to the transaction")) + case Errors.CONCURRENT_TRANSACTIONS | + Errors.COORDINATOR_LOAD_IN_PROGRESS | + Errors.COORDINATOR_NOT_AVAILABLE | + Errors.NOT_COORDINATOR => Some(new NotEnoughReplicasException( + s"Unable to verify the partition has been added to the transaction. Underlying error: ${error.toString}")) + case _ => None + } + topicPartition -> LogAppendResult( + LogAppendInfo.UNKNOWN_LOG_APPEND_INFO, + Some(customException.getOrElse(error.exception)), + hasCustomErrorMessage = customException.isDefined + ) + } + + appendRecords( + timeout = timeout, + requiredAcks = requiredAcks, + internalTopicsAllowed = internalTopicsAllowed, + origin = origin, + entriesPerPartition = entriesPerPartition, + responseCallback = responseCallback, + recordValidationStatsCallback = recordValidationStatsCallback, + requestLocal = newRequestLocal, + verificationGuards = verificationGuards, + preAppendErrors = errorResults + ) + } + + if (transactionalProducerInfo.size < 1) { + postVerificationCallback(Map.empty[TopicPartition, Errors], requestLocal, Map.empty[TopicPartition, VerificationGuard]) + return + } + maybeStartTransactionVerificationForPartitions(topicPartitionBatchInfo, transactionalId, + transactionalProducerInfo.head._1, transactionalProducerInfo.head._2, requestLocal, postVerificationCallback) + } + private def sendInvalidRequiredAcksResponse(entries: Map[TopicPartition, MemoryRecords], responseCallback: Map[TopicPartition, PartitionResponse] => Unit): Unit = { // If required.acks is outside accepted range, something is wrong with the client @@ -872,114 +950,110 @@ class ReplicaManager(val config: KafkaConfig, responseCallback(responseStatus) } - /** - * Apply the postVerificationCallback asynchronously only after verifying the partitions have been added to the transaction. - * The postVerificationCallback takes the arguments of the requestLocal for the thread that will be doing the append as - * well as a mapping of topic partitions to LogAppendResult for the partitions that saw errors when verifying. - * - * This method will start the verification process for all the topicPartitions in entriesPerPartition and supply the - * postVerificationCallback to be run on a request handler thread when the response is received. - * - * @param entriesPerPartition the records per partition to be appended and therefore need verification - * @param transactionVerificationEntries the object that will store the entries to verify, the errors, and the verification guards - * @param transactionalId the id for the transaction - * @param requestLocal container for the stateful instances scoped to this request -- this must correspond to the - * thread calling this method - * @param postVerificationCallback the method to be called when verification completes and the verification errors - * and the thread's RequestLocal are supplied - */ - def appendRecordsWithTransactionVerification(entriesPerPartition: Map[TopicPartition, MemoryRecords], - transactionVerificationEntries: TransactionVerificationEntries, - transactionalId: String, - requestLocal: RequestLocal, - postVerificationCallback: RequestLocal => Map[TopicPartition, LogAppendResult] => Unit): Unit = { - if (transactionalId != null && config.transactionPartitionVerificationEnable && addPartitionsToTxnManager.isDefined) - partitionEntriesForVerification(transactionVerificationEntries, entriesPerPartition) - - val onVerificationComplete: (RequestLocal, Map[TopicPartition, Errors]) => Unit = - executePostVerificationCallback( - transactionVerificationEntries, - postVerificationCallback, + def maybeStartTransactionVerificationForPartition( + topicPartition: TopicPartition, + transactionalId: String, + producerId: Long, + producerEpoch: Short, + baseSequence: Int, + requestLocal: RequestLocal, + callback: (Errors, RequestLocal, VerificationGuard) => Unit + ): Unit = { + def generalizedCallback(preAppendErrors: Map[TopicPartition, Errors], + newRequestLocal: RequestLocal, + verificationGuards: Map[TopicPartition, VerificationGuard]): Unit = { + callback(preAppendErrors.getOrElse(topicPartition, Errors.NONE), newRequestLocal, verificationGuards.getOrElse(topicPartition, VerificationGuard.SENTINEL)) + } + + maybeStartTransactionVerificationForPartitions( + Map(topicPartition -> baseSequence), + transactionalId, + producerId, + producerEpoch, + requestLocal, + generalizedCallback + ) + } + + def maybeStartTransactionVerificationForPartitions( + topicPartitionBatchInfo: Map[TopicPartition, Int], + transactionalId: String, + producerId: Long, + producerEpoch: Short, + requestLocal: RequestLocal, + callback: (Map[TopicPartition, Errors], RequestLocal, Map[TopicPartition, VerificationGuard]) => Unit + ): Unit = { + // Skip verification if the request is not transactional or transaction verification is disabled. + if (transactionalId == null || + !config.transactionPartitionVerificationEnable + || addPartitionsToTxnManager.isEmpty + ) { + callback(Map.empty[TopicPartition, Errors], requestLocal, Map.empty[TopicPartition, VerificationGuard]) + return + } + + // Wrap the callback to be handled on an arbitrary request handler thread when transaction verification is complete. + val verificationGuards = mutable.Map[TopicPartition, VerificationGuard]() + val errors = mutable.Map[TopicPartition, Errors]() + + topicPartitionBatchInfo.map { case (topicPartition, baseSequence) => + val errorOrGuard = maybeStartTransactionVerificationForPartition( + topicPartition, + producerId, + producerEpoch, + baseSequence ) - if (transactionVerificationEntries.unverified.isEmpty) { - onVerificationComplete(requestLocal, transactionVerificationEntries.errors.toMap) - } else { - // For unverified entries, send a request to verify. When verified, the append process will proceed via the callback. - // We verify above that all partitions use the same producer ID. - val batchInfo = transactionVerificationEntries.unverified.head._2.firstBatch() - addPartitionsToTxnManager.foreach(_.verifyTransaction( - transactionalId = transactionalId, - producerId = batchInfo.producerId, - producerEpoch = batchInfo.producerEpoch, - topicPartitions = transactionVerificationEntries.unverified.keySet.toSeq, - callback = KafkaRequestHandler.wrapAsyncCallback(onVerificationComplete, requestLocal) - )) + errorOrGuard match { + case Left(error) => errors.put(topicPartition, error) + case Right(verificationGuard) => if (verificationGuard != VerificationGuard.SENTINEL) + verificationGuards.put(topicPartition, verificationGuard) + } } - } - /** - * A helper method to compile the results from the transaction verification and call the postVerificationCallback. - * - * @param transactionVerificationEntries the object that will store the entries to verify, the errors, and the verification guards - * @param postVerificationCallback the method to be called when verification completes and the verification errors - * and the thread's RequestLocal are supplied - * @param requestLocal container for the stateful instances scoped to this request -- this must correspond to the - * thread calling this method - * - */ - private def executePostVerificationCallback(transactionVerificationEntries: TransactionVerificationEntries, - postVerificationCallback: RequestLocal => Map[TopicPartition, LogAppendResult] => Unit) - (requestLocal: RequestLocal, unverifiedEntries: Map[TopicPartition, Errors]): Unit = { - val errorResults = (unverifiedEntries ++ transactionVerificationEntries.errors).map { - case (topicPartition, error) => - // translate transaction coordinator errors to known producer response errors - val customException = - error match { - case Errors.INVALID_TXN_STATE => Some(error.exception("Partition was not added to the transaction")) - case Errors.CONCURRENT_TRANSACTIONS | - Errors.COORDINATOR_LOAD_IN_PROGRESS | - Errors.COORDINATOR_NOT_AVAILABLE | - Errors.NOT_COORDINATOR => Some(new NotEnoughReplicasException( - s"Unable to verify the partition has been added to the transaction. Underlying error: ${error.toString}")) - case _ => None - } - topicPartition -> LogAppendResult( - LogAppendInfo.UNKNOWN_LOG_APPEND_INFO, - Some(customException.getOrElse(error.exception)), - hasCustomErrorMessage = customException.isDefined - ) + // All partitions are already verified + if (verificationGuards.isEmpty) { + callback(Map.empty[TopicPartition, Errors], requestLocal, Map.empty[TopicPartition, VerificationGuard]) + return } - postVerificationCallback(requestLocal)(errorResults) - } - private def partitionEntriesForVerification(transactionVerificationEntries: TransactionVerificationEntries, entriesPerPartition: Map[TopicPartition, MemoryRecords]): TransactionVerificationEntries = { - val transactionalProducerIds = mutable.HashSet[Long]() - entriesPerPartition.foreach { case (topicPartition, records) => - try { - // Produce requests (only requests that require verification) should only have one batch per partition in "batches" but check all just to be safe. - val transactionalBatches = records.batches.asScala.filter(batch => batch.hasProducerId && batch.isTransactional) - transactionalBatches.foreach(batch => transactionalProducerIds.add(batch.producerId)) - - // If there is no producer ID or transactional records in the batches, no need to verify. - if (transactionalBatches.nonEmpty) { - // We return VerificationGuard if the partition needs to be verified. If no state is present, no need to verify. - val firstBatch = records.firstBatch - val verificationGuard = getPartitionOrException(topicPartition).maybeStartTransactionVerification(firstBatch.producerId, firstBatch.baseSequence, firstBatch.producerEpoch) - if (verificationGuard != VerificationGuard.SENTINEL) { - transactionVerificationEntries.verificationGuards.put(topicPartition, verificationGuard) - transactionVerificationEntries.unverified.put(topicPartition, records) - } - } - } catch { - case e: Exception => transactionVerificationEntries.errors.put(topicPartition, Errors.forException(e)) - } + def invokeCallback( + requestLocal: RequestLocal, + verificationErrors: Map[TopicPartition, Errors] + ): Unit = { + callback(errors ++ verificationErrors, requestLocal, verificationGuards.toMap) } - // We should have exactly one producer ID for transactional records - if (transactionalProducerIds.size > 1) { - throw new InvalidPidMappingException("Transactional records contained more than one producer ID") + + // Wrap the callback to be handled on an arbitrary request handler thread when transaction verification is complete. + val verificationCallback = KafkaRequestHandler.wrapAsyncCallback( + invokeCallback, + requestLocal + ) + + addPartitionsToTxnManager.get.verifyTransaction( + transactionalId = transactionalId, + producerId = producerId, + producerEpoch = producerEpoch, + topicPartitions = verificationGuards.keys.toSeq, + callback = verificationCallback + ) + + } + + private def maybeStartTransactionVerificationForPartition( + topicPartition: TopicPartition, + producerId: Long, + producerEpoch: Short, + baseSequence: Int + ): Either[Errors, VerificationGuard] = { + try { + val verificationGuard = getPartitionOrException(topicPartition) + .maybeStartTransactionVerification(producerId, baseSequence, producerEpoch) + Right(verificationGuard) + } catch { + case e: Exception => + Left(Errors.forException(e)) } - transactionVerificationEntries } /** diff --git a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala index 9fb31882fa91c..be06ccdb2e214 100644 --- a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala @@ -23,7 +23,6 @@ import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.locks.Lock import kafka.coordinator.AbstractCoordinatorConcurrencyTest._ import kafka.log.UnifiedLog -import kafka.server.ReplicaManager.TransactionVerificationEntries import kafka.server._ import kafka.utils._ import kafka.zk.KafkaZkClient @@ -169,6 +168,19 @@ object AbstractCoordinatorConcurrencyTest { watchKeys = Collections.newSetFromMap(new ConcurrentHashMap[TopicPartitionOperationKey, java.lang.Boolean]()).asScala } + override def maybeStartTransactionVerificationForPartition( + topicPartition: TopicPartition, + transactionalId: String, + producerId: Long, + producerEpoch: Short, + baseSequence: Int, + requestLocal: RequestLocal, + callback: (Errors, RequestLocal, VerificationGuard) => Unit + ): Unit = { + // Skip verification + callback(Errors.NONE, requestLocal, VerificationGuard.SENTINEL) + } + override def tryCompleteActions(): Unit = watchKeys.map(producePurgatory.checkAndComplete) override def appendRecords(timeout: Long, @@ -211,14 +223,6 @@ object AbstractCoordinatorConcurrencyTest { producePurgatory.tryCompleteElseWatch(delayedProduce, producerRequestKeys) } - override def appendRecordsWithTransactionVerification(entriesPerPartition: Map[TopicPartition, MemoryRecords], - transactionVerificationEntries: TransactionVerificationEntries, - transactionalId: String, - requestLocal: RequestLocal, - postVerificationCallback: RequestLocal => Map[TopicPartition, LogAppendResult] => Unit): Unit = { - postVerificationCallback(requestLocal)(Map.empty) - } - override def getMagic(topicPartition: TopicPartition): Option[Byte] = { Some(RecordBatch.MAGIC_VALUE_V2) } diff --git a/core/src/test/scala/unit/kafka/coordinator/group/CoordinatorPartitionWriterTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/CoordinatorPartitionWriterTest.scala index 121a1f119a1ce..283a938ea1110 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/CoordinatorPartitionWriterTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/CoordinatorPartitionWriterTest.scala @@ -112,6 +112,7 @@ class CoordinatorPartitionWriterTest { ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), + ArgumentMatchers.any(), ArgumentMatchers.any() )).thenAnswer( _ => { callbackCapture.getValue.apply(Map( @@ -187,6 +188,7 @@ class CoordinatorPartitionWriterTest { ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), + ArgumentMatchers.any(), ArgumentMatchers.any() )).thenAnswer(_ => { callbackCapture.getValue.apply(Map( @@ -267,6 +269,7 @@ class CoordinatorPartitionWriterTest { ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), + ArgumentMatchers.any(), ArgumentMatchers.any() )).thenAnswer(_ => { callbackCapture.getValue.apply(Map( diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala index 9de43311abdbb..39a558610192a 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala @@ -19,7 +19,7 @@ package kafka.coordinator.group import java.util.{Optional, OptionalInt} import kafka.common.OffsetAndMetadata -import kafka.server.{DelayedOperationPurgatory, HostedPartition, KafkaConfig, LogAppendResult, ReplicaManager, RequestLocal} +import kafka.server.{DelayedOperationPurgatory, HostedPartition, KafkaConfig, ReplicaManager, RequestLocal} import kafka.utils._ import org.apache.kafka.common.{TopicIdPartition, TopicPartition, Uuid} import org.apache.kafka.common.protocol.Errors @@ -38,7 +38,7 @@ import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity import org.apache.kafka.server.util.timer.MockTimer import org.apache.kafka.server.util.{KafkaScheduler, MockTime} -import org.apache.kafka.storage.internals.log.{AppendOrigin, LogAppendInfo} +import org.apache.kafka.storage.internals.log.{AppendOrigin, VerificationGuard} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} import org.junit.jupiter.params.ParameterizedTest @@ -48,7 +48,7 @@ import org.mockito.ArgumentMatchers.{any, anyLong, anyShort} import org.mockito.Mockito.{mock, when} import scala.jdk.CollectionConverters._ -import scala.collection.{Seq, mutable, Map => SMap} +import scala.collection.{Seq, mutable} import scala.collection.mutable.ArrayBuffer import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future, Promise, TimeoutException} @@ -3791,19 +3791,18 @@ class GroupCoordinatorTest { val producerId = 1000L val producerEpoch: Short = 2 - def verifyErrors(errors: Map[TopicIdPartition, Errors]): Unit = { + def verifyErrors(error: Errors): Unit = { val commitOffsetResult = commitTransactionalOffsets(groupId, producerId, producerEpoch, Map(tip1 -> offset1, tip2 -> offset2), - verificationErrors = errors) - errors.foreach { - case (tip, error) => assertEquals(error, commitOffsetResult(tip)) - } + verificationError = error) + assertEquals(error, commitOffsetResult(tip1)) + assertEquals(error, commitOffsetResult(tip2)) } - verifyErrors(Map(tip1 -> Errors.INVALID_TXN_STATE, tip2 -> Errors.INVALID_PRODUCER_ID_MAPPING)) - verifyErrors(Map(tip2 -> Errors.INVALID_TXN_STATE)) + verifyErrors(Errors.INVALID_PRODUCER_ID_MAPPING) + verifyErrors(Errors.INVALID_TXN_STATE) } @Test @@ -4110,25 +4109,20 @@ class GroupCoordinatorTest { memberId: String = JoinGroupRequest.UNKNOWN_MEMBER_ID, groupInstanceId: Option[String] = Option.empty, generationId: Int = JoinGroupRequest.UNKNOWN_GENERATION_ID, - verificationErrors: Map[TopicIdPartition, Errors] = Map.empty[TopicIdPartition, Errors]) : CommitOffsetCallbackParams = { + verificationError: Errors = Errors.NONE) : CommitOffsetCallbackParams = { val (responseFuture, responseCallback) = setupCommitOffsetsCallback val capturedArgument: ArgumentCaptor[scala.collection.Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[scala.collection.Map[TopicPartition, PartitionResponse] => Unit]) // Since transactional ID is only used in appendRecords, we can use a dummy value. Ensure it passes through. val transactionalId = "dummy-txn-id" + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupCoordinator.partitionFor(groupId)) - val preAppendErrors = verificationErrors.map { case (tp, error) => - tp.topicPartition -> LogAppendResult( - LogAppendInfo.UNKNOWN_LOG_APPEND_INFO, - Some(error.exception()), - hasCustomErrorMessage = false) - } + val postVerificationCallback: ArgumentCaptor[(Errors, RequestLocal, VerificationGuard) => Unit] = ArgumentCaptor.forClass(classOf[(Errors, RequestLocal, VerificationGuard) => Unit]) - val postVerificationCallback: ArgumentCaptor[RequestLocal => SMap[TopicPartition, LogAppendResult] => Unit] = ArgumentCaptor.forClass( - classOf[RequestLocal => SMap[TopicPartition, LogAppendResult] => Unit]) - when(replicaManager.appendRecordsWithTransactionVerification(any(), any(), ArgumentMatchers.eq(transactionalId), any(), postVerificationCallback.capture())).thenAnswer( - _ => postVerificationCallback.getValue()(RequestLocal.NoCaching)(preAppendErrors) + when(replicaManager.maybeStartTransactionVerificationForPartition(ArgumentMatchers.eq(offsetTopicPartition), ArgumentMatchers.eq(transactionalId), + ArgumentMatchers.eq(producerId), ArgumentMatchers.eq(producerEpoch), any(), any(), postVerificationCallback.capture())).thenAnswer( + _ => postVerificationCallback.getValue()(verificationError, RequestLocal.NoCaching, VerificationGuard.SENTINEL) ) when(replicaManager.appendRecords(anyLong, anyShort(), @@ -4144,7 +4138,7 @@ class GroupCoordinatorTest { any() )).thenAnswer(_ => { capturedArgument.getValue.apply( - Map(new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupCoordinator.partitionFor(groupId)) -> + Map(offsetTopicPartition -> new PartitionResponse(Errors.NONE, 0L, RecordBatch.NO_TIMESTAMP, 0L) ) ) diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala index cfe9659272d76..0c30a9c2a2f6f 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala @@ -27,8 +27,7 @@ import javax.management.ObjectName import kafka.cluster.Partition import kafka.common.OffsetAndMetadata import kafka.log.UnifiedLog -import kafka.server.ReplicaManager.TransactionVerificationEntries -import kafka.server.{HostedPartition, KafkaConfig, LogAppendResult, ReplicaManager, RequestLocal} +import kafka.server.{HostedPartition, KafkaConfig, ReplicaManager, RequestLocal} import kafka.utils.TestUtils import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription @@ -48,11 +47,9 @@ import org.apache.kafka.server.common.MetadataVersion import org.apache.kafka.server.common.MetadataVersion._ import org.apache.kafka.server.metrics.KafkaYammerMetrics import org.apache.kafka.server.util.{KafkaScheduler, MockTime} -import org.apache.kafka.storage.internals.log.{AppendOrigin, FetchDataInfo, FetchIsolation, LogAppendInfo, LogOffsetMetadata} +import org.apache.kafka.storage.internals.log.{AppendOrigin, FetchDataInfo, FetchIsolation, LogAppendInfo, LogOffsetMetadata, VerificationGuard} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} -import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.EnumSource import org.mockito.{ArgumentCaptor, ArgumentMatchers} import org.mockito.ArgumentMatchers.{any, anyInt, anyLong, anyShort} import org.mockito.Mockito.{mock, reset, times, verify, when} @@ -1269,6 +1266,7 @@ class GroupMetadataManagerTest { val group = new GroupMetadata(groupId, Empty, time) groupMetadataManager.addGroup(group) + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupMetadataManager.partitionFor(group.groupId)) val offsets = immutable.Map(topicIdPartition -> OffsetAndMetadata(offset, "", time.milliseconds())) expectAppendMessage(Errors.NONE) @@ -1278,7 +1276,7 @@ class GroupMetadataManagerTest { } assertEquals(0, TestUtils.totalMetricValue(metrics, "offset-commit-count")) - groupMetadataManager.storeOffsets(group, memberId, offsets, callback) + groupMetadataManager.storeOffsets(group, memberId, offsetTopicPartition, offsets, callback, verificationGuard = None) assertTrue(group.hasOffsets) assertFalse(commitErrors.isEmpty) @@ -1323,16 +1321,19 @@ class GroupMetadataManagerTest { val group = new GroupMetadata(groupId, Empty, time) groupMetadataManager.addGroup(group) + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupMetadataManager.partitionFor(group.groupId)) val offsetAndMetadata = OffsetAndMetadata(offset, "", time.milliseconds()) val offsets = immutable.Map(topicIdPartition -> offsetAndMetadata) val capturedResponseCallback: ArgumentCaptor[Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[Map[TopicPartition, PartitionResponse] => Unit]) + when(replicaManager.getMagic(any())).thenReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE)) var commitErrors: Option[immutable.Map[TopicIdPartition, Errors]] = None def callback(errors: immutable.Map[TopicIdPartition, Errors]): Unit = { commitErrors = Some(errors) } + val verificationGuard = new VerificationGuard() - storeOffsetsWithVerification(group, memberId, offsets, callback, producerId, producerEpoch) + groupMetadataManager.storeOffsets(group, memberId, offsetTopicPartition, offsets, callback, producerId, producerEpoch, verificationGuard = Some(verificationGuard)) assertTrue(group.hasOffsets) assertTrue(group.allOffsets.isEmpty) @@ -1346,8 +1347,9 @@ class GroupMetadataManagerTest { any(), any(), any(), - any(), + ArgumentMatchers.eq(Map(offsetTopicPartition -> verificationGuard)), any()) + verify(replicaManager).getMagic(any()) capturedResponseCallback.getValue.apply(Map(groupTopicPartition -> new PartitionResponse(Errors.NONE, 0L, RecordBatch.NO_TIMESTAMP, 0L))) @@ -1373,14 +1375,18 @@ class GroupMetadataManagerTest { val group = new GroupMetadata(groupId, Empty, time) groupMetadataManager.addGroup(group) + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupMetadataManager.partitionFor(group.groupId)) val offsets = immutable.Map(topicIdPartition -> OffsetAndMetadata(offset, "", time.milliseconds())) + when(replicaManager.getMagic(any())).thenReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE)) + var commitErrors: Option[immutable.Map[TopicIdPartition, Errors]] = None def callback(errors: immutable.Map[TopicIdPartition, Errors]): Unit = { commitErrors = Some(errors) } + val verificationGuard = new VerificationGuard() - storeOffsetsWithVerification(group, memberId, offsets, callback, producerId, producerEpoch) + groupMetadataManager.storeOffsets(group, memberId, offsetTopicPartition, offsets, callback, producerId, producerEpoch, verificationGuard = Some(verificationGuard)) assertTrue(group.hasOffsets) assertTrue(group.allOffsets.isEmpty) val capturedResponseCallback = verifyAppendAndCaptureCallback() @@ -1404,8 +1410,9 @@ class GroupMetadataManagerTest { any(), any(), any(), - any(), + ArgumentMatchers.eq(Map(offsetTopicPartition -> verificationGuard)), any()) + verify(replicaManager).getMagic(any()) } @Test @@ -1421,14 +1428,18 @@ class GroupMetadataManagerTest { val group = new GroupMetadata(groupId, Empty, time) groupMetadataManager.addGroup(group) + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupMetadataManager.partitionFor(group.groupId)) val offsets = immutable.Map(topicIdPartition -> OffsetAndMetadata(offset, "", time.milliseconds())) + when(replicaManager.getMagic(any())).thenReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE)) + var commitErrors: Option[immutable.Map[TopicIdPartition, Errors]] = None def callback(errors: immutable.Map[TopicIdPartition, Errors]): Unit = { commitErrors = Some(errors) } + val verificationGuard = new VerificationGuard() - storeOffsetsWithVerification(group, memberId, offsets, callback, producerId, producerEpoch) + groupMetadataManager.storeOffsets(group, memberId, offsetTopicPartition, offsets, callback, producerId, producerEpoch, verificationGuard = Some(verificationGuard)) assertTrue(group.hasOffsets) assertTrue(group.allOffsets.isEmpty) val capturedResponseCallback = verifyAppendAndCaptureCallback() @@ -1452,59 +1463,9 @@ class GroupMetadataManagerTest { any(), any(), any(), - any(), - any()) - } - - @ParameterizedTest - @EnumSource(value = classOf[Errors], names = Array("INVALID_TXN_STATE", "INVALID_PRODUCER_ID_MAPPING")) - def testTransactionalCommitOffsetTransactionalErrors(error: Errors): Unit = { - val memberId = "" - val topicId = Uuid.randomUuid() - val topicIdPartition1 = new TopicIdPartition(topicId, 0, "foo") - val topicIdPartition2 = new TopicIdPartition(topicId, 1, "foo") - val offset = 37 - val producerId = 232L - val producerEpoch = 0.toShort - - groupMetadataManager.addOwnedPartition(groupPartitionId) - - val group = new GroupMetadata(groupId, Empty, time) - groupMetadataManager.addGroup(group) - - val offsets = immutable.Map(topicIdPartition1 -> OffsetAndMetadata(offset, "", time.milliseconds()), - topicIdPartition2 -> OffsetAndMetadata(offset, "", time.milliseconds())) - - var commitErrors: Option[immutable.Map[TopicIdPartition, Errors]] = None - def callback(errors: immutable.Map[TopicIdPartition, Errors]): Unit = { - commitErrors = Some(errors) - } - - storeOffsetsWithVerification(group, memberId, offsets, callback, producerId, producerEpoch, immutable.Map(topicIdPartition1 -> error)) - assertTrue(group.hasOffsets) - val capturedResponseCallback = verifyAppendAndCaptureCallback() - capturedResponseCallback.getValue.apply(Map(groupTopicPartition -> - new PartitionResponse(Errors.NONE, 0L, RecordBatch.NO_TIMESTAMP, 0L))) - - group.completePendingTxnOffsetCommit(producerId, isCommit = true) - assertTrue(commitErrors.isDefined) - assertEquals(error, commitErrors.get(topicIdPartition1)) - assertEquals(Errors.NONE, commitErrors.get(topicIdPartition2)) - assertTrue(group.offset(topicIdPartition1.topicPartition).isEmpty) - assertTrue(group.offset(topicIdPartition2.topicPartition).isDefined) - - verify(replicaManager).appendRecords(anyLong(), - anyShort(), - internalTopicsAllowed = ArgumentMatchers.eq(true), - origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), - any[Map[TopicPartition, MemoryRecords]], - any(), - any[Option[ReentrantLock]], - any(), - any(), - any(), - any(), + ArgumentMatchers.eq(Map(offsetTopicPartition -> verificationGuard)), any()) + verify(replicaManager).getMagic(any()) } @Test @@ -1519,6 +1480,7 @@ class GroupMetadataManagerTest { val group = new GroupMetadata(groupId, Empty, time) groupMetadataManager.addGroup(group) + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupMetadataManager.partitionFor(group.groupId)) val offsets = immutable.Map(topicIdPartition -> OffsetAndMetadata(offset, "", time.milliseconds())) var commitErrors: Option[immutable.Map[TopicIdPartition, Errors]] = None @@ -1526,7 +1488,7 @@ class GroupMetadataManagerTest { commitErrors = Some(errors) } - groupMetadataManager.storeOffsets(group, memberId, offsets, callback) + groupMetadataManager.storeOffsets(group, memberId, offsetTopicPartition, offsets, callback, verificationGuard = None) assertFalse(commitErrors.isEmpty) val maybeError = commitErrors.get.get(topicIdPartition) @@ -1558,6 +1520,7 @@ class GroupMetadataManagerTest { val group = new GroupMetadata(groupId, Empty, time) groupMetadataManager.addGroup(group) + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupMetadataManager.partitionFor(group.groupId)) val offsets = immutable.Map(topicIdPartition -> OffsetAndMetadata(offset, "", time.milliseconds())) when(replicaManager.getMagic(any())).thenReturn(Some(RecordBatch.CURRENT_MAGIC_VALUE)) @@ -1568,7 +1531,7 @@ class GroupMetadataManagerTest { } assertEquals(0, TestUtils.totalMetricValue(metrics, "offset-commit-count")) - groupMetadataManager.storeOffsets(group, memberId, offsets, callback) + groupMetadataManager.storeOffsets(group, memberId, offsetTopicPartition, offsets, callback, verificationGuard = None) assertTrue(group.hasOffsets) val capturedResponseCallback = verifyAppendAndCaptureCallback() capturedResponseCallback.getValue.apply(Map(groupTopicPartition -> @@ -1606,6 +1569,7 @@ class GroupMetadataManagerTest { val group = new GroupMetadata(groupId, Empty, time) groupMetadataManager.addGroup(group) + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupMetadataManager.partitionFor(group.groupId)) val offsets = immutable.Map( topicIdPartition -> OffsetAndMetadata(offset, "", time.milliseconds()), // This will failed @@ -1620,7 +1584,7 @@ class GroupMetadataManagerTest { } assertEquals(0, TestUtils.totalMetricValue(metrics, "offset-commit-count")) - groupMetadataManager.storeOffsets(group, memberId, offsets, callback) + groupMetadataManager.storeOffsets(group, memberId, offsetTopicPartition, offsets, callback, verificationGuard = None) assertTrue(group.hasOffsets) val capturedResponseCallback = verifyAppendAndCaptureCallback() capturedResponseCallback.getValue.apply(Map(groupTopicPartition -> @@ -1671,6 +1635,7 @@ class GroupMetadataManagerTest { val group = new GroupMetadata(groupId, Empty, time) groupMetadataManager.addGroup(group) + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupMetadataManager.partitionFor(group.groupId)) val offsets = immutable.Map( topicIdPartition -> OffsetAndMetadata(offset, "s" * (offsetConfig.maxMetadataSize + 1) , time.milliseconds()) ) @@ -1681,7 +1646,7 @@ class GroupMetadataManagerTest { } assertEquals(0, TestUtils.totalMetricValue(metrics, "offset-commit-count")) - groupMetadataManager.storeOffsets(group, memberId, offsets, callback) + groupMetadataManager.storeOffsets(group, memberId, offsetTopicPartition, offsets, callback, verificationGuard = None) assertFalse(group.hasOffsets) assertFalse(commitErrors.isEmpty) @@ -1714,6 +1679,7 @@ class GroupMetadataManagerTest { val group = new GroupMetadata(groupId, Empty, time) groupMetadataManager.addGroup(group) + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupMetadataManager.partitionFor(group.groupId)) // expire the offset after 1 millisecond val startMs = time.milliseconds val offsets = immutable.Map( @@ -1727,7 +1693,7 @@ class GroupMetadataManagerTest { commitErrors = Some(errors) } - groupMetadataManager.storeOffsets(group, memberId, offsets, callback) + groupMetadataManager.storeOffsets(group, memberId, offsetTopicPartition, offsets, callback, verificationGuard = None) assertTrue(group.hasOffsets) assertFalse(commitErrors.isEmpty) @@ -1870,6 +1836,7 @@ class GroupMetadataManagerTest { val group = new GroupMetadata(groupId, Empty, time) groupMetadataManager.addGroup(group) + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupMetadataManager.partitionFor(group.groupId)) // expire the offset after 1 millisecond val startMs = time.milliseconds val offsets = immutable.Map( @@ -1883,7 +1850,7 @@ class GroupMetadataManagerTest { commitErrors = Some(errors) } - groupMetadataManager.storeOffsets(group, memberId, offsets, callback) + groupMetadataManager.storeOffsets(group, memberId, offsetTopicPartition, offsets, callback, verificationGuard = None) assertTrue(group.hasOffsets) assertFalse(commitErrors.isEmpty) @@ -1953,6 +1920,7 @@ class GroupMetadataManagerTest { group.transitionTo(PreparingRebalance) group.initNextGeneration() + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupMetadataManager.partitionFor(group.groupId)) val startMs = time.milliseconds // old clients, expiry timestamp is explicitly set val tp1OffsetAndMetadata = OffsetAndMetadata(offset, "", startMs, startMs + 1) @@ -1971,7 +1939,7 @@ class GroupMetadataManagerTest { commitErrors = Some(errors) } - groupMetadataManager.storeOffsets(group, memberId, offsets, callback) + groupMetadataManager.storeOffsets(group, memberId, offsetTopicPartition, offsets, callback, verificationGuard = None) assertTrue(group.hasOffsets) assertFalse(commitErrors.isEmpty) @@ -2131,6 +2099,7 @@ class GroupMetadataManagerTest { val group = new GroupMetadata(groupId, Empty, time) groupMetadataManager.addGroup(group) + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupMetadataManager.partitionFor(group.groupId)) // expire the offset after 1 and 3 milliseconds (old clients) and after default retention (new clients) val startMs = time.milliseconds // old clients, expiry timestamp is explicitly set @@ -2146,7 +2115,7 @@ class GroupMetadataManagerTest { commitErrors = Some(errors) } - groupMetadataManager.storeOffsets(group, memberId, offsets, callback) + groupMetadataManager.storeOffsets(group, memberId, offsetTopicPartition, offsets, callback, verificationGuard = None) assertTrue(group.hasOffsets) assertFalse(commitErrors.isEmpty) @@ -2239,6 +2208,7 @@ class GroupMetadataManagerTest { group.initNextGeneration() group.transitionTo(Stable) + val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupMetadataManager.partitionFor(group.groupId)) val startMs = time.milliseconds val t1p0OffsetAndMetadata = OffsetAndMetadata(offset, "", startMs) @@ -2260,7 +2230,7 @@ class GroupMetadataManagerTest { commitErrors = Some(errors) } - groupMetadataManager.storeOffsets(group, memberId, offsets, callback) + groupMetadataManager.storeOffsets(group, memberId, offsetTopicPartition, offsets, callback, verificationGuard = None) assertTrue(group.hasOffsets) assertFalse(commitErrors.isEmpty) @@ -3109,29 +3079,4 @@ class GroupMetadataManagerTest { assertTrue(group.offset(topicPartition).map(_.expireTimestamp).contains(None)) } } - - def storeOffsetsWithVerification(group: GroupMetadata, - memberId: String, - offsets: immutable.Map[TopicIdPartition, OffsetAndMetadata], - callback: immutable.Map[TopicIdPartition, Errors] => Unit, - producerId: Long, - producerEpoch: Short, - errors: immutable.Map[TopicIdPartition, Errors] = immutable.Map.empty[TopicIdPartition, Errors]): Unit = { - // Consider any non-error topic partition as verified. - val verifiedOffsets = offsets.filter { case (tp, _) => - !errors.contains(tp) - } - val preAppendErrors = errors.map { case (tp, error) => - tp.topicPartition -> LogAppendResult( - LogAppendInfo.UNKNOWN_LOG_APPEND_INFO, - Some(error.exception()), - hasCustomErrorMessage = false) - } - - val records = groupMetadataManager.generateOffsetRecords(RecordBatch.CURRENT_MAGIC_VALUE, true, group.groupId, verifiedOffsets, producerId, producerEpoch) - val putCacheCallback = groupMetadataManager.createPutCacheCallback(true, group, memberId, offsets, verifiedOffsets, callback, producerId, records, preAppendErrors) - - // Pass in new verification guard since the Log/ReplicaManager code is mocked. - groupMetadataManager.storeOffsetsAfterVerification(group, verifiedOffsets, records, putCacheCallback, producerId, new TransactionVerificationEntries, preAppendErrors) - } } diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 2b922c5074d1a..9be1296f2354b 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -30,7 +30,6 @@ import kafka.coordinator.transaction.{InitProducerIdResult, TransactionCoordinat import kafka.log.UnifiedLog import kafka.network.{RequestChannel, RequestMetrics} import kafka.server.QuotaFactory.QuotaManagers -import kafka.server.ReplicaManager.TransactionVerificationEntries import kafka.server.metadata.{ConfigRepository, KRaftMetadataCache, MockConfigRepository, ZkMetadataCache} import kafka.utils.{Log4jController, TestUtils} import kafka.zk.KafkaZkClient @@ -100,7 +99,7 @@ import org.apache.kafka.server.common.{Features, MetadataVersion} import org.apache.kafka.server.common.MetadataVersion.{IBP_0_10_2_IV0, IBP_2_2_IV1} import org.apache.kafka.server.metrics.ClientMetricsTestUtils import org.apache.kafka.server.util.MockTime -import org.apache.kafka.storage.internals.log.{AppendOrigin, FetchParams, FetchPartitionData, LogAppendInfo, LogConfig} +import org.apache.kafka.storage.internals.log.{AppendOrigin, FetchParams, FetchPartitionData, LogConfig} class KafkaApisTest { private val requestChannel: RequestChannel = mock(classOf[RequestChannel]) @@ -2447,18 +2446,16 @@ class KafkaApisTest { .build(version.toShort) val request = buildRequest(produceRequest) - when(replicaManager.appendRecords(anyLong, + when(replicaManager.handleProduceAppend(anyLong, anyShort, ArgumentMatchers.eq(false), ArgumentMatchers.eq(AppendOrigin.CLIENT), any(), - responseCallback.capture(), - any(), - any(), - any(), any(), + responseCallback.capture(), any(), any(), + any() )).thenAnswer(_ => responseCallback.getValue.apply(Map(tp -> new PartitionResponse(Errors.INVALID_PRODUCER_EPOCH)))) when(clientRequestQuotaManager.maybeRecordAndGetThrottleTimeMs(any[RequestChannel.Request](), @@ -2507,15 +2504,13 @@ class KafkaApisTest { .build(version.toShort) val request = buildRequest(produceRequest) - when(replicaManager.appendRecords(anyLong, + when(replicaManager.handleProduceAppend(anyLong, anyShort, ArgumentMatchers.eq(false), ArgumentMatchers.eq(AppendOrigin.CLIENT), any(), - responseCallback.capture(), - any(), - any(), any(), + responseCallback.capture(), any(), any(), any()) @@ -2574,15 +2569,13 @@ class KafkaApisTest { .build(version.toShort) val request = buildRequest(produceRequest) - when(replicaManager.appendRecords(anyLong, + when(replicaManager.handleProduceAppend(anyLong, anyShort, ArgumentMatchers.eq(false), ArgumentMatchers.eq(AppendOrigin.CLIENT), any(), - responseCallback.capture(), - any(), - any(), any(), + responseCallback.capture(), any(), any(), any()) @@ -2640,15 +2633,13 @@ class KafkaApisTest { .build(version.toShort) val request = buildRequest(produceRequest) - when(replicaManager.appendRecords(anyLong, + when(replicaManager.handleProduceAppend(anyLong, anyShort, ArgumentMatchers.eq(false), ArgumentMatchers.eq(AppendOrigin.CLIENT), any(), - responseCallback.capture(), - any(), - any(), any(), + responseCallback.capture(), any(), any(), any()) @@ -2690,11 +2681,6 @@ class KafkaApisTest { reset(replicaManager, clientQuotaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) - val responseCallback: ArgumentCaptor[Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[Map[TopicPartition, PartitionResponse] => Unit]) - val postVerificationCallback: ArgumentCaptor[RequestLocal => Map[TopicPartition, LogAppendResult] => Unit] = ArgumentCaptor.forClass( - classOf[RequestLocal => Map[TopicPartition, LogAppendResult] => Unit]) - val transactionVerificationEntries: ArgumentCaptor[TransactionVerificationEntries] = ArgumentCaptor.forClass(classOf[TransactionVerificationEntries]) - val tp = new TopicPartition("topic", 0) val produceRequest = ProduceRequest.forCurrentMagic(new ProduceRequestData() @@ -2713,41 +2699,19 @@ class KafkaApisTest { val kafkaApis = createKafkaApis() val requestLocal = RequestLocal.withThreadConfinedCaching - val newRequestLocal = RequestLocal.NoCaching - val preAppendErrors = Map(tp -> LogAppendResult( - LogAppendInfo.UNKNOWN_LOG_APPEND_INFO, - Some(Errors.INVALID_TXN_STATE.exception()), - hasCustomErrorMessage = false)) - - when(replicaManager.appendRecordsWithTransactionVerification(any(), transactionVerificationEntries.capture(), any(), any(), postVerificationCallback.capture())).thenAnswer( - _ => { - val callback = postVerificationCallback.getValue() - callback(newRequestLocal)(preAppendErrors) - } - ) - - kafkaApis.handleProduceRequest(request, requestLocal) - verify(replicaManager).appendRecordsWithTransactionVerification( - any(), - any(), - ArgumentMatchers.eq(transactionalId), - ArgumentMatchers.eq(requestLocal), - ArgumentMatchers.eq(postVerificationCallback.getValue()) - ) + kafkaApis.handleProduceRequest(request, requestLocal) - verify(replicaManager).appendRecords(anyLong, + verify(replicaManager).handleProduceAppend(anyLong, anyShort, ArgumentMatchers.eq(false), ArgumentMatchers.eq(AppendOrigin.CLIENT), + ArgumentMatchers.eq(transactionalId), any(), - responseCallback.capture(), any(), any(), - ArgumentMatchers.eq(newRequestLocal), any(), - ArgumentMatchers.eq(transactionVerificationEntries.getValue().verificationGuards), - ArgumentMatchers.eq(preAppendErrors)) + any()) } } diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index 047a17a293d91..8d7a130b0ec07 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -657,7 +657,7 @@ class ReplicaManagerTest { val sequence = 9 val records = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, epoch, sequence, new SimpleRecord(time.milliseconds(), s"message $sequence".getBytes)) - appendRecords(replicaManager, new TopicPartition(topic, 0), records, transactionalId = transactionalId).onFire { response => + appendRecordsWithTransactionVerification(replicaManager, new TopicPartition(topic, 0), records, transactionalId = transactionalId).onFire { response => assertEquals(Errors.NONE, response.error) } assertLateTransactionCount(Some(0)) @@ -721,7 +721,7 @@ class ReplicaManagerTest { for (sequence <- 0 until numRecords) { val records = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, epoch, sequence, new SimpleRecord(s"message $sequence".getBytes)) - appendRecords(replicaManager, new TopicPartition(topic, 0), records, transactionalId = transactionalId).onFire { response => + appendRecordsWithTransactionVerification(replicaManager, new TopicPartition(topic, 0), records, transactionalId = transactionalId).onFire { response => assertEquals(Errors.NONE, response.error) } } @@ -842,7 +842,7 @@ class ReplicaManagerTest { for (sequence <- 0 until numRecords) { val records = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, epoch, sequence, new SimpleRecord(s"message $sequence".getBytes)) - appendRecords(replicaManager, new TopicPartition(topic, 0), records, transactionalId = transactionalId).onFire { response => + appendRecordsWithTransactionVerification(replicaManager, new TopicPartition(topic, 0), records, transactionalId = transactionalId).onFire { response => assertEquals(Errors.NONE, response.error) } } @@ -2165,7 +2165,7 @@ class ReplicaManagerTest { // If we supply no transactional ID and idempotent records, we do not verify. val idempotentRecords = MemoryRecords.withIdempotentRecords(CompressionType.NONE, producerId, producerEpoch, sequence, new SimpleRecord("message".getBytes)) - appendRecords(replicaManager, tp0, idempotentRecords) + appendRecordsWithTransactionVerification(replicaManager, tp0, idempotentRecords, transactionalId = null) verify(addPartitionsToTxnManager, times(0)).verifyTransaction(any(), any(), any(), any(), any[AddPartitionsToTxnManager.AppendCallback]()) assertEquals(VerificationGuard.SENTINEL, getVerificationGuard(replicaManager, tp0, producerId)) @@ -2175,7 +2175,7 @@ class ReplicaManagerTest { val idempotentRecords2 = MemoryRecords.withIdempotentRecords(CompressionType.NONE, producerId, producerEpoch, sequence, new SimpleRecord("message".getBytes)) - appendRecordsToMultipleTopicsWithVerification(replicaManager, Map(tp0 -> transactionalRecords, tp1 -> idempotentRecords2), transactionalId) + appendRecordsToMultipleTopicsWithTransactionVerification(replicaManager, Map(tp0 -> transactionalRecords, tp1 -> idempotentRecords2), transactionalId) verify(addPartitionsToTxnManager, times(1)).verifyTransaction( ArgumentMatchers.eq(transactionalId), ArgumentMatchers.eq(producerId), @@ -2210,7 +2210,7 @@ class ReplicaManagerTest { new SimpleRecord("message".getBytes)) // We should add these partitions to the manager to verify. - val result = appendRecordsWithVerification(replicaManager, tp0, transactionalRecords, origin = appendOrigin, transactionalId = transactionalId) + val result = appendRecordsWithTransactionVerification(replicaManager, tp0, transactionalRecords, origin = appendOrigin, transactionalId = transactionalId) val appendCallback = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(1)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2229,7 +2229,7 @@ class ReplicaManagerTest { assertEquals(verificationGuard, getVerificationGuard(replicaManager, tp0, producerId)) // This time verification is successful. - appendRecordsWithVerification(replicaManager, tp0, transactionalRecords, origin = appendOrigin, transactionalId = transactionalId) + appendRecordsWithTransactionVerification(replicaManager, tp0, transactionalRecords, origin = appendOrigin, transactionalId = transactionalId) val appendCallback2 = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(2)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2268,7 +2268,7 @@ class ReplicaManagerTest { new SimpleRecord("message".getBytes)) // We should add these partitions to the manager to verify. - val result = appendRecordsWithVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) + val result = appendRecordsWithTransactionVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) val appendCallback = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(1)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2290,7 +2290,7 @@ class ReplicaManagerTest { val transactionalRecords2 = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, producerEpoch, sequence + 1, new SimpleRecord("message".getBytes)) - val result2 = appendRecordsWithVerification(replicaManager, tp0, transactionalRecords2, transactionalId = transactionalId) + val result2 = appendRecordsWithTransactionVerification(replicaManager, tp0, transactionalRecords2, transactionalId = transactionalId) val appendCallback2 = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(2)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2333,7 +2333,7 @@ class ReplicaManagerTest { val transactionalRecords = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, producerEpoch, sequence, new SimpleRecord(s"message $sequence".getBytes)) - appendRecordsToMultipleTopicsWithVerification(replicaManager, Map(tp0 -> transactionalRecords, tp1 -> transactionalRecords), transactionalId).onFire { responses => + appendRecordsToMultipleTopicsWithTransactionVerification(replicaManager, Map(tp0 -> transactionalRecords, tp1 -> transactionalRecords), transactionalId).onFire { responses => responses.foreach { entry => assertEquals(Errors.NONE, entry._2.error) } @@ -2373,7 +2373,7 @@ class ReplicaManagerTest { new SimpleRecord(s"message $sequence".getBytes))) assertThrows(classOf[InvalidPidMappingException], - () => appendRecordsToMultipleTopicsWithVerification(replicaManager, transactionalRecords, transactionalId = transactionalId)) + () => appendRecordsToMultipleTopicsWithTransactionVerification(replicaManager, transactionalRecords, transactionalId = transactionalId)) // We should not add these partitions to the manager to verify. verify(addPartitionsToTxnManager, times(0)).verifyTransaction(any(), any(), any(), any(), any()) } finally { @@ -2396,7 +2396,7 @@ class ReplicaManagerTest { new SimpleRecord("message".getBytes)) // We should not add these partitions to the manager to verify, but instead throw an error. - appendRecords(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId).onFire { response => + appendRecordsWithTransactionVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId).onFire { response => assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, response.error) } verify(addPartitionsToTxnManager, times(0)).verifyTransaction(any(), any(), any(), any(), any()) @@ -2427,8 +2427,9 @@ class ReplicaManagerTest { val transactionalRecords = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, producerEpoch, sequence, new SimpleRecord(s"message $sequence".getBytes)) - appendRecords(replicaManager, tp, transactionalRecords, transactionalId = transactionalId) + val result = appendRecordsWithTransactionVerification(replicaManager, tp, transactionalRecords, transactionalId = transactionalId) assertEquals(VerificationGuard.SENTINEL, getVerificationGuard(replicaManager, tp, producerId)) + assertEquals(Errors.NONE, result.assertFired.error) // We should not add these partitions to the manager to verify. verify(addPartitionsToTxnManager, times(0)).verifyTransaction(any(), any(), any(), any(), any()) @@ -2444,7 +2445,7 @@ class ReplicaManagerTest { val moreTransactionalRecords = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, producerEpoch, sequence + 1, new SimpleRecord("message".getBytes)) - appendRecords(replicaManager, tp, moreTransactionalRecords, transactionalId = transactionalId) + appendRecordsWithTransactionVerification(replicaManager, tp, moreTransactionalRecords, transactionalId = transactionalId) verify(addPartitionsToTxnManager, times(0)).verifyTransaction(any(), any(), any(), any(), any()) assertEquals(VerificationGuard.SENTINEL, getVerificationGuard(replicaManager, tp, producerId)) assertTrue(replicaManager.localLog(tp).get.hasOngoingTransaction(producerId)) @@ -2472,7 +2473,7 @@ class ReplicaManagerTest { new SimpleRecord("message".getBytes)) // We should add these partitions to the manager to verify. - val result = appendRecordsWithVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) + val result = appendRecordsWithTransactionVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) val appendCallback = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(1)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2498,7 +2499,7 @@ class ReplicaManagerTest { assertEquals(verificationGuard, getVerificationGuard(replicaManager, tp0, producerId)) // This time we do not verify - appendRecordsWithVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) + appendRecordsWithTransactionVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) verify(addPartitionsToTxnManager, times(1)).verifyTransaction(any(), any(), any(), any(), any()) assertEquals(VerificationGuard.SENTINEL, getVerificationGuard(replicaManager, tp0, producerId)) assertTrue(replicaManager.localLog(tp0).get.hasOngoingTransaction(producerId)) @@ -2527,7 +2528,7 @@ class ReplicaManagerTest { // Start verification and return the coordinator related errors. val expectedMessage = s"Unable to verify the partition has been added to the transaction. Underlying error: ${error.toString}" - val result = appendRecordsWithVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) + val result = appendRecordsWithTransactionVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) val appendCallback = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(1)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2915,8 +2916,7 @@ class ReplicaManagerTest { partition: TopicPartition, records: MemoryRecords, origin: AppendOrigin = AppendOrigin.CLIENT, - requiredAcks: Short = -1, - transactionalId: String = null): CallbackResult[PartitionResponse] = { + requiredAcks: Short = -1): CallbackResult[PartitionResponse] = { val result = new CallbackResult[PartitionResponse]() def appendCallback(responses: Map[TopicPartition, PartitionResponse]): Unit = { val response = responses.get(partition) @@ -2936,50 +2936,35 @@ class ReplicaManagerTest { result } - private def appendRecordsToMultipleTopicsWithVerification(replicaManager: ReplicaManager, - entriesToAppend: Map[TopicPartition, MemoryRecords], - transactionalId: String, - origin: AppendOrigin = AppendOrigin.CLIENT, - requiredAcks: Short = -1): CallbackResult[Map[TopicPartition, PartitionResponse]] = { + private def appendRecordsToMultipleTopicsWithTransactionVerification(replicaManager: ReplicaManager, + entriesToAppend: Map[TopicPartition, MemoryRecords], + transactionalId: String, + origin: AppendOrigin = AppendOrigin.CLIENT, + requiredAcks: Short = -1): CallbackResult[Map[TopicPartition, PartitionResponse]] = { val result = new CallbackResult[Map[TopicPartition, PartitionResponse]]() def appendCallback(responses: Map[TopicPartition, PartitionResponse]): Unit = { responses.foreach( response => assertTrue(responses.get(response._1).isDefined)) result.fire(responses) } - - val transactionVerificationEntries = new ReplicaManager.TransactionVerificationEntries - - def postVerificationCallback(newRequestLocal: RequestLocal) - (errorResults: Map[TopicPartition, LogAppendResult]): Unit = { - replicaManager.appendRecords( + replicaManager.handleProduceAppend( timeout = 1000, requiredAcks = requiredAcks, internalTopicsAllowed = false, origin = origin, + transactionalId = transactionalId, entriesPerPartition = entriesToAppend, responseCallback = appendCallback, - requestLocal = newRequestLocal, - preAppendErrors = errorResults ) - } - - replicaManager.appendRecordsWithTransactionVerification( - entriesToAppend, - transactionVerificationEntries, - transactionalId, - RequestLocal.NoCaching, - postVerificationCallback - ) result } - private def appendRecordsWithVerification(replicaManager: ReplicaManager, - partition: TopicPartition, - records: MemoryRecords, - origin: AppendOrigin = AppendOrigin.CLIENT, - requiredAcks: Short = -1, - transactionalId: String): CallbackResult[PartitionResponse] = { + private def appendRecordsWithTransactionVerification(replicaManager: ReplicaManager, + partition: TopicPartition, + records: MemoryRecords, + origin: AppendOrigin = AppendOrigin.CLIENT, + requiredAcks: Short = -1, + transactionalId: String): CallbackResult[PartitionResponse] = { val result = new CallbackResult[PartitionResponse]() def appendCallback(responses: Map[TopicPartition, PartitionResponse]): Unit = { @@ -2989,27 +2974,14 @@ class ReplicaManagerTest { } val entriesPerPartition = Map(partition -> records) - val transactionVerificationEntries = new ReplicaManager.TransactionVerificationEntries - def postVerificationCallback(newRequestLocal: RequestLocal) - (errorResults: Map[TopicPartition, LogAppendResult]): Unit = { - replicaManager.appendRecords( - timeout = 1000, - requiredAcks = requiredAcks, - internalTopicsAllowed = false, - origin = origin, - entriesPerPartition = entriesPerPartition, - responseCallback = appendCallback, - requestLocal = newRequestLocal, - preAppendErrors = errorResults - ) - } - - replicaManager.appendRecordsWithTransactionVerification( - entriesPerPartition, - transactionVerificationEntries, - transactionalId, - RequestLocal.NoCaching, - postVerificationCallback + replicaManager.handleProduceAppend( + timeout = 1000, + requiredAcks = requiredAcks, + internalTopicsAllowed = false, + origin = origin, + transactionalId = transactionalId, + entriesPerPartition = entriesPerPartition, + responseCallback = appendCallback, ) result From 965ec39460324d77d55c90771f23ac0c9c2ad70b Mon Sep 17 00:00:00 2001 From: Justine Date: Wed, 6 Dec 2023 15:11:50 -0800 Subject: [PATCH 09/24] remove package private scoping that is not needed --- .../scala/kafka/coordinator/group/GroupMetadataManager.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala index 4c5622ebe9d12..ad1fd580ddf86 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala @@ -1005,7 +1005,7 @@ class GroupMetadataManager(brokerId: Int, /* * Check if the offset metadata length is valid */ - private[coordinator] def validateOffsetMetadataLength(metadata: String) : Boolean = { + private def validateOffsetMetadataLength(metadata: String) : Boolean = { metadata == null || metadata.length() <= config.maxMetadataSize } @@ -1026,7 +1026,7 @@ class GroupMetadataManager(brokerId: Int, * @param partition Partition of GroupMetadataTopic * @return Some(MessageFormatVersion) if replica is local, None otherwise */ - private[coordinator] def getMagic(partition: Int): Option[Byte] = + private def getMagic(partition: Int): Option[Byte] = replicaManager.getMagic(new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, partition)) /** From 8ecbe4de101afe554d983783f883a28eccafef53 Mon Sep 17 00:00:00 2001 From: Justine Date: Thu, 7 Dec 2023 09:45:15 -0800 Subject: [PATCH 10/24] Remove produce path refactor --- .../group/GroupMetadataManager.scala | 2 +- .../main/scala/kafka/server/KafkaApis.scala | 12 +- .../scala/kafka/server/ReplicaManager.scala | 302 +++++++++++++----- .../AbstractCoordinatorConcurrencyTest.scala | 21 +- .../CoordinatorPartitionWriterTest.scala | 3 - .../group/GroupCoordinatorTest.scala | 10 +- .../group/GroupMetadataManagerTest.scala | 20 +- .../TransactionStateManagerTest.scala | 6 - .../unit/kafka/server/KafkaApisTest.scala | 33 +- .../kafka/server/ReplicaManagerTest.scala | 82 ++--- 10 files changed, 311 insertions(+), 180 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala index ad1fd580ddf86..06e1dbe9477cf 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala @@ -331,7 +331,7 @@ class GroupMetadataManager(brokerId: Int, verificationGuards: Map[TopicPartition, VerificationGuard] = Map.empty, preAppendErrors: Map[TopicPartition, LogAppendResult] = Map.empty): Unit = { // call replica manager to append the group message - replicaManager.appendRecords( + replicaManager.appendForGroup( timeout = config.offsetCommitTimeoutMs.toLong, requiredAcks = config.offsetCommitRequiredAcks, internalTopicsAllowed = true, diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 6c6a4fab301d1..2b27d4647ac6f 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -708,21 +708,23 @@ class KafkaApis(val requestChannel: RequestChannel, } } - val internalTopicsAllowed = request.header.clientId == AdminUtils.ADMIN_CLIENT_ID - if (authorizedRequestInfo.isEmpty) sendResponseCallback(Map.empty) else { - replicaManager.handleProduceAppend( + val internalTopicsAllowed = request.header.clientId == AdminUtils.ADMIN_CLIENT_ID + + // call the replica manager to append messages to the replicas + replicaManager.appendRecords( timeout = produceRequest.timeout.toLong, requiredAcks = produceRequest.acks, internalTopicsAllowed = internalTopicsAllowed, origin = AppendOrigin.CLIENT, - transactionalId = produceRequest.transactionalId, entriesPerPartition = authorizedRequestInfo, + requestLocal = requestLocal, responseCallback = sendResponseCallback, recordValidationStatsCallback = processingStatsCallback, - requestLocal = requestLocal) + transactionalId = produceRequest.transactionalId() + ) // if the request is put into the purgatory, it will have a held reference and hence cannot be garbage collected; // hence we clear its data here in order to let GC reclaim its memory since it is already appended to log diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index 0b517f721502f..e6e9e4f2a941d 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -763,11 +763,9 @@ class ReplicaManager(val config: KafkaConfig, * @param responseCallback callback for sending the response * @param delayedProduceLock lock for the delayed actions * @param recordValidationStatsCallback callback for updating stats on record conversions - * @param requestLocal container for the stateful instances scoped to this request -- this must correspond to the - * thread calling this method + * @param requestLocal container for the stateful instances scoped to this request + * @param transactionalId transactional ID if the request is from a producer and the producer is transactional * @param actionQueue the action queue to use. ReplicaManager#defaultActionQueue is used by default. - * @param verificationGuards the mapping from topic partition to verification guards if transaction verification is used - * @param preAppendErrors the mapping from topic partition to LogAppendResult for errors that occurred before appending */ def appendRecords(timeout: Long, requiredAcks: Short, @@ -778,24 +776,116 @@ class ReplicaManager(val config: KafkaConfig, delayedProduceLock: Option[Lock] = None, recordValidationStatsCallback: Map[TopicPartition, RecordValidationStats] => Unit = _ => (), requestLocal: RequestLocal = RequestLocal.NoCaching, - actionQueue: ActionQueue = this.defaultActionQueue, - verificationGuards: Map[TopicPartition, VerificationGuard] = Map.empty, - preAppendErrors: Map[TopicPartition, LogAppendResult] = Map.empty): Unit = { - if (!isValidRequiredAcks(requiredAcks)) { - sendInvalidRequiredAcksResponse(entriesPerPartition, responseCallback) - return - } + transactionalId: String = null, + actionQueue: ActionQueue = this.defaultActionQueue): Unit = { + if (isValidRequiredAcks(requiredAcks)) { + + val verificationGuards: mutable.Map[TopicPartition, VerificationGuard] = mutable.Map[TopicPartition, VerificationGuard]() + val (verifiedEntriesPerPartition, notYetVerifiedEntriesPerPartition, errorsPerPartition) = + if (transactionalId == null || !config.transactionPartitionVerificationEnable) + (entriesPerPartition, Map.empty[TopicPartition, MemoryRecords], Map.empty[TopicPartition, Errors]) + else { + val verifiedEntries = mutable.Map[TopicPartition, MemoryRecords]() + val unverifiedEntries = mutable.Map[TopicPartition, MemoryRecords]() + val errorEntries = mutable.Map[TopicPartition, Errors]() + partitionEntriesForVerification(verificationGuards, entriesPerPartition, verifiedEntries, unverifiedEntries, errorEntries) + (verifiedEntries.toMap, unverifiedEntries.toMap, errorEntries.toMap) + } - val nonErrorEntriesPerPartition = entriesPerPartition.filter { - case (tp, _) => !preAppendErrors.contains(tp) + if (notYetVerifiedEntriesPerPartition.isEmpty || addPartitionsToTxnManager.isEmpty) { + appendEntries(verifiedEntriesPerPartition, internalTopicsAllowed, origin, requiredAcks, verificationGuards.toMap, + errorsPerPartition, recordValidationStatsCallback, timeout, responseCallback, delayedProduceLock, actionQueue)(requestLocal, Map.empty) + } else { + // For unverified entries, send a request to verify. When verified, the append process will proceed via the callback. + // We verify above that all partitions use the same producer ID. + val batchInfo = notYetVerifiedEntriesPerPartition.head._2.firstBatch() + addPartitionsToTxnManager.foreach(_.verifyTransaction( + transactionalId = transactionalId, + producerId = batchInfo.producerId, + producerEpoch = batchInfo.producerEpoch, + topicPartitions = notYetVerifiedEntriesPerPartition.keySet.toSeq, + callback = KafkaRequestHandler.wrapAsyncCallback( + appendEntries( + entriesPerPartition, + internalTopicsAllowed, + origin, + requiredAcks, + verificationGuards.toMap, + errorsPerPartition, + recordValidationStatsCallback, + timeout, + responseCallback, + delayedProduceLock, + actionQueue + ), + requestLocal) + )) + } + } else { + // If required.acks is outside accepted range, something is wrong with the client + // Just return an error and don't handle the request at all + val responseStatus = entriesPerPartition.map { case (topicPartition, _) => + topicPartition -> new PartitionResponse( + Errors.INVALID_REQUIRED_ACKS, + LogAppendInfo.UNKNOWN_LOG_APPEND_INFO.firstOffset, + RecordBatch.NO_TIMESTAMP, + LogAppendInfo.UNKNOWN_LOG_APPEND_INFO.logStartOffset + ) + } + responseCallback(responseStatus) } + } + /* + * Note: This method can be used as a callback in a different request thread. Ensure that correct RequestLocal + * is passed when executing this method. Accessing non-thread-safe data structures should be avoided if possible. + */ + private def appendEntries(allEntries: Map[TopicPartition, MemoryRecords], + internalTopicsAllowed: Boolean, + origin: AppendOrigin, + requiredAcks: Short, + verificationGuards: Map[TopicPartition, VerificationGuard], + errorsPerPartition: Map[TopicPartition, Errors], + recordConversionStatsCallback: Map[TopicPartition, RecordValidationStats] => Unit, + timeout: Long, + responseCallback: Map[TopicPartition, PartitionResponse] => Unit, + delayedProduceLock: Option[Lock], + actionQueue: ActionQueue) + (requestLocal: RequestLocal, unverifiedEntries: Map[TopicPartition, Errors]): Unit = { val sTime = time.milliseconds + val verifiedEntries = + if (unverifiedEntries.isEmpty) + allEntries + else + allEntries.filter { case (tp, _) => + !unverifiedEntries.contains(tp) + } + val localProduceResults = appendToLocalLog(internalTopicsAllowed = internalTopicsAllowed, - origin, nonErrorEntriesPerPartition, requiredAcks, requestLocal, verificationGuards.toMap) + origin, verifiedEntries, requiredAcks, requestLocal, verificationGuards.toMap) debug("Produce to local log in %d ms".format(time.milliseconds - sTime)) - val allResults = localProduceResults ++ preAppendErrors + val errorResults = (unverifiedEntries ++ errorsPerPartition).map { + case (topicPartition, error) => + // translate transaction coordinator errors to known producer response errors + val customException = + error match { + case Errors.INVALID_TXN_STATE => Some(error.exception("Partition was not added to the transaction")) + case Errors.CONCURRENT_TRANSACTIONS | + Errors.COORDINATOR_LOAD_IN_PROGRESS | + Errors.COORDINATOR_NOT_AVAILABLE | + Errors.NOT_COORDINATOR => Some(new NotEnoughReplicasException( + s"Unable to verify the partition has been added to the transaction. Underlying error: ${error.toString}")) + case _ => None + } + topicPartition -> LogAppendResult( + LogAppendInfo.UNKNOWN_LOG_APPEND_INFO, + Some(customException.getOrElse(error.exception)), + hasCustomErrorMessage = customException.isDefined + ) + } + + val allResults = localProduceResults ++ errorResults val produceStatus = allResults.map { case (topicPartition, result) => topicPartition -> ProducePartitionStatus( result.info.lastOffset + 1, // required offset @@ -830,15 +920,15 @@ class ReplicaManager(val config: KafkaConfig, } } - recordValidationStatsCallback(localProduceResults.map { case (k, v) => k -> v.info.recordValidationStats }) + recordConversionStatsCallback(localProduceResults.map { case (k, v) => k -> v.info.recordValidationStats }) - if (delayedProduceRequestRequired(requiredAcks, entriesPerPartition, allResults)) { + if (delayedProduceRequestRequired(requiredAcks, allEntries, allResults)) { // create delayed produce operation val produceMetadata = ProduceMetadata(requiredAcks, produceStatus) val delayedProduce = new DelayedProduce(timeout, produceMetadata, this, responseCallback, delayedProduceLock) // create a list of (topic, partition) pairs to use as keys for this delayed produce operation - val producerRequestKeys = entriesPerPartition.keys.map(TopicPartitionOperationKey(_)).toSeq + val producerRequestKeys = allEntries.keys.map(TopicPartitionOperationKey(_)).toSeq // try to complete the request immediately, otherwise put it into the purgatory // this is because while the delayed produce operation is being created, new @@ -851,8 +941,49 @@ class ReplicaManager(val config: KafkaConfig, } } + private def partitionEntriesForVerification(verificationGuards: mutable.Map[TopicPartition, VerificationGuard], + entriesPerPartition: Map[TopicPartition, MemoryRecords], + verifiedEntries: mutable.Map[TopicPartition, MemoryRecords], + unverifiedEntries: mutable.Map[TopicPartition, MemoryRecords], + errorEntries: mutable.Map[TopicPartition, Errors]): Unit = { + val transactionalProducerIds = mutable.HashSet[Long]() + entriesPerPartition.foreach { case (topicPartition, records) => + try { + // Produce requests (only requests that require verification) should only have one batch per partition in "batches" but check all just to be safe. + val transactionalBatches = records.batches.asScala.filter(batch => batch.hasProducerId && batch.isTransactional) + transactionalBatches.foreach(batch => transactionalProducerIds.add(batch.producerId)) + + if (transactionalBatches.nonEmpty) { + // We return VerificationGuard if the partition needs to be verified. If no state is present, no need to verify. + val firstBatch = records.firstBatch + val verificationGuard = getPartitionOrException(topicPartition).maybeStartTransactionVerification(firstBatch.producerId, firstBatch.baseSequence, firstBatch.producerEpoch) + if (verificationGuard != VerificationGuard.SENTINEL) { + verificationGuards.put(topicPartition, verificationGuard) + unverifiedEntries.put(topicPartition, records) + } else + verifiedEntries.put(topicPartition, records) + } else { + // If there is no producer ID or transactional records in the batches, no need to verify. + verifiedEntries.put(topicPartition, records) + } + } catch { + case e: Exception => errorEntries.put(topicPartition, Errors.forException(e)) + } + } + // We should have exactly one producer ID for transactional records + if (transactionalProducerIds.size > 1) { + throw new InvalidPidMappingException("Transactional records contained more than one producer ID") + } + } + /** - * Handles the produce request by starting any transactional verification before appending. + * Append messages to offsets topic, and wait for them to be replicated to other replicas; + * the callback function will be triggered either when timeout or the required acks are satisfied; + * if the callback function itself is already synchronized on some object then pass this object to avoid deadlock. + * + * Noted that all pending delayed check operations are stored in a queue. All callers to ReplicaManager.appendRecords() + * are expected to call ActionQueue.tryCompleteActions for all affected partitions, without holding any conflicting + * locks. * * @param timeout maximum time we will wait to append before returning * @param requiredAcks number of replicas who must acknowledge the append before sending the response @@ -866,73 +997,88 @@ class ReplicaManager(val config: KafkaConfig, * thread calling this method * @param actionQueue the action queue to use. ReplicaManager#defaultActionQueue is used by default. * @param verificationGuards the mapping from topic partition to verification guards if transaction verification is used + * @param preAppendErrors the mapping from topic partition to LogAppendResult for errors that occurred before appending */ - def handleProduceAppend(timeout: Long, - requiredAcks: Short, - internalTopicsAllowed: Boolean, - origin: AppendOrigin, - transactionalId: String, - entriesPerPartition: Map[TopicPartition, MemoryRecords], - responseCallback: Map[TopicPartition, PartitionResponse] => Unit, - recordValidationStatsCallback: Map[TopicPartition, RecordValidationStats] => Unit = _ => (), - requestLocal: RequestLocal = RequestLocal.NoCaching, - actionQueue: ActionQueue = this.defaultActionQueue): Unit = { - - val transactionalProducerInfo = mutable.HashSet[(Long, Short)]() - val topicPartitionBatchInfo = mutable.Map[TopicPartition, Int]() - entriesPerPartition.foreach { case (topicPartition, records) => - // Produce requests (only requests that require verification) should only have one batch per partition in "batches" but check all just to be safe. - val transactionalBatches = records.batches.asScala.filter(batch => batch.hasProducerId && batch.isTransactional) - transactionalBatches.foreach(batch => transactionalProducerInfo.add(batch.producerId, batch.producerEpoch)) - if (!transactionalBatches.isEmpty) topicPartitionBatchInfo.put(topicPartition, records.firstBatch.baseSequence) + def appendForGroup(timeout: Long, + requiredAcks: Short, + internalTopicsAllowed: Boolean, + origin: AppendOrigin, + entriesPerPartition: Map[TopicPartition, MemoryRecords], + responseCallback: Map[TopicPartition, PartitionResponse] => Unit, + delayedProduceLock: Option[Lock] = None, + recordValidationStatsCallback: Map[TopicPartition, RecordValidationStats] => Unit = _ => (), + requestLocal: RequestLocal = RequestLocal.NoCaching, + actionQueue: ActionQueue = this.defaultActionQueue, + verificationGuards: Map[TopicPartition, VerificationGuard] = Map.empty, + preAppendErrors: Map[TopicPartition, LogAppendResult] = Map.empty): Unit = { + if (!isValidRequiredAcks(requiredAcks)) { + sendInvalidRequiredAcksResponse(entriesPerPartition, responseCallback) + return } - if (transactionalProducerInfo.size > 1) { - throw new InvalidPidMappingException("Transactional records contained more than one producer ID") + + val nonErrorEntriesPerPartition = entriesPerPartition.filter { + case (tp, _) => !preAppendErrors.contains(tp) } - def postVerificationCallback(preAppendErrors: Map[TopicPartition, Errors], - newRequestLocal: RequestLocal, - verificationGuards: Map[TopicPartition, VerificationGuard]): Unit = { - val errorResults = preAppendErrors.map { - case (topicPartition, error) => - // translate transaction coordinator errors to known producer response errors - val customException = - error match { - case Errors.INVALID_TXN_STATE => Some(error.exception("Partition was not added to the transaction")) - case Errors.CONCURRENT_TRANSACTIONS | - Errors.COORDINATOR_LOAD_IN_PROGRESS | - Errors.COORDINATOR_NOT_AVAILABLE | - Errors.NOT_COORDINATOR => Some(new NotEnoughReplicasException( - s"Unable to verify the partition has been added to the transaction. Underlying error: ${error.toString}")) - case _ => None - } - topicPartition -> LogAppendResult( - LogAppendInfo.UNKNOWN_LOG_APPEND_INFO, - Some(customException.getOrElse(error.exception)), - hasCustomErrorMessage = customException.isDefined - ) - } + val sTime = time.milliseconds + val localProduceResults = appendToLocalLog(internalTopicsAllowed = internalTopicsAllowed, + origin, nonErrorEntriesPerPartition, requiredAcks, requestLocal, verificationGuards.toMap) + debug("Produce to local log in %d ms".format(time.milliseconds - sTime)) - appendRecords( - timeout = timeout, - requiredAcks = requiredAcks, - internalTopicsAllowed = internalTopicsAllowed, - origin = origin, - entriesPerPartition = entriesPerPartition, - responseCallback = responseCallback, - recordValidationStatsCallback = recordValidationStatsCallback, - requestLocal = newRequestLocal, - verificationGuards = verificationGuards, - preAppendErrors = errorResults - ) + val allResults = localProduceResults ++ preAppendErrors + val produceStatus = allResults.map { case (topicPartition, result) => + topicPartition -> ProducePartitionStatus( + result.info.lastOffset + 1, // required offset + new PartitionResponse( + result.error, + result.info.firstOffset, + result.info.lastOffset, + result.info.logAppendTime, + result.info.logStartOffset, + result.info.recordErrors, + result.errorMessage + ) + ) // response status } - if (transactionalProducerInfo.size < 1) { - postVerificationCallback(Map.empty[TopicPartition, Errors], requestLocal, Map.empty[TopicPartition, VerificationGuard]) - return + actionQueue.add { + () => + allResults.foreach { case (topicPartition, result) => + val requestKey = TopicPartitionOperationKey(topicPartition) + result.info.leaderHwChange match { + case LeaderHwChange.INCREASED => + // some delayed operations may be unblocked after HW changed + delayedProducePurgatory.checkAndComplete(requestKey) + delayedFetchPurgatory.checkAndComplete(requestKey) + delayedDeleteRecordsPurgatory.checkAndComplete(requestKey) + case LeaderHwChange.SAME => + // probably unblock some follower fetch requests since log end offset has been updated + delayedFetchPurgatory.checkAndComplete(requestKey) + case LeaderHwChange.NONE => + // nothing + } + } + } + + recordValidationStatsCallback(localProduceResults.map { case (k, v) => k -> v.info.recordValidationStats }) + + if (delayedProduceRequestRequired(requiredAcks, entriesPerPartition, allResults)) { + // create delayed produce operation + val produceMetadata = ProduceMetadata(requiredAcks, produceStatus) + val delayedProduce = new DelayedProduce(timeout, produceMetadata, this, responseCallback, delayedProduceLock) + + // create a list of (topic, partition) pairs to use as keys for this delayed produce operation + val producerRequestKeys = entriesPerPartition.keys.map(TopicPartitionOperationKey(_)).toSeq + + // try to complete the request immediately, otherwise put it into the purgatory + // this is because while the delayed produce operation is being created, new + // requests may arrive and hence make this operation completable. + delayedProducePurgatory.tryCompleteElseWatch(delayedProduce, producerRequestKeys) + } else { + // we can respond immediately + val produceResponseStatus = produceStatus.map { case (k, status) => k -> status.responseStatus } + responseCallback(produceResponseStatus) } - maybeStartTransactionVerificationForPartitions(topicPartitionBatchInfo, transactionalId, - transactionalProducerInfo.head._1, transactionalProducerInfo.head._2, requestLocal, postVerificationCallback) } private def sendInvalidRequiredAcksResponse(entries: Map[TopicPartition, MemoryRecords], diff --git a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala index be06ccdb2e214..a25e69e3697f4 100644 --- a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala @@ -183,6 +183,22 @@ object AbstractCoordinatorConcurrencyTest { override def tryCompleteActions(): Unit = watchKeys.map(producePurgatory.checkAndComplete) + override def appendForGroup(timeout: Long, + requiredAcks: Short, + internalTopicsAllowed: Boolean, + origin: AppendOrigin, + entriesPerPartition: Map[TopicPartition, MemoryRecords], + responseCallback: Map[TopicPartition, PartitionResponse] => Unit, + delayedProduceLock: Option[Lock] = None, + recordValidationStatsCallback: Map[TopicPartition, RecordValidationStats] => Unit = _ => (), + requestLocal: RequestLocal = RequestLocal.NoCaching, + actionQueue: ActionQueue = null, + verificationGuards: Map[TopicPartition, VerificationGuard] = Map.empty, + preAppendErrors: Map[TopicPartition, LogAppendResult] = Map.empty): Unit = { + appendRecords(timeout, requiredAcks, internalTopicsAllowed, origin, entriesPerPartition, responseCallback, + delayedProduceLock, recordValidationStatsCallback, requestLocal, null, actionQueue) + } + override def appendRecords(timeout: Long, requiredAcks: Short, internalTopicsAllowed: Boolean, @@ -192,9 +208,8 @@ object AbstractCoordinatorConcurrencyTest { delayedProduceLock: Option[Lock] = None, processingStatsCallback: Map[TopicPartition, RecordValidationStats] => Unit = _ => (), requestLocal: RequestLocal = RequestLocal.NoCaching, - actionQueue: ActionQueue = null, - verificationGuards: Map[TopicPartition, VerificationGuard] = Map.empty, - preAppendErrors: Map[TopicPartition, LogAppendResult] = Map.empty): Unit = { + transactionalId: String = null, + actionQueue: ActionQueue = null): Unit = { if (entriesPerPartition.isEmpty) return diff --git a/core/src/test/scala/unit/kafka/coordinator/group/CoordinatorPartitionWriterTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/CoordinatorPartitionWriterTest.scala index 283a938ea1110..121a1f119a1ce 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/CoordinatorPartitionWriterTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/CoordinatorPartitionWriterTest.scala @@ -112,7 +112,6 @@ class CoordinatorPartitionWriterTest { ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), - ArgumentMatchers.any(), ArgumentMatchers.any() )).thenAnswer( _ => { callbackCapture.getValue.apply(Map( @@ -188,7 +187,6 @@ class CoordinatorPartitionWriterTest { ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), - ArgumentMatchers.any(), ArgumentMatchers.any() )).thenAnswer(_ => { callbackCapture.getValue.apply(Map( @@ -269,7 +267,6 @@ class CoordinatorPartitionWriterTest { ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any(), - ArgumentMatchers.any(), ArgumentMatchers.any() )).thenAnswer(_ => { callbackCapture.getValue.apply(Map( diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala index 39a558610192a..d429dc779654b 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala @@ -3893,7 +3893,7 @@ class GroupCoordinatorTest { val capturedArgument: ArgumentCaptor[scala.collection.Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[scala.collection.Map[TopicPartition, PartitionResponse] => Unit]) - when(replicaManager.appendRecords(anyLong, + when(replicaManager.appendForGroup(anyLong, anyShort(), internalTopicsAllowed = ArgumentMatchers.eq(true), origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), @@ -3930,7 +3930,7 @@ class GroupCoordinatorTest { val capturedArgument: ArgumentCaptor[scala.collection.Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[scala.collection.Map[TopicPartition, PartitionResponse] => Unit]) - when(replicaManager.appendRecords(anyLong, + when(replicaManager.appendForGroup(anyLong, anyShort(), internalTopicsAllowed = ArgumentMatchers.eq(true), origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), @@ -4077,7 +4077,7 @@ class GroupCoordinatorTest { val capturedArgument: ArgumentCaptor[scala.collection.Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[scala.collection.Map[TopicPartition, PartitionResponse] => Unit]) - when(replicaManager.appendRecords(anyLong, + when(replicaManager.appendForGroup(anyLong, anyShort(), internalTopicsAllowed = ArgumentMatchers.eq(true), origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), @@ -4114,7 +4114,7 @@ class GroupCoordinatorTest { val capturedArgument: ArgumentCaptor[scala.collection.Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[scala.collection.Map[TopicPartition, PartitionResponse] => Unit]) - // Since transactional ID is only used in appendRecords, we can use a dummy value. Ensure it passes through. + // Since transactional ID is only used for verification, we can use a dummy value. Ensure it passes through. val transactionalId = "dummy-txn-id" val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupCoordinator.partitionFor(groupId)) @@ -4124,7 +4124,7 @@ class GroupCoordinatorTest { ArgumentMatchers.eq(producerId), ArgumentMatchers.eq(producerEpoch), any(), any(), postVerificationCallback.capture())).thenAnswer( _ => postVerificationCallback.getValue()(verificationError, RequestLocal.NoCaching, VerificationGuard.SENTINEL) ) - when(replicaManager.appendRecords(anyLong, + when(replicaManager.appendForGroup(anyLong, anyShort(), internalTopicsAllowed = ArgumentMatchers.eq(true), origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala index 0c30a9c2a2f6f..6ff81060979e3 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala @@ -1175,7 +1175,7 @@ class GroupMetadataManagerTest { groupMetadataManager.storeGroup(group, Map.empty, callback) assertEquals(Some(expectedError), maybeError) - verify(replicaManager).appendRecords(anyLong(), + verify(replicaManager).appendForGroup(anyLong(), anyShort(), internalTopicsAllowed = ArgumentMatchers.eq(true), origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), @@ -1214,7 +1214,7 @@ class GroupMetadataManagerTest { groupMetadataManager.storeGroup(group, Map(memberId -> Array[Byte]()), callback) assertEquals(Some(Errors.NONE), maybeError) - verify(replicaManager).appendRecords(anyLong(), + verify(replicaManager).appendForGroup(anyLong(), anyShort(), internalTopicsAllowed = ArgumentMatchers.eq(true), origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), @@ -1292,7 +1292,7 @@ class GroupMetadataManagerTest { assertEquals(Errors.NONE, partitionResponse.error) assertEquals(offset, partitionResponse.offset) - verify(replicaManager).appendRecords(anyLong(), + verify(replicaManager).appendForGroup(anyLong(), anyShort(), internalTopicsAllowed = ArgumentMatchers.eq(true), origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), @@ -1337,7 +1337,7 @@ class GroupMetadataManagerTest { assertTrue(group.hasOffsets) assertTrue(group.allOffsets.isEmpty) - verify(replicaManager).appendRecords(anyLong(), + verify(replicaManager).appendForGroup(anyLong(), anyShort(), internalTopicsAllowed = ArgumentMatchers.eq(true), origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), @@ -1400,7 +1400,7 @@ class GroupMetadataManagerTest { assertFalse(group.hasOffsets) assertTrue(group.allOffsets.isEmpty) - verify(replicaManager).appendRecords(anyLong(), + verify(replicaManager).appendForGroup(anyLong(), anyShort(), internalTopicsAllowed = ArgumentMatchers.eq(true), origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), @@ -1453,7 +1453,7 @@ class GroupMetadataManagerTest { assertFalse(group.hasOffsets) assertTrue(group.allOffsets.isEmpty) - verify(replicaManager).appendRecords(anyLong(), + verify(replicaManager).appendForGroup(anyLong(), anyShort(), internalTopicsAllowed = ArgumentMatchers.eq(true), origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), @@ -1609,7 +1609,7 @@ class GroupMetadataManagerTest { cachedOffsets.get(topicIdPartitionFailed.topicPartition).map(_.offset) ) - verify(replicaManager).appendRecords(anyLong(), + verify(replicaManager).appendForGroup(anyLong(), anyShort(), internalTopicsAllowed = ArgumentMatchers.eq(true), origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), @@ -1719,7 +1719,7 @@ class GroupMetadataManagerTest { assertEquals(Some(OffsetFetchResponse.INVALID_OFFSET), cachedOffsets.get(topicIdPartition1.topicPartition).map(_.offset)) assertEquals(Some(offset), cachedOffsets.get(topicIdPartition2.topicPartition).map(_.offset)) - verify(replicaManager).appendRecords(anyLong(), + verify(replicaManager).appendForGroup(anyLong(), anyShort(), internalTopicsAllowed = ArgumentMatchers.eq(true), origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), @@ -2831,7 +2831,7 @@ class GroupMetadataManagerTest { private def verifyAppendAndCaptureCallback(): ArgumentCaptor[Map[TopicPartition, PartitionResponse] => Unit] = { val capturedArgument: ArgumentCaptor[Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[Map[TopicPartition, PartitionResponse] => Unit]) - verify(replicaManager).appendRecords(anyLong(), + verify(replicaManager).appendForGroup(anyLong(), anyShort(), internalTopicsAllowed = ArgumentMatchers.eq(true), origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), @@ -2849,7 +2849,7 @@ class GroupMetadataManagerTest { private def expectAppendMessage(error: Errors): ArgumentCaptor[Map[TopicPartition, MemoryRecords]] = { val capturedCallback: ArgumentCaptor[Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[Map[TopicPartition, PartitionResponse] => Unit]) val capturedRecords: ArgumentCaptor[Map[TopicPartition, MemoryRecords]] = ArgumentCaptor.forClass(classOf[Map[TopicPartition, MemoryRecords]]) - when(replicaManager.appendRecords(anyLong(), + when(replicaManager.appendForGroup(anyLong(), anyShort(), internalTopicsAllowed = ArgumentMatchers.eq(true), origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), diff --git a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala index d870900d01893..94ffe7a979557 100644 --- a/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/transaction/TransactionStateManagerTest.scala @@ -658,7 +658,6 @@ class TransactionStateManagerTest { any(), any(), any(), - any(), any() ) @@ -704,7 +703,6 @@ class TransactionStateManagerTest { any(), any(), any(), - any(), any() ) @@ -747,7 +745,6 @@ class TransactionStateManagerTest { any(), any(), any(), - any(), any()) assertEquals(Set.empty, listExpirableTransactionalIds()) @@ -806,7 +803,6 @@ class TransactionStateManagerTest { any(), any(), any(), - any(), any() ) @@ -957,7 +953,6 @@ class TransactionStateManagerTest { any(), any(), any(), - any(), any() )).thenAnswer(_ => callbackCapture.getValue.apply( recordsCapture.getValue.map { case (topicPartition, records) => @@ -1110,7 +1105,6 @@ class TransactionStateManagerTest { any(), any(), any(), - any(), any() )).thenAnswer(_ => capturedArgument.getValue.apply( Map(new TopicPartition(TRANSACTION_STATE_TOPIC_NAME, partitionId) -> diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 9be1296f2354b..a49358f9914ff 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -2446,15 +2446,16 @@ class KafkaApisTest { .build(version.toShort) val request = buildRequest(produceRequest) - when(replicaManager.handleProduceAppend(anyLong, + when(replicaManager.appendRecords(anyLong, anyShort, ArgumentMatchers.eq(false), ArgumentMatchers.eq(AppendOrigin.CLIENT), any(), - any(), responseCallback.capture(), any(), any(), + any(), + any(), any() )).thenAnswer(_ => responseCallback.getValue.apply(Map(tp -> new PartitionResponse(Errors.INVALID_PRODUCER_EPOCH)))) @@ -2504,15 +2505,16 @@ class KafkaApisTest { .build(version.toShort) val request = buildRequest(produceRequest) - when(replicaManager.handleProduceAppend(anyLong, + when(replicaManager.appendRecords(anyLong, anyShort, ArgumentMatchers.eq(false), ArgumentMatchers.eq(AppendOrigin.CLIENT), any(), - any(), responseCallback.capture(), any(), any(), + any(), + any(), any()) ).thenAnswer(_ => responseCallback.getValue.apply(Map(tp -> new PartitionResponse(Errors.NOT_LEADER_OR_FOLLOWER)))) @@ -2569,15 +2571,16 @@ class KafkaApisTest { .build(version.toShort) val request = buildRequest(produceRequest) - when(replicaManager.handleProduceAppend(anyLong, + when(replicaManager.appendRecords(anyLong, anyShort, ArgumentMatchers.eq(false), ArgumentMatchers.eq(AppendOrigin.CLIENT), any(), - any(), responseCallback.capture(), any(), any(), + any(), + any(), any()) ).thenAnswer(_ => responseCallback.getValue.apply(Map(tp -> new PartitionResponse(Errors.NOT_LEADER_OR_FOLLOWER)))) @@ -2633,15 +2636,16 @@ class KafkaApisTest { .build(version.toShort) val request = buildRequest(produceRequest) - when(replicaManager.handleProduceAppend(anyLong, + when(replicaManager.appendRecords(anyLong, anyShort, ArgumentMatchers.eq(false), ArgumentMatchers.eq(AppendOrigin.CLIENT), any(), - any(), responseCallback.capture(), any(), any(), + any(), + any(), any()) ).thenAnswer(_ => responseCallback.getValue.apply(Map(tp -> new PartitionResponse(Errors.NOT_LEADER_OR_FOLLOWER)))) @@ -2681,6 +2685,8 @@ class KafkaApisTest { reset(replicaManager, clientQuotaManager, clientRequestQuotaManager, requestChannel, txnCoordinator) + val responseCallback: ArgumentCaptor[Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[Map[TopicPartition, PartitionResponse] => Unit]) + val tp = new TopicPartition("topic", 0) val produceRequest = ProduceRequest.forCurrentMagic(new ProduceRequestData() @@ -2698,19 +2704,19 @@ class KafkaApisTest { val request = buildRequest(produceRequest) val kafkaApis = createKafkaApis() - val requestLocal = RequestLocal.withThreadConfinedCaching - kafkaApis.handleProduceRequest(request, requestLocal) + kafkaApis.handleProduceRequest(request, RequestLocal.withThreadConfinedCaching) - verify(replicaManager).handleProduceAppend(anyLong, + verify(replicaManager).appendRecords(anyLong, anyShort, ArgumentMatchers.eq(false), ArgumentMatchers.eq(AppendOrigin.CLIENT), - ArgumentMatchers.eq(transactionalId), any(), + responseCallback.capture(), any(), any(), any(), + ArgumentMatchers.eq(transactionalId), any()) } } @@ -2838,7 +2844,6 @@ class KafkaApisTest { any(), ArgumentMatchers.eq(requestLocal), any(), - any(), any() )).thenAnswer(_ => responseCallback.getValue.apply(Map(tp2 -> new PartitionResponse(Errors.NONE)))) @@ -2971,7 +2976,6 @@ class KafkaApisTest { any(), ArgumentMatchers.eq(requestLocal), any(), - any(), any() )).thenAnswer(_ => responseCallback.getValue.apply(Map(tp2 -> new PartitionResponse(Errors.NONE)))) @@ -3006,7 +3010,6 @@ class KafkaApisTest { any(), ArgumentMatchers.eq(requestLocal), any(), - any(), any()) } diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index 8d7a130b0ec07..a4ececf219752 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -657,7 +657,7 @@ class ReplicaManagerTest { val sequence = 9 val records = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, epoch, sequence, new SimpleRecord(time.milliseconds(), s"message $sequence".getBytes)) - appendRecordsWithTransactionVerification(replicaManager, new TopicPartition(topic, 0), records, transactionalId = transactionalId).onFire { response => + appendRecords(replicaManager, new TopicPartition(topic, 0), records, transactionalId = transactionalId).onFire { response => assertEquals(Errors.NONE, response.error) } assertLateTransactionCount(Some(0)) @@ -721,7 +721,7 @@ class ReplicaManagerTest { for (sequence <- 0 until numRecords) { val records = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, epoch, sequence, new SimpleRecord(s"message $sequence".getBytes)) - appendRecordsWithTransactionVerification(replicaManager, new TopicPartition(topic, 0), records, transactionalId = transactionalId).onFire { response => + appendRecords(replicaManager, new TopicPartition(topic, 0), records, transactionalId = transactionalId).onFire { response => assertEquals(Errors.NONE, response.error) } } @@ -842,7 +842,7 @@ class ReplicaManagerTest { for (sequence <- 0 until numRecords) { val records = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, epoch, sequence, new SimpleRecord(s"message $sequence".getBytes)) - appendRecordsWithTransactionVerification(replicaManager, new TopicPartition(topic, 0), records, transactionalId = transactionalId).onFire { response => + appendRecords(replicaManager, new TopicPartition(topic, 0), records, transactionalId = transactionalId).onFire { response => assertEquals(Errors.NONE, response.error) } } @@ -2165,7 +2165,7 @@ class ReplicaManagerTest { // If we supply no transactional ID and idempotent records, we do not verify. val idempotentRecords = MemoryRecords.withIdempotentRecords(CompressionType.NONE, producerId, producerEpoch, sequence, new SimpleRecord("message".getBytes)) - appendRecordsWithTransactionVerification(replicaManager, tp0, idempotentRecords, transactionalId = null) + appendRecords(replicaManager, tp0, idempotentRecords) verify(addPartitionsToTxnManager, times(0)).verifyTransaction(any(), any(), any(), any(), any[AddPartitionsToTxnManager.AppendCallback]()) assertEquals(VerificationGuard.SENTINEL, getVerificationGuard(replicaManager, tp0, producerId)) @@ -2175,7 +2175,7 @@ class ReplicaManagerTest { val idempotentRecords2 = MemoryRecords.withIdempotentRecords(CompressionType.NONE, producerId, producerEpoch, sequence, new SimpleRecord("message".getBytes)) - appendRecordsToMultipleTopicsWithTransactionVerification(replicaManager, Map(tp0 -> transactionalRecords, tp1 -> idempotentRecords2), transactionalId) + appendRecordsToMultipleTopics(replicaManager, Map(tp0 -> transactionalRecords, tp1 -> idempotentRecords2), transactionalId) verify(addPartitionsToTxnManager, times(1)).verifyTransaction( ArgumentMatchers.eq(transactionalId), ArgumentMatchers.eq(producerId), @@ -2210,7 +2210,7 @@ class ReplicaManagerTest { new SimpleRecord("message".getBytes)) // We should add these partitions to the manager to verify. - val result = appendRecordsWithTransactionVerification(replicaManager, tp0, transactionalRecords, origin = appendOrigin, transactionalId = transactionalId) + val result = appendRecords(replicaManager, tp0, transactionalRecords, origin = appendOrigin, transactionalId = transactionalId) val appendCallback = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(1)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2229,7 +2229,7 @@ class ReplicaManagerTest { assertEquals(verificationGuard, getVerificationGuard(replicaManager, tp0, producerId)) // This time verification is successful. - appendRecordsWithTransactionVerification(replicaManager, tp0, transactionalRecords, origin = appendOrigin, transactionalId = transactionalId) + appendRecords(replicaManager, tp0, transactionalRecords, origin = appendOrigin, transactionalId = transactionalId) val appendCallback2 = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(2)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2268,7 +2268,7 @@ class ReplicaManagerTest { new SimpleRecord("message".getBytes)) // We should add these partitions to the manager to verify. - val result = appendRecordsWithTransactionVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) + val result = appendRecords(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) val appendCallback = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(1)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2290,7 +2290,7 @@ class ReplicaManagerTest { val transactionalRecords2 = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, producerEpoch, sequence + 1, new SimpleRecord("message".getBytes)) - val result2 = appendRecordsWithTransactionVerification(replicaManager, tp0, transactionalRecords2, transactionalId = transactionalId) + val result2 = appendRecords(replicaManager, tp0, transactionalRecords2, transactionalId = transactionalId) val appendCallback2 = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(2)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2333,7 +2333,7 @@ class ReplicaManagerTest { val transactionalRecords = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, producerEpoch, sequence, new SimpleRecord(s"message $sequence".getBytes)) - appendRecordsToMultipleTopicsWithTransactionVerification(replicaManager, Map(tp0 -> transactionalRecords, tp1 -> transactionalRecords), transactionalId).onFire { responses => + appendRecordsToMultipleTopics(replicaManager, Map(tp0 -> transactionalRecords, tp1 -> transactionalRecords), transactionalId).onFire { responses => responses.foreach { entry => assertEquals(Errors.NONE, entry._2.error) } @@ -2373,7 +2373,7 @@ class ReplicaManagerTest { new SimpleRecord(s"message $sequence".getBytes))) assertThrows(classOf[InvalidPidMappingException], - () => appendRecordsToMultipleTopicsWithTransactionVerification(replicaManager, transactionalRecords, transactionalId = transactionalId)) + () => appendRecordsToMultipleTopics(replicaManager, transactionalRecords, transactionalId = transactionalId)) // We should not add these partitions to the manager to verify. verify(addPartitionsToTxnManager, times(0)).verifyTransaction(any(), any(), any(), any(), any()) } finally { @@ -2396,7 +2396,7 @@ class ReplicaManagerTest { new SimpleRecord("message".getBytes)) // We should not add these partitions to the manager to verify, but instead throw an error. - appendRecordsWithTransactionVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId).onFire { response => + appendRecords(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId).onFire { response => assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, response.error) } verify(addPartitionsToTxnManager, times(0)).verifyTransaction(any(), any(), any(), any(), any()) @@ -2427,9 +2427,8 @@ class ReplicaManagerTest { val transactionalRecords = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, producerEpoch, sequence, new SimpleRecord(s"message $sequence".getBytes)) - val result = appendRecordsWithTransactionVerification(replicaManager, tp, transactionalRecords, transactionalId = transactionalId) + appendRecords(replicaManager, tp, transactionalRecords, transactionalId = transactionalId) assertEquals(VerificationGuard.SENTINEL, getVerificationGuard(replicaManager, tp, producerId)) - assertEquals(Errors.NONE, result.assertFired.error) // We should not add these partitions to the manager to verify. verify(addPartitionsToTxnManager, times(0)).verifyTransaction(any(), any(), any(), any(), any()) @@ -2445,7 +2444,7 @@ class ReplicaManagerTest { val moreTransactionalRecords = MemoryRecords.withTransactionalRecords(CompressionType.NONE, producerId, producerEpoch, sequence + 1, new SimpleRecord("message".getBytes)) - appendRecordsWithTransactionVerification(replicaManager, tp, moreTransactionalRecords, transactionalId = transactionalId) + appendRecords(replicaManager, tp, moreTransactionalRecords, transactionalId = transactionalId) verify(addPartitionsToTxnManager, times(0)).verifyTransaction(any(), any(), any(), any(), any()) assertEquals(VerificationGuard.SENTINEL, getVerificationGuard(replicaManager, tp, producerId)) assertTrue(replicaManager.localLog(tp).get.hasOngoingTransaction(producerId)) @@ -2473,7 +2472,7 @@ class ReplicaManagerTest { new SimpleRecord("message".getBytes)) // We should add these partitions to the manager to verify. - val result = appendRecordsWithTransactionVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) + val result = appendRecords(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) val appendCallback = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(1)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2499,7 +2498,7 @@ class ReplicaManagerTest { assertEquals(verificationGuard, getVerificationGuard(replicaManager, tp0, producerId)) // This time we do not verify - appendRecordsWithTransactionVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) + appendRecords(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) verify(addPartitionsToTxnManager, times(1)).verifyTransaction(any(), any(), any(), any(), any()) assertEquals(VerificationGuard.SENTINEL, getVerificationGuard(replicaManager, tp0, producerId)) assertTrue(replicaManager.localLog(tp0).get.hasOngoingTransaction(producerId)) @@ -2528,7 +2527,7 @@ class ReplicaManagerTest { // Start verification and return the coordinator related errors. val expectedMessage = s"Unable to verify the partition has been added to the transaction. Underlying error: ${error.toString}" - val result = appendRecordsWithTransactionVerification(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) + val result = appendRecords(replicaManager, tp0, transactionalRecords, transactionalId = transactionalId) val appendCallback = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) verify(addPartitionsToTxnManager, times(1)).verifyTransaction( ArgumentMatchers.eq(transactionalId), @@ -2916,7 +2915,8 @@ class ReplicaManagerTest { partition: TopicPartition, records: MemoryRecords, origin: AppendOrigin = AppendOrigin.CLIENT, - requiredAcks: Short = -1): CallbackResult[PartitionResponse] = { + requiredAcks: Short = -1, + transactionalId: String = null): CallbackResult[PartitionResponse] = { val result = new CallbackResult[PartitionResponse]() def appendCallback(responses: Map[TopicPartition, PartitionResponse]): Unit = { val response = responses.get(partition) @@ -2931,57 +2931,31 @@ class ReplicaManagerTest { origin = origin, entriesPerPartition = Map(partition -> records), responseCallback = appendCallback, + transactionalId = transactionalId, ) result } - private def appendRecordsToMultipleTopicsWithTransactionVerification(replicaManager: ReplicaManager, - entriesToAppend: Map[TopicPartition, MemoryRecords], - transactionalId: String, - origin: AppendOrigin = AppendOrigin.CLIENT, - requiredAcks: Short = -1): CallbackResult[Map[TopicPartition, PartitionResponse]] = { + private def appendRecordsToMultipleTopics(replicaManager: ReplicaManager, + entriesToAppend: Map[TopicPartition, MemoryRecords], + transactionalId: String, + origin: AppendOrigin = AppendOrigin.CLIENT, + requiredAcks: Short = -1): CallbackResult[Map[TopicPartition, PartitionResponse]] = { val result = new CallbackResult[Map[TopicPartition, PartitionResponse]]() def appendCallback(responses: Map[TopicPartition, PartitionResponse]): Unit = { responses.foreach( response => assertTrue(responses.get(response._1).isDefined)) result.fire(responses) } - replicaManager.handleProduceAppend( - timeout = 1000, - requiredAcks = requiredAcks, - internalTopicsAllowed = false, - origin = origin, - transactionalId = transactionalId, - entriesPerPartition = entriesToAppend, - responseCallback = appendCallback, - ) - result - } - - private def appendRecordsWithTransactionVerification(replicaManager: ReplicaManager, - partition: TopicPartition, - records: MemoryRecords, - origin: AppendOrigin = AppendOrigin.CLIENT, - requiredAcks: Short = -1, - transactionalId: String): CallbackResult[PartitionResponse] = { - val result = new CallbackResult[PartitionResponse]() - - def appendCallback(responses: Map[TopicPartition, PartitionResponse]): Unit = { - val response = responses.get(partition) - assertTrue(response.isDefined) - result.fire(response.get) - } - - val entriesPerPartition = Map(partition -> records) - replicaManager.handleProduceAppend( + replicaManager.appendRecords( timeout = 1000, requiredAcks = requiredAcks, internalTopicsAllowed = false, origin = origin, - transactionalId = transactionalId, - entriesPerPartition = entriesPerPartition, + entriesPerPartition = entriesToAppend, responseCallback = appendCallback, + transactionalId = transactionalId, ) result From 99e8577fb59f91b678adf3f586ca3fbfbfd1285a Mon Sep 17 00:00:00 2001 From: Justine Date: Thu, 7 Dec 2023 09:50:14 -0800 Subject: [PATCH 11/24] spacing cleanups --- .../transaction/TransactionStateManager.scala | 6 +- .../scala/kafka/server/ReplicaManager.scala | 58 +++++++++---------- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala index 10445dcb99443..af46bfcc95c63 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionStateManager.scala @@ -281,12 +281,12 @@ class TransactionStateManager(brokerId: Int, inReadLock(stateLock) { replicaManager.appendRecords( - timeout = config.requestTimeoutMs, - requiredAcks = TransactionLog.EnforcedRequiredAcks, + config.requestTimeoutMs, + TransactionLog.EnforcedRequiredAcks, internalTopicsAllowed = true, origin = AppendOrigin.COORDINATOR, entriesPerPartition = Map(transactionPartition -> tombstoneRecords), - responseCallback = removeFromCacheCallback, + removeFromCacheCallback, requestLocal = RequestLocal.NoCaching) } } diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index e6e9e4f2a941d..ed8271c200d3d 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -902,22 +902,21 @@ class ReplicaManager(val config: KafkaConfig, } actionQueue.add { - () => - allResults.foreach { case (topicPartition, result) => - val requestKey = TopicPartitionOperationKey(topicPartition) - result.info.leaderHwChange match { - case LeaderHwChange.INCREASED => - // some delayed operations may be unblocked after HW changed - delayedProducePurgatory.checkAndComplete(requestKey) - delayedFetchPurgatory.checkAndComplete(requestKey) - delayedDeleteRecordsPurgatory.checkAndComplete(requestKey) - case LeaderHwChange.SAME => - // probably unblock some follower fetch requests since log end offset has been updated - delayedFetchPurgatory.checkAndComplete(requestKey) - case LeaderHwChange.NONE => - // nothing - } + () => allResults.foreach { case (topicPartition, result) => + val requestKey = TopicPartitionOperationKey(topicPartition) + result.info.leaderHwChange match { + case LeaderHwChange.INCREASED => + // some delayed operations may be unblocked after HW changed + delayedProducePurgatory.checkAndComplete(requestKey) + delayedFetchPurgatory.checkAndComplete(requestKey) + delayedDeleteRecordsPurgatory.checkAndComplete(requestKey) + case LeaderHwChange.SAME => + // probably unblock some follower fetch requests since log end offset has been updated + delayedFetchPurgatory.checkAndComplete(requestKey) + case LeaderHwChange.NONE => + // nothing } + } } recordConversionStatsCallback(localProduceResults.map { case (k, v) => k -> v.info.recordValidationStats }) @@ -1042,22 +1041,21 @@ class ReplicaManager(val config: KafkaConfig, } actionQueue.add { - () => - allResults.foreach { case (topicPartition, result) => - val requestKey = TopicPartitionOperationKey(topicPartition) - result.info.leaderHwChange match { - case LeaderHwChange.INCREASED => - // some delayed operations may be unblocked after HW changed - delayedProducePurgatory.checkAndComplete(requestKey) - delayedFetchPurgatory.checkAndComplete(requestKey) - delayedDeleteRecordsPurgatory.checkAndComplete(requestKey) - case LeaderHwChange.SAME => - // probably unblock some follower fetch requests since log end offset has been updated - delayedFetchPurgatory.checkAndComplete(requestKey) - case LeaderHwChange.NONE => - // nothing - } + () => allResults.foreach { case (topicPartition, result) => + val requestKey = TopicPartitionOperationKey(topicPartition) + result.info.leaderHwChange match { + case LeaderHwChange.INCREASED => + // some delayed operations may be unblocked after HW changed + delayedProducePurgatory.checkAndComplete(requestKey) + delayedFetchPurgatory.checkAndComplete(requestKey) + delayedDeleteRecordsPurgatory.checkAndComplete(requestKey) + case LeaderHwChange.SAME => + // probably unblock some follower fetch requests since log end offset has been updated + delayedFetchPurgatory.checkAndComplete(requestKey) + case LeaderHwChange.NONE => + // nothing } + } } recordValidationStatsCallback(localProduceResults.map { case (k, v) => k -> v.info.recordValidationStats }) From a83b1362c042c093ba1bc9a880aad3914d834a12 Mon Sep 17 00:00:00 2001 From: Justine Date: Thu, 7 Dec 2023 09:52:04 -0800 Subject: [PATCH 12/24] space --- core/src/main/scala/kafka/server/ReplicaManager.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index ed8271c200d3d..bb8b8eff4a2b0 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -914,7 +914,7 @@ class ReplicaManager(val config: KafkaConfig, // probably unblock some follower fetch requests since log end offset has been updated delayedFetchPurgatory.checkAndComplete(requestKey) case LeaderHwChange.NONE => - // nothing + // nothing } } } @@ -1053,7 +1053,7 @@ class ReplicaManager(val config: KafkaConfig, // probably unblock some follower fetch requests since log end offset has been updated delayedFetchPurgatory.checkAndComplete(requestKey) case LeaderHwChange.NONE => - // nothing + // nothing } } } From 56a44ba72bcf26393af0406afe6587e35c248e2e Mon Sep 17 00:00:00 2001 From: Justine Date: Thu, 7 Dec 2023 09:53:56 -0800 Subject: [PATCH 13/24] remove simple fix --- .../src/main/scala/kafka/server/KafkaRequestHandler.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala index 225c5425bb616..f5974b1e93f6e 100755 --- a/core/src/main/scala/kafka/server/KafkaRequestHandler.scala +++ b/core/src/main/scala/kafka/server/KafkaRequestHandler.scala @@ -67,17 +67,17 @@ object KafkaRequestHandler { if (requestChannel == null || currentRequest == null) { if (!bypassThreadCheck) throw new IllegalStateException("Attempted to reschedule to request handler thread from non-request handler thread.") - t => asyncCompletionCallback(requestLocal, t) + T => asyncCompletionCallback(requestLocal, T) } else { - t => { + T => { if (threadCurrentRequest.get() == currentRequest) { // If the callback is actually executed on the same request thread, we can directly execute // it without re-scheduling it. - asyncCompletionCallback(requestLocal, t) + asyncCompletionCallback(requestLocal, T) } else { // The requestChannel and request are captured in this lambda, so when it's executed on the callback thread // we can re-schedule the original callback on a request thread and update the metrics accordingly. - requestChannel.sendCallbackRequest(RequestChannel.CallbackRequest(newRequestLocal => asyncCompletionCallback(newRequestLocal, t), currentRequest)) + requestChannel.sendCallbackRequest(RequestChannel.CallbackRequest(newRequestLocal => asyncCompletionCallback(newRequestLocal, T), currentRequest)) } } } From e6a14cae826918abe07cdd3018d72024e5603fa1 Mon Sep 17 00:00:00 2001 From: Justine Date: Thu, 7 Dec 2023 15:54:41 -0800 Subject: [PATCH 14/24] cleanups --- .../coordinator/group/GroupMetadataManager.scala | 11 ++++------- core/src/main/scala/kafka/server/ReplicaManager.scala | 5 ++++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala index 06e1dbe9477cf..4a0b45dbe72c5 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala @@ -344,7 +344,7 @@ class GroupMetadataManager(brokerId: Int, preAppendErrors = preAppendErrors) } - def generateOffsetRecords(magicValue: Byte, + private def generateOffsetRecords(magicValue: Byte, isTxnOffsetCommit: Boolean, groupId: String, offsetTopicPartition: TopicPartition, @@ -372,13 +372,13 @@ class GroupMetadataManager(brokerId: Int, Map(offsetTopicPartition -> builder.build()) } - def createPutCacheCallback(isTxnOffsetCommit: Boolean, + private def createPutCacheCallback(isTxnOffsetCommit: Boolean, group: GroupMetadata, consumerId: String, offsetMetadata: immutable.Map[TopicIdPartition, OffsetAndMetadata], filteredOffsetMetadata: Map[TopicIdPartition, OffsetAndMetadata], responseCallback: immutable.Map[TopicIdPartition, Errors] => Unit, - producerId: Long = RecordBatch.NO_PRODUCER_ID, + producerId: Long, records: Map[TopicPartition, MemoryRecords], preAppendErrors: Map[TopicPartition, LogAppendResult] = Map.empty): Map[TopicPartition, PartitionResponse] => Unit = { val offsetTopicPartition = new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, partitionFor(group.groupId)) @@ -502,10 +502,7 @@ class GroupMetadataManager(brokerId: Int, val records = generateOffsetRecords(magicOpt.get, isTxnOffsetCommit, group.groupId, offsetTopicPartition, filteredOffsetMetadata, producerId, producerEpoch) val putCacheCallback = createPutCacheCallback(isTxnOffsetCommit, group, consumerId, offsetMetadata, filteredOffsetMetadata, responseCallback, producerId, records) - val verificationGuards = verificationGuard match { - case Some(guard) => Map(offsetTopicPartition -> guard) - case None => Map.empty[TopicPartition, VerificationGuard] - } + val verificationGuards = verificationGuard.map(guard => offsetTopicPartition -> guard).toMap group.inLock { if (isTxnOffsetCommit) { diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index bb8b8eff4a2b0..f387171852979 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -1106,7 +1106,10 @@ class ReplicaManager(val config: KafkaConfig, def generalizedCallback(preAppendErrors: Map[TopicPartition, Errors], newRequestLocal: RequestLocal, verificationGuards: Map[TopicPartition, VerificationGuard]): Unit = { - callback(preAppendErrors.getOrElse(topicPartition, Errors.NONE), newRequestLocal, verificationGuards.getOrElse(topicPartition, VerificationGuard.SENTINEL)) + callback( + preAppendErrors.getOrElse(topicPartition, Errors.NONE), + newRequestLocal, + verificationGuards.getOrElse(topicPartition, VerificationGuard.SENTINEL)) } maybeStartTransactionVerificationForPartitions( From de5dfadebb7ab99b22af7fea29b269ffef0db2d1 Mon Sep 17 00:00:00 2001 From: Justine Date: Fri, 8 Dec 2023 09:59:58 -0800 Subject: [PATCH 15/24] Update comments simplify locking --- .../group/GroupMetadataManager.scala | 28 ++++++++++--------- .../scala/kafka/server/ReplicaManager.scala | 5 ++++ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala index 4a0b45dbe72c5..840a1996ab4a9 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala @@ -324,6 +324,7 @@ class GroupMetadataManager(brokerId: Int, } } + // This method should be called under the group lock to ensure atomicity of the update to the the in-memory and persisted state. private def appendForGroup(group: GroupMetadata, records: Map[TopicPartition, MemoryRecords], requestLocal: RequestLocal, @@ -439,6 +440,8 @@ class GroupMetadataManager(brokerId: Int, | Errors.INVALID_FETCH_SIZE => Errors.INVALID_COMMIT_OFFSET_SIZE + // We may see INVALID_TXN_STATE or INVALID_PID_MAPPING here due to transaction verification. + // They can be returned without mapping to a new error. case other => other } } @@ -462,6 +465,9 @@ class GroupMetadataManager(brokerId: Int, /** * Store offsets by appending it to the replicated log and then inserting to cache + * + * This method should be called under the group lock to ensure validations and updates are all performed + * atomically. */ def storeOffsets(group: GroupMetadata, consumerId: String, @@ -472,12 +478,10 @@ class GroupMetadataManager(brokerId: Int, producerEpoch: Short = RecordBatch.NO_PRODUCER_EPOCH, requestLocal: RequestLocal = RequestLocal.NoCaching, verificationGuard: Option[VerificationGuard]): Unit = { - group.inLock { - if (!group.hasReceivedConsistentOffsetCommits) - warn(s"group: ${group.groupId} with leader: ${group.leaderOrNull} has received offset commits from consumers as well " + - s"as transactional producers. Mixing both types of offset commits will generally result in surprises and " + - s"should be avoided.") - } + if (!group.hasReceivedConsistentOffsetCommits) + warn(s"group: ${group.groupId} with leader: ${group.leaderOrNull} has received offset commits from consumers as well " + + s"as transactional producers. Mixing both types of offset commits will generally result in surprises and " + + s"should be avoided.") val filteredOffsetMetadata = offsetMetadata.filter { case (_, offsetAndMetadata) => validateOffsetMetadataLength(offsetAndMetadata.metadata) @@ -504,13 +508,11 @@ class GroupMetadataManager(brokerId: Int, val verificationGuards = verificationGuard.map(guard => offsetTopicPartition -> guard).toMap - group.inLock { - if (isTxnOffsetCommit) { - addProducerGroup(producerId, group.groupId) - group.prepareTxnOffsetCommit(producerId, offsetMetadata) - } else { - group.prepareOffsetCommit(offsetMetadata) - } + if (isTxnOffsetCommit) { + addProducerGroup(producerId, group.groupId) + group.prepareTxnOffsetCommit(producerId, offsetMetadata) + } else { + group.prepareOffsetCommit(offsetMetadata) } appendForGroup(group, records, requestLocal, putCacheCallback, verificationGuards) diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index f387171852979..6e3f965c30f8c 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -979,11 +979,16 @@ class ReplicaManager(val config: KafkaConfig, * Append messages to offsets topic, and wait for them to be replicated to other replicas; * the callback function will be triggered either when timeout or the required acks are satisfied; * if the callback function itself is already synchronized on some object then pass this object to avoid deadlock. + * This method should not return until the write to the local log is completed because updating offsets requires updating + * the in-memory and persisted state under a lock together. * * Noted that all pending delayed check operations are stored in a queue. All callers to ReplicaManager.appendRecords() * are expected to call ActionQueue.tryCompleteActions for all affected partitions, without holding any conflicting * locks. * + * If appending transactional records, call maybeStartTransactionVerificationForPartition(s) and call this method in the callback. + * For example, the GroupCoordinator will pass `doCommitTxnOffsets` as the post-verification callback, and that method eventually call this. + * * @param timeout maximum time we will wait to append before returning * @param requiredAcks number of replicas who must acknowledge the append before sending the response * @param internalTopicsAllowed boolean indicating whether internal topics can be appended to From 66fff8276b2fbe2257749e70a8a1ca4657f61ef7 Mon Sep 17 00:00:00 2001 From: Justine Date: Fri, 8 Dec 2023 16:36:26 -0800 Subject: [PATCH 16/24] applying cherrypick --- .../group/GroupMetadataManager.scala | 19 +- .../scala/kafka/server/ReplicaManager.scala | 208 +++++++++--------- .../AbstractCoordinatorConcurrencyTest.scala | 11 +- .../group/GroupCoordinatorTest.scala | 22 +- .../group/GroupMetadataManagerTest.scala | 56 +---- 5 files changed, 116 insertions(+), 200 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala index 840a1996ab4a9..c120b74e01ab0 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala @@ -325,24 +325,23 @@ class GroupMetadataManager(brokerId: Int, } // This method should be called under the group lock to ensure atomicity of the update to the the in-memory and persisted state. - private def appendForGroup(group: GroupMetadata, - records: Map[TopicPartition, MemoryRecords], - requestLocal: RequestLocal, - callback: Map[TopicPartition, PartitionResponse] => Unit, - verificationGuards: Map[TopicPartition, VerificationGuard] = Map.empty, - preAppendErrors: Map[TopicPartition, LogAppendResult] = Map.empty): Unit = { + private def appendForGroup( + group: GroupMetadata, + records: Map[TopicPartition, MemoryRecords], + requestLocal: RequestLocal, + callback: Map[TopicPartition, PartitionResponse] => Unit, + verificationGuards: Map[TopicPartition, VerificationGuard] = Map.empty + ): Unit = { // call replica manager to append the group message replicaManager.appendForGroup( timeout = config.offsetCommitTimeoutMs.toLong, requiredAcks = config.offsetCommitRequiredAcks, - internalTopicsAllowed = true, - origin = AppendOrigin.COORDINATOR, entriesPerPartition = records, delayedProduceLock = Some(group.lock), responseCallback = callback, requestLocal = requestLocal, - verificationGuards = verificationGuards, - preAppendErrors = preAppendErrors) + verificationGuards = verificationGuards + ) } private def generateOffsetRecords(magicValue: Byte, diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index 6e3f965c30f8c..b97e328617d53 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -886,58 +886,22 @@ class ReplicaManager(val config: KafkaConfig, } val allResults = localProduceResults ++ errorResults - val produceStatus = allResults.map { case (topicPartition, result) => - topicPartition -> ProducePartitionStatus( - result.info.lastOffset + 1, // required offset - new PartitionResponse( - result.error, - result.info.firstOffset, - result.info.lastOffset, - result.info.logAppendTime, - result.info.logStartOffset, - result.info.recordErrors, - result.errorMessage - ) - ) // response status - } - - actionQueue.add { - () => allResults.foreach { case (topicPartition, result) => - val requestKey = TopicPartitionOperationKey(topicPartition) - result.info.leaderHwChange match { - case LeaderHwChange.INCREASED => - // some delayed operations may be unblocked after HW changed - delayedProducePurgatory.checkAndComplete(requestKey) - delayedFetchPurgatory.checkAndComplete(requestKey) - delayedDeleteRecordsPurgatory.checkAndComplete(requestKey) - case LeaderHwChange.SAME => - // probably unblock some follower fetch requests since log end offset has been updated - delayedFetchPurgatory.checkAndComplete(requestKey) - case LeaderHwChange.NONE => - // nothing - } - } - } - - recordConversionStatsCallback(localProduceResults.map { case (k, v) => k -> v.info.recordValidationStats }) + val produceStatus = buildProducePartitionStatus(allResults) - if (delayedProduceRequestRequired(requiredAcks, allEntries, allResults)) { - // create delayed produce operation - val produceMetadata = ProduceMetadata(requiredAcks, produceStatus) - val delayedProduce = new DelayedProduce(timeout, produceMetadata, this, responseCallback, delayedProduceLock) - - // create a list of (topic, partition) pairs to use as keys for this delayed produce operation - val producerRequestKeys = allEntries.keys.map(TopicPartitionOperationKey(_)).toSeq + addCompletePurgatoryAction(actionQueue, allResults) + recordConversionStatsCallback(localProduceResults.map { case (k, v) => + k -> v.info.recordValidationStats + }) - // try to complete the request immediately, otherwise put it into the purgatory - // this is because while the delayed produce operation is being created, new - // requests may arrive and hence make this operation completable. - delayedProducePurgatory.tryCompleteElseWatch(delayedProduce, producerRequestKeys) - } else { - // we can respond immediately - val produceResponseStatus = produceStatus.map { case (k, status) => k -> status.responseStatus } - responseCallback(produceResponseStatus) - } + maybeAddDelayedProduce( + requiredAcks, + delayedProduceLock, + timeout, + allEntries, + allResults, + produceStatus, + responseCallback + ) } private def partitionEntriesForVerification(verificationGuards: mutable.Map[TopicPartition, VerificationGuard], @@ -975,6 +939,48 @@ class ReplicaManager(val config: KafkaConfig, } } + private def buildProducePartitionStatus( + results: Map[TopicPartition, LogAppendResult] + ): Map[TopicPartition, ProducePartitionStatus] = { + results.map { case (topicPartition, result) => + topicPartition -> ProducePartitionStatus( + result.info.lastOffset + 1, // required offset + new PartitionResponse( + result.error, + result.info.firstOffset, + result.info.lastOffset, + result.info.logAppendTime, + result.info.logStartOffset, + result.info.recordErrors, + result.errorMessage + ) + ) + } + } + + private def addCompletePurgatoryAction( + actionQueue: ActionQueue, + appendResults: Map[TopicPartition, LogAppendResult] + ): Unit = { + actionQueue.add { + () => appendResults.foreach { case (topicPartition, result) => + val requestKey = TopicPartitionOperationKey(topicPartition) + result.info.leaderHwChange match { + case LeaderHwChange.INCREASED => + // some delayed operations may be unblocked after HW changed + delayedProducePurgatory.checkAndComplete(requestKey) + delayedFetchPurgatory.checkAndComplete(requestKey) + delayedDeleteRecordsPurgatory.checkAndComplete(requestKey) + case LeaderHwChange.SAME => + // probably unblock some follower fetch requests since log end offset has been updated + delayedFetchPurgatory.checkAndComplete(requestKey) + case LeaderHwChange.NONE => + // nothing + } + } + } + } + /** * Append messages to offsets topic, and wait for them to be replicated to other replicas; * the callback function will be triggered either when timeout or the required acks are satisfied; @@ -996,79 +1002,65 @@ class ReplicaManager(val config: KafkaConfig, * @param entriesPerPartition the records per partition to be appended * @param responseCallback callback for sending the response * @param delayedProduceLock lock for the delayed actions - * @param recordValidationStatsCallback callback for updating stats on record conversions * @param requestLocal container for the stateful instances scoped to this request -- this must correspond to the * thread calling this method - * @param actionQueue the action queue to use. ReplicaManager#defaultActionQueue is used by default. * @param verificationGuards the mapping from topic partition to verification guards if transaction verification is used - * @param preAppendErrors the mapping from topic partition to LogAppendResult for errors that occurred before appending */ - def appendForGroup(timeout: Long, - requiredAcks: Short, - internalTopicsAllowed: Boolean, - origin: AppendOrigin, - entriesPerPartition: Map[TopicPartition, MemoryRecords], - responseCallback: Map[TopicPartition, PartitionResponse] => Unit, - delayedProduceLock: Option[Lock] = None, - recordValidationStatsCallback: Map[TopicPartition, RecordValidationStats] => Unit = _ => (), - requestLocal: RequestLocal = RequestLocal.NoCaching, - actionQueue: ActionQueue = this.defaultActionQueue, - verificationGuards: Map[TopicPartition, VerificationGuard] = Map.empty, - preAppendErrors: Map[TopicPartition, LogAppendResult] = Map.empty): Unit = { + def appendForGroup( + timeout: Long, + requiredAcks: Short, + entriesPerPartition: Map[TopicPartition, MemoryRecords], + responseCallback: Map[TopicPartition, PartitionResponse] => Unit, + delayedProduceLock: Option[Lock], + requestLocal: RequestLocal, + verificationGuards: Map[TopicPartition, VerificationGuard] + ): Unit = { if (!isValidRequiredAcks(requiredAcks)) { sendInvalidRequiredAcksResponse(entriesPerPartition, responseCallback) return } - val nonErrorEntriesPerPartition = entriesPerPartition.filter { - case (tp, _) => !preAppendErrors.contains(tp) - } - val sTime = time.milliseconds - val localProduceResults = appendToLocalLog(internalTopicsAllowed = internalTopicsAllowed, - origin, nonErrorEntriesPerPartition, requiredAcks, requestLocal, verificationGuards.toMap) + val localProduceResults = appendToLocalLog( + internalTopicsAllowed = true, + origin = AppendOrigin.COORDINATOR, + entriesPerPartition, + requiredAcks, + requestLocal, + verificationGuards.toMap + ) + debug("Produce to local log in %d ms".format(time.milliseconds - sTime)) - val allResults = localProduceResults ++ preAppendErrors - val produceStatus = allResults.map { case (topicPartition, result) => - topicPartition -> ProducePartitionStatus( - result.info.lastOffset + 1, // required offset - new PartitionResponse( - result.error, - result.info.firstOffset, - result.info.lastOffset, - result.info.logAppendTime, - result.info.logStartOffset, - result.info.recordErrors, - result.errorMessage - ) - ) // response status - } + val allResults = localProduceResults + val produceStatus = buildProducePartitionStatus(allResults) - actionQueue.add { - () => allResults.foreach { case (topicPartition, result) => - val requestKey = TopicPartitionOperationKey(topicPartition) - result.info.leaderHwChange match { - case LeaderHwChange.INCREASED => - // some delayed operations may be unblocked after HW changed - delayedProducePurgatory.checkAndComplete(requestKey) - delayedFetchPurgatory.checkAndComplete(requestKey) - delayedDeleteRecordsPurgatory.checkAndComplete(requestKey) - case LeaderHwChange.SAME => - // probably unblock some follower fetch requests since log end offset has been updated - delayedFetchPurgatory.checkAndComplete(requestKey) - case LeaderHwChange.NONE => - // nothing - } - } - } + addCompletePurgatoryAction(defaultActionQueue, allResults) - recordValidationStatsCallback(localProduceResults.map { case (k, v) => k -> v.info.recordValidationStats }) + maybeAddDelayedProduce( + requiredAcks, + delayedProduceLock, + timeout, + entriesPerPartition, + allResults, + produceStatus, + responseCallback + ) + } - if (delayedProduceRequestRequired(requiredAcks, entriesPerPartition, allResults)) { + private def maybeAddDelayedProduce( + requiredAcks: Short, + delayedProduceLock: Option[Lock], + timeoutMs: Long, + entriesPerPartition: Map[TopicPartition, MemoryRecords], + initialAppendResults: Map[TopicPartition, LogAppendResult], + initialProduceStatus: Map[TopicPartition, ProducePartitionStatus], + responseCallback: Map[TopicPartition, PartitionResponse] => Unit, + ): Unit = { + if (delayedProduceRequestRequired(requiredAcks, entriesPerPartition, initialAppendResults)) { // create delayed produce operation - val produceMetadata = ProduceMetadata(requiredAcks, produceStatus) - val delayedProduce = new DelayedProduce(timeout, produceMetadata, this, responseCallback, delayedProduceLock) + val produceMetadata = ProduceMetadata(requiredAcks, initialProduceStatus) + val delayedProduce = new DelayedProduce(timeoutMs, produceMetadata, this, responseCallback, delayedProduceLock) // create a list of (topic, partition) pairs to use as keys for this delayed produce operation val producerRequestKeys = entriesPerPartition.keys.map(TopicPartitionOperationKey(_)).toSeq @@ -1079,7 +1071,7 @@ class ReplicaManager(val config: KafkaConfig, delayedProducePurgatory.tryCompleteElseWatch(delayedProduce, producerRequestKeys) } else { // we can respond immediately - val produceResponseStatus = produceStatus.map { case (k, status) => k -> status.responseStatus } + val produceResponseStatus = initialProduceStatus.map { case (k, status) => k -> status.responseStatus } responseCallback(produceResponseStatus) } } diff --git a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala index a25e69e3697f4..bfa36abfb4de2 100644 --- a/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/AbstractCoordinatorConcurrencyTest.scala @@ -185,18 +185,13 @@ object AbstractCoordinatorConcurrencyTest { override def appendForGroup(timeout: Long, requiredAcks: Short, - internalTopicsAllowed: Boolean, - origin: AppendOrigin, entriesPerPartition: Map[TopicPartition, MemoryRecords], responseCallback: Map[TopicPartition, PartitionResponse] => Unit, delayedProduceLock: Option[Lock] = None, - recordValidationStatsCallback: Map[TopicPartition, RecordValidationStats] => Unit = _ => (), requestLocal: RequestLocal = RequestLocal.NoCaching, - actionQueue: ActionQueue = null, - verificationGuards: Map[TopicPartition, VerificationGuard] = Map.empty, - preAppendErrors: Map[TopicPartition, LogAppendResult] = Map.empty): Unit = { - appendRecords(timeout, requiredAcks, internalTopicsAllowed, origin, entriesPerPartition, responseCallback, - delayedProduceLock, recordValidationStatsCallback, requestLocal, null, actionQueue) + verificationGuards: Map[TopicPartition, VerificationGuard] = Map.empty): Unit = { + appendRecords(timeout, requiredAcks, true, AppendOrigin.COORDINATOR, entriesPerPartition, responseCallback, + delayedProduceLock, requestLocal = requestLocal) } override def appendRecords(timeout: Long, diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala index d429dc779654b..d55519c2f955f 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala @@ -38,7 +38,7 @@ import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity import org.apache.kafka.server.util.timer.MockTimer import org.apache.kafka.server.util.{KafkaScheduler, MockTime} -import org.apache.kafka.storage.internals.log.{AppendOrigin, VerificationGuard} +import org.apache.kafka.storage.internals.log.VerificationGuard import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} import org.junit.jupiter.params.ParameterizedTest @@ -3895,16 +3895,11 @@ class GroupCoordinatorTest { when(replicaManager.appendForGroup(anyLong, anyShort(), - internalTopicsAllowed = ArgumentMatchers.eq(true), - origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), any[Map[TopicPartition, MemoryRecords]], capturedArgument.capture(), any[Option[ReentrantLock]], any(), any(), - any(), - any(), - any() )).thenAnswer(_ => { capturedArgument.getValue.apply( Map(new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId) -> @@ -3932,15 +3927,10 @@ class GroupCoordinatorTest { when(replicaManager.appendForGroup(anyLong, anyShort(), - internalTopicsAllowed = ArgumentMatchers.eq(true), - origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), any[Map[TopicPartition, MemoryRecords]], capturedArgument.capture(), any[Option[ReentrantLock]], any(), - any(), - any(), - any(), any())).thenAnswer(_ => { capturedArgument.getValue.apply( Map(new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId) -> @@ -4079,15 +4069,10 @@ class GroupCoordinatorTest { when(replicaManager.appendForGroup(anyLong, anyShort(), - internalTopicsAllowed = ArgumentMatchers.eq(true), - origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), any[Map[TopicPartition, MemoryRecords]], capturedArgument.capture(), any[Option[ReentrantLock]], any(), - any(), - any(), - any(), any() )).thenAnswer(_ => { capturedArgument.getValue.apply( @@ -4126,15 +4111,10 @@ class GroupCoordinatorTest { ) when(replicaManager.appendForGroup(anyLong, anyShort(), - internalTopicsAllowed = ArgumentMatchers.eq(true), - origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), any[Map[TopicPartition, MemoryRecords]], capturedArgument.capture(), any[Option[ReentrantLock]], any(), - any(), - any(), - any(), any() )).thenAnswer(_ => { capturedArgument.getValue.apply( diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala index 6ff81060979e3..d43a12255989b 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupMetadataManagerTest.scala @@ -1177,15 +1177,10 @@ class GroupMetadataManagerTest { verify(replicaManager).appendForGroup(anyLong(), anyShort(), - internalTopicsAllowed = ArgumentMatchers.eq(true), - origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), any(), any(), any[Option[ReentrantLock]], any(), - any(), - any(), - any(), any()) verify(replicaManager).getMagic(any()) } @@ -1216,15 +1211,10 @@ class GroupMetadataManagerTest { verify(replicaManager).appendForGroup(anyLong(), anyShort(), - internalTopicsAllowed = ArgumentMatchers.eq(true), - origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), any(), any(), any[Option[ReentrantLock]], any(), - any(), - any(), - any(), any()) verify(replicaManager).getMagic(any()) } @@ -1294,15 +1284,10 @@ class GroupMetadataManagerTest { verify(replicaManager).appendForGroup(anyLong(), anyShort(), - internalTopicsAllowed = ArgumentMatchers.eq(true), - origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), any(), any(), any[Option[ReentrantLock]], any(), - any(), - any(), - any(), any()) // Will update sensor after commit assertEquals(1, TestUtils.totalMetricValue(metrics, "offset-commit-count")) @@ -1339,16 +1324,11 @@ class GroupMetadataManagerTest { verify(replicaManager).appendForGroup(anyLong(), anyShort(), - internalTopicsAllowed = ArgumentMatchers.eq(true), - origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), any[Map[TopicPartition, MemoryRecords]], capturedResponseCallback.capture(), any[Option[ReentrantLock]], any(), - any(), - any(), - ArgumentMatchers.eq(Map(offsetTopicPartition -> verificationGuard)), - any()) + ArgumentMatchers.eq(Map(offsetTopicPartition -> verificationGuard))) verify(replicaManager).getMagic(any()) capturedResponseCallback.getValue.apply(Map(groupTopicPartition -> new PartitionResponse(Errors.NONE, 0L, RecordBatch.NO_TIMESTAMP, 0L))) @@ -1402,16 +1382,11 @@ class GroupMetadataManagerTest { verify(replicaManager).appendForGroup(anyLong(), anyShort(), - internalTopicsAllowed = ArgumentMatchers.eq(true), - origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), any[Map[TopicPartition, MemoryRecords]], any(), any[Option[ReentrantLock]], any(), - any(), - any(), - ArgumentMatchers.eq(Map(offsetTopicPartition -> verificationGuard)), - any()) + ArgumentMatchers.eq(Map(offsetTopicPartition -> verificationGuard))) verify(replicaManager).getMagic(any()) } @@ -1455,16 +1430,11 @@ class GroupMetadataManagerTest { verify(replicaManager).appendForGroup(anyLong(), anyShort(), - internalTopicsAllowed = ArgumentMatchers.eq(true), - origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), any[Map[TopicPartition, MemoryRecords]], any(), any[Option[ReentrantLock]], any(), - any(), - any(), - ArgumentMatchers.eq(Map(offsetTopicPartition -> verificationGuard)), - any()) + ArgumentMatchers.eq(Map(offsetTopicPartition -> verificationGuard))) verify(replicaManager).getMagic(any()) } @@ -1611,15 +1581,10 @@ class GroupMetadataManagerTest { verify(replicaManager).appendForGroup(anyLong(), anyShort(), - internalTopicsAllowed = ArgumentMatchers.eq(true), - origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), any[Map[TopicPartition, MemoryRecords]], any(), any[Option[ReentrantLock]], any(), - any(), - any(), - any(), any()) verify(replicaManager).getMagic(any()) assertEquals(1, TestUtils.totalMetricValue(metrics, "offset-commit-count")) @@ -1721,15 +1686,10 @@ class GroupMetadataManagerTest { verify(replicaManager).appendForGroup(anyLong(), anyShort(), - internalTopicsAllowed = ArgumentMatchers.eq(true), - origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), any(), any(), any[Option[ReentrantLock]], any(), - any(), - any(), - any(), any()) verify(replicaManager, times(2)).getMagic(any()) } @@ -2833,15 +2793,10 @@ class GroupMetadataManagerTest { val capturedArgument: ArgumentCaptor[Map[TopicPartition, PartitionResponse] => Unit] = ArgumentCaptor.forClass(classOf[Map[TopicPartition, PartitionResponse] => Unit]) verify(replicaManager).appendForGroup(anyLong(), anyShort(), - internalTopicsAllowed = ArgumentMatchers.eq(true), - origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), any[Map[TopicPartition, MemoryRecords]], capturedArgument.capture(), any[Option[ReentrantLock]], any(), - any(), - any(), - any(), any()) capturedArgument } @@ -2851,15 +2806,10 @@ class GroupMetadataManagerTest { val capturedRecords: ArgumentCaptor[Map[TopicPartition, MemoryRecords]] = ArgumentCaptor.forClass(classOf[Map[TopicPartition, MemoryRecords]]) when(replicaManager.appendForGroup(anyLong(), anyShort(), - internalTopicsAllowed = ArgumentMatchers.eq(true), - origin = ArgumentMatchers.eq(AppendOrigin.COORDINATOR), capturedRecords.capture(), capturedCallback.capture(), any[Option[ReentrantLock]], any(), - any(), - any(), - any(), any() )).thenAnswer(_ => { capturedCallback.getValue.apply( From 046425ce887ec3e792d424c0552807b7b34d307b Mon Sep 17 00:00:00 2001 From: Justine Date: Fri, 8 Dec 2023 17:12:41 -0800 Subject: [PATCH 17/24] Fix tests --- .../group/GroupCoordinatorTest.scala | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala index d55519c2f955f..2f78f28ab9ff5 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala @@ -3898,8 +3898,8 @@ class GroupCoordinatorTest { any[Map[TopicPartition, MemoryRecords]], capturedArgument.capture(), any[Option[ReentrantLock]], - any(), - any(), + any(classOf[RequestLocal]), + any[Map[TopicPartition, VerificationGuard]] )).thenAnswer(_ => { capturedArgument.getValue.apply( Map(new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId) -> @@ -3930,8 +3930,9 @@ class GroupCoordinatorTest { any[Map[TopicPartition, MemoryRecords]], capturedArgument.capture(), any[Option[ReentrantLock]], - any(), - any())).thenAnswer(_ => { + any(classOf[RequestLocal]), + any[Map[TopicPartition, VerificationGuard]] + )).thenAnswer(_ => { capturedArgument.getValue.apply( Map(new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId) -> new PartitionResponse(Errors.NONE, 0L, RecordBatch.NO_TIMESTAMP, 0L) @@ -4072,8 +4073,8 @@ class GroupCoordinatorTest { any[Map[TopicPartition, MemoryRecords]], capturedArgument.capture(), any[Option[ReentrantLock]], - any(), - any() + any(classOf[RequestLocal]), + any[Map[TopicPartition, VerificationGuard]] )).thenAnswer(_ => { capturedArgument.getValue.apply( Map(new TopicPartition(Topic.GROUP_METADATA_TOPIC_NAME, groupPartitionId) -> @@ -4114,8 +4115,8 @@ class GroupCoordinatorTest { any[Map[TopicPartition, MemoryRecords]], capturedArgument.capture(), any[Option[ReentrantLock]], - any(), - any() + any(classOf[RequestLocal]), + any[Map[TopicPartition, VerificationGuard]] )).thenAnswer(_ => { capturedArgument.getValue.apply( Map(offsetTopicPartition -> From 2c1f6a97e99682431737b9d125e04b0b8cb3977e Mon Sep 17 00:00:00 2001 From: Justine Date: Fri, 8 Dec 2023 17:22:29 -0800 Subject: [PATCH 18/24] add error conversion --- .../main/scala/kafka/server/ReplicaManager.scala | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index b97e328617d53..5346436c97046 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -1165,7 +1165,18 @@ class ReplicaManager(val config: KafkaConfig, requestLocal: RequestLocal, verificationErrors: Map[TopicPartition, Errors] ): Unit = { - callback(errors ++ verificationErrors, requestLocal, verificationGuards.toMap) + // Map transaction coordinator errors to known errors for the response + val convertedErrors = verificationErrors.map { case (tp, error) => + error match { + case Errors.CONCURRENT_TRANSACTIONS | + Errors.COORDINATOR_LOAD_IN_PROGRESS | + Errors.COORDINATOR_NOT_AVAILABLE | + Errors.NOT_COORDINATOR => tp -> Errors.NOT_ENOUGH_REPLICAS + case _ => tp -> error + } + + } + callback(errors ++ convertedErrors, requestLocal, verificationGuards.toMap) } // Wrap the callback to be handled on an arbitrary request handler thread when transaction verification is complete. From 3f6c044e8bf2b85ff2c0d8a31449e9e8506ce4f6 Mon Sep 17 00:00:00 2001 From: Justine Date: Fri, 8 Dec 2023 20:21:30 -0800 Subject: [PATCH 19/24] add test --- .../kafka/server/ReplicaManagerTest.scala | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index a4ececf219752..5c38a9f763cf6 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -2547,6 +2547,41 @@ class ReplicaManagerTest { } } + @ParameterizedTest + @EnumSource(value = classOf[Errors], names = Array("NOT_COORDINATOR", "CONCURRENT_TRANSACTIONS", "COORDINATOR_LOAD_IN_PROGRESS", "COORDINATOR_NOT_AVAILABLE")) + def testMaybeVerificationErrorConversions(error: Errors): Unit = { + val tp0 = new TopicPartition(topic, 0) + val transactionalId = "txn-id" + val producerId = 24L + val producerEpoch = 0.toShort + val addPartitionsToTxnManager = mock(classOf[AddPartitionsToTxnManager]) + + val replicaManager = setUpReplicaManagerWithMockedAddPartitionsToTxnManager(addPartitionsToTxnManager, List(tp0)) + try { + replicaManager.becomeLeaderOrFollower(1, + makeLeaderAndIsrRequest(topicIds(tp0.topic), tp0, Seq(0, 1), LeaderAndIsr(1, List(0, 1))), + (_, _) => ()) + + // Start verification and return the coordinator related errors. + val result = maybeStartTransactionVerificationForPartition(replicaManager, tp0, transactionalId, producerId, producerEpoch) + val appendCallback = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) + verify(addPartitionsToTxnManager, times(1)).verifyTransaction( + ArgumentMatchers.eq(transactionalId), + ArgumentMatchers.eq(producerId), + ArgumentMatchers.eq(producerEpoch), + ArgumentMatchers.eq(Seq(tp0)), + appendCallback.capture() + ) + + // Confirm we did not write to the log and instead returned the converted error with the correct error message. + val callback: AddPartitionsToTxnManager.AppendCallback = appendCallback.getValue() + callback(Map(tp0 -> error).toMap) + assertEquals(Errors.NOT_ENOUGH_REPLICAS, result.assertFired.left.getOrElse(Errors.NONE)) + } finally { + replicaManager.shutdown(checkpointHW = false) + } + } + @ParameterizedTest @ValueSource(booleans = Array(true, false)) def testFullLeaderAndIsrStrayPartitions(zkMigrationEnabled: Boolean): Unit = { @@ -2961,6 +2996,32 @@ class ReplicaManagerTest { result } + private def maybeStartTransactionVerificationForPartition(replicaManager: ReplicaManager, + topicPartition: TopicPartition, + transactionalId: String, + producerId: Long, + producerEpoch: Short, + baseSequence: Int = 0): CallbackResult[Either[Errors, VerificationGuard]] = { + val result = new CallbackResult[Either[Errors, VerificationGuard]]() + def postVerificationCallback(error: Errors, + requestLocal: RequestLocal, + verificationGuard: VerificationGuard): Unit = { + val errorOrGuard = if (error != Errors.NONE) Left(error) else Right(verificationGuard) + result.fire(errorOrGuard) + } + + replicaManager.maybeStartTransactionVerificationForPartition( + topicPartition, + transactionalId, + producerId, + producerEpoch, + baseSequence, + RequestLocal.NoCaching, + postVerificationCallback + ) + result + } + private def fetchPartitionAsConsumer( replicaManager: ReplicaManager, partition: TopicIdPartition, From 3a3160ddb9ecaee77e7ec9ca86e423f41bb86420 Mon Sep 17 00:00:00 2001 From: Justine Date: Mon, 11 Dec 2023 13:35:06 -0800 Subject: [PATCH 20/24] Fix error code --- .../kafka/coordinator/group/GroupCoordinator.scala | 6 +++++- .../coordinator/group/GroupCoordinatorTest.scala | 11 ++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala index 3ba981fd2b372..b84ab47e0acd3 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala @@ -920,7 +920,11 @@ private[group] class GroupCoordinator( verificationGuard: VerificationGuard ): Unit = { if (error != Errors.NONE) { - responseCallback(offsetMetadata.map { case (k, _) => k -> error }) + val finalError = error match { + case Errors.NOT_ENOUGH_REPLICAS => Errors.COORDINATOR_NOT_AVAILABLE + case e => e + } + responseCallback(offsetMetadata.map { case (k, _) => k -> finalError }) } else { doTxnCommitOffsets(group, memberId, groupInstanceId, generationId, producerId, producerEpoch, offsetTopicPartition, offsetMetadata, newRequestLocal, responseCallback, Some(verificationGuard)) diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala index 2f78f28ab9ff5..6999516d723fe 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala @@ -3791,18 +3791,19 @@ class GroupCoordinatorTest { val producerId = 1000L val producerEpoch: Short = 2 - def verifyErrors(error: Errors): Unit = { + def verifyErrors(error: Errors, expectedError: Errors): Unit = { val commitOffsetResult = commitTransactionalOffsets(groupId, producerId, producerEpoch, Map(tip1 -> offset1, tip2 -> offset2), verificationError = error) - assertEquals(error, commitOffsetResult(tip1)) - assertEquals(error, commitOffsetResult(tip2)) + assertEquals(expectedError, commitOffsetResult(tip1)) + assertEquals(expectedError, commitOffsetResult(tip2)) } - verifyErrors(Errors.INVALID_PRODUCER_ID_MAPPING) - verifyErrors(Errors.INVALID_TXN_STATE) + verifyErrors(Errors.INVALID_PRODUCER_ID_MAPPING, Errors.INVALID_PRODUCER_ID_MAPPING) + verifyErrors(Errors.INVALID_TXN_STATE, Errors.INVALID_TXN_STATE) + verifyErrors(Errors.NOT_ENOUGH_REPLICAS, Errors.COORDINATOR_NOT_AVAILABLE) } @Test From 4f757e2c8797e784e073c0c40d4d4c82eb8ce00f Mon Sep 17 00:00:00 2001 From: Justine Date: Mon, 11 Dec 2023 15:14:59 -0800 Subject: [PATCH 21/24] Fix other error --- .../scala/kafka/server/ReplicaManager.scala | 4 +-- .../kafka/server/ReplicaManagerTest.scala | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index 5346436c97046..1ec95af077d6a 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -1155,9 +1155,9 @@ class ReplicaManager(val config: KafkaConfig, } } - // All partitions are already verified + // No partitions require verification. if (verificationGuards.isEmpty) { - callback(Map.empty[TopicPartition, Errors], requestLocal, Map.empty[TopicPartition, VerificationGuard]) + callback(errors.toMap, requestLocal, Map.empty[TopicPartition, VerificationGuard]) return } diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index 5c38a9f763cf6..4e7827ffbbe69 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -2582,6 +2582,31 @@ class ReplicaManagerTest { } } + @Test + def testPreVerificationError(): Unit = { + val tp0 = new TopicPartition(topic, 0) + val transactionalId = "txn-id" + val producerId = 24L + val producerEpoch = 0.toShort + val addPartitionsToTxnManager = mock(classOf[AddPartitionsToTxnManager]) + + val replicaManager = setUpReplicaManagerWithMockedAddPartitionsToTxnManager(addPartitionsToTxnManager, List(tp0)) + try { + val result = maybeStartTransactionVerificationForPartition(replicaManager, tp0, transactionalId, producerId, producerEpoch) + val appendCallback = ArgumentCaptor.forClass(classOf[AddPartitionsToTxnManager.AppendCallback]) + verify(addPartitionsToTxnManager, times(0)).verifyTransaction( + ArgumentMatchers.eq(transactionalId), + ArgumentMatchers.eq(producerId), + ArgumentMatchers.eq(producerEpoch), + ArgumentMatchers.eq(Seq(tp0)), + appendCallback.capture() + ) + assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, result.assertFired.left.getOrElse(Errors.NONE)) + } finally { + replicaManager.shutdown(checkpointHW = false) + } + } + @ParameterizedTest @ValueSource(booleans = Array(true, false)) def testFullLeaderAndIsrStrayPartitions(zkMigrationEnabled: Boolean): Unit = { From ac80ae00bfc222ab4aa280029daa2e9ecb04334f Mon Sep 17 00:00:00 2001 From: Justine Date: Mon, 11 Dec 2023 18:15:14 -0800 Subject: [PATCH 22/24] More error fixes and other cleanups --- .../coordinator/group/GroupCoordinator.scala | 5 +- .../group/GroupMetadataManager.scala | 54 +++++++++---------- .../scala/kafka/server/ReplicaManager.scala | 41 ++++++++++++-- .../group/GroupCoordinatorTest.scala | 2 + 4 files changed, 65 insertions(+), 37 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala index b84ab47e0acd3..0f241079365dd 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala @@ -920,10 +920,7 @@ private[group] class GroupCoordinator( verificationGuard: VerificationGuard ): Unit = { if (error != Errors.NONE) { - val finalError = error match { - case Errors.NOT_ENOUGH_REPLICAS => Errors.COORDINATOR_NOT_AVAILABLE - case e => e - } + val finalError = GroupMetadataManager.maybeConvertError(error) responseCallback(offsetMetadata.map { case (k, _) => k -> finalError }) } else { doTxnCommitOffsets(group, memberId, groupInstanceId, generationId, producerId, producerEpoch, diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala index c120b74e01ab0..506908628b34b 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala @@ -26,6 +26,7 @@ import java.util.concurrent.locks.ReentrantLock import java.util.concurrent.ConcurrentHashMap import com.yammer.metrics.core.Gauge import kafka.common.OffsetAndMetadata +import kafka.coordinator.group.GroupMetadataManager.maybeConvertError import kafka.server.{LogAppendResult, ReplicaManager, RequestLocal} import kafka.utils.CoreUtils.inLock import kafka.utils.Implicits._ @@ -283,36 +284,7 @@ class GroupMetadataManager(brokerId: Int, } else { debug(s"Metadata from group ${group.groupId} with generation $generationId failed when appending to log " + s"due to ${status.error.exceptionName}") - - // transform the log append error code to the corresponding the commit status error code - status.error match { - case Errors.UNKNOWN_TOPIC_OR_PARTITION - | Errors.NOT_ENOUGH_REPLICAS - | Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND => - Errors.COORDINATOR_NOT_AVAILABLE - - case Errors.NOT_LEADER_OR_FOLLOWER - | Errors.KAFKA_STORAGE_ERROR => - Errors.NOT_COORDINATOR - - case Errors.REQUEST_TIMED_OUT => - Errors.REBALANCE_IN_PROGRESS - - case Errors.MESSAGE_TOO_LARGE - | Errors.RECORD_LIST_TOO_LARGE - | Errors.INVALID_FETCH_SIZE => - - error(s"Appending metadata message for group ${group.groupId} generation $generationId failed due to " + - s"${status.error.exceptionName}, returning UNKNOWN error code to the client") - - Errors.UNKNOWN_SERVER_ERROR - - case other => - error(s"Appending metadata message for group ${group.groupId} generation $generationId failed " + - s"due to unexpected error: ${status.error.exceptionName}") - - other - } + maybeConvertError(status.error) } responseCallback(responseError) @@ -1378,6 +1350,28 @@ object GroupMetadataManager { "%X".format(BigInt(1, bytes)) } + def maybeConvertError(error: Errors) : Errors = { + error match { + case Errors.UNKNOWN_TOPIC_OR_PARTITION + | Errors.NOT_ENOUGH_REPLICAS + | Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND => + Errors.COORDINATOR_NOT_AVAILABLE + + case Errors.NOT_LEADER_OR_FOLLOWER + | Errors.KAFKA_STORAGE_ERROR => + Errors.NOT_COORDINATOR + + case Errors.MESSAGE_TOO_LARGE + | Errors.RECORD_LIST_TOO_LARGE + | Errors.INVALID_FETCH_SIZE => + Errors.INVALID_COMMIT_OFFSET_SIZE + + // We may see INVALID_TXN_STATE or INVALID_PID_MAPPING here due to transaction verification. + // They can be returned without mapping to a new error. + case other => other + } + } + } case class GroupTopicPartition(group: String, topicPartition: TopicPartition) { diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index 1ec95af077d6a..1f65265298893 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -997,8 +997,6 @@ class ReplicaManager(val config: KafkaConfig, * * @param timeout maximum time we will wait to append before returning * @param requiredAcks number of replicas who must acknowledge the append before sending the response - * @param internalTopicsAllowed boolean indicating whether internal topics can be appended to - * @param origin source of the append request (ie, client, replication, coordinator) * @param entriesPerPartition the records per partition to be appended * @param responseCallback callback for sending the response * @param delayedProduceLock lock for the delayed actions @@ -1020,6 +1018,12 @@ class ReplicaManager(val config: KafkaConfig, return } + // This should only be called on __consumer_offsets until KAFKA-15987 is complete + entriesPerPartition.keys.foreach { tp => + if (!tp.topic.equals(Topic.GROUP_METADATA_TOPIC_NAME)) + throw new IllegalArgumentException() + } + val sTime = time.milliseconds val localProduceResults = appendToLocalLog( internalTopicsAllowed = true, @@ -1091,6 +1095,21 @@ class ReplicaManager(val config: KafkaConfig, responseCallback(responseStatus) } + /** + * + * @param topicPartition the topic partition to maybe verify + * @param transactionalId the transactional id for the transaction + * @param producerId the producer id for the producer writing to the transaction + * @param producerEpoch the epoch of the producer writing to the transaction + * @param baseSequence the base sequence of the first record in the batch we are trying to append + * @param requestLocal container for the stateful instances scoped to this request -- this must correspond to the + * thread calling this method + * @param callback the method to execute once the verification is either completed or returns an error + * + * When the verification returns, the callback will be supplied the error if it exists or Errors.NONE. + * If the verification guard exists, it will also be supplied. Otherwise the SENTINEL verification guard will be returned. + * This guard can not be used for verification and any appends that attenpt to use it will fail. + */ def maybeStartTransactionVerificationForPartition( topicPartition: TopicPartition, transactionalId: String, @@ -1119,6 +1138,23 @@ class ReplicaManager(val config: KafkaConfig, ) } + /** + * + * @param topicPartitionBatchInfo the topic partitions to maybe verify mapped to the base sequence of their first record batch + * @param transactionalId the transactional id for the transaction + * @param producerId the producer id for the producer writing to the transaction + * @param producerEpoch the epoch of the producer writing to the transaction + * @param requestLocal container for the stateful instances scoped to this request -- this must correspond to the + * thread calling this method + * @param callback the method to execute once the verification is either completed or returns an error + * + * When the verification returns, the callback will be supplied the errors per topic partition if there were errors. + * The callback will also be supplied the verification guards per partition if they exist. It is possible to have an + * error and a verification guard for a topic partition if the topic partition was unable to be verified by the transaction + * coordinator. Transaction coordinator errors are mapped to append-friendly errors. The callback is wrapped so that it + * is scheduled on a request handler thread. There, it should be called with that request handler thread's thread local and + * not the one supplied to this method. + */ def maybeStartTransactionVerificationForPartitions( topicPartitionBatchInfo: Map[TopicPartition, Int], transactionalId: String, @@ -1136,7 +1172,6 @@ class ReplicaManager(val config: KafkaConfig, return } - // Wrap the callback to be handled on an arbitrary request handler thread when transaction verification is complete. val verificationGuards = mutable.Map[TopicPartition, VerificationGuard]() val errors = mutable.Map[TopicPartition, Errors]() diff --git a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala index 6999516d723fe..2b6e226ec64bc 100644 --- a/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala +++ b/core/src/test/scala/unit/kafka/coordinator/group/GroupCoordinatorTest.scala @@ -3804,6 +3804,8 @@ class GroupCoordinatorTest { verifyErrors(Errors.INVALID_PRODUCER_ID_MAPPING, Errors.INVALID_PRODUCER_ID_MAPPING) verifyErrors(Errors.INVALID_TXN_STATE, Errors.INVALID_TXN_STATE) verifyErrors(Errors.NOT_ENOUGH_REPLICAS, Errors.COORDINATOR_NOT_AVAILABLE) + verifyErrors(Errors.NOT_LEADER_OR_FOLLOWER, Errors.NOT_COORDINATOR) + verifyErrors(Errors.KAFKA_STORAGE_ERROR, Errors.NOT_COORDINATOR) } @Test From 13dd6549e9ef6b9a9a463fe34608d24cd82f8730 Mon Sep 17 00:00:00 2001 From: Justine Date: Tue, 12 Dec 2023 09:33:42 -0800 Subject: [PATCH 23/24] fix silly error mix up --- .../group/GroupMetadataManager.scala | 52 +++++++++++-------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala index 506908628b34b..f96d8edc828ea 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala @@ -284,7 +284,36 @@ class GroupMetadataManager(brokerId: Int, } else { debug(s"Metadata from group ${group.groupId} with generation $generationId failed when appending to log " + s"due to ${status.error.exceptionName}") - maybeConvertError(status.error) + + // transform the log append error code to the corresponding the commit status error code + status.error match { + case Errors.UNKNOWN_TOPIC_OR_PARTITION + | Errors.NOT_ENOUGH_REPLICAS + | Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND => + Errors.COORDINATOR_NOT_AVAILABLE + + case Errors.NOT_LEADER_OR_FOLLOWER + | Errors.KAFKA_STORAGE_ERROR => + Errors.NOT_COORDINATOR + + case Errors.REQUEST_TIMED_OUT => + Errors.REBALANCE_IN_PROGRESS + + case Errors.MESSAGE_TOO_LARGE + | Errors.RECORD_LIST_TOO_LARGE + | Errors.INVALID_FETCH_SIZE => + + error(s"Appending metadata message for group ${group.groupId} generation $generationId failed due to " + + s"${status.error.exceptionName}, returning UNKNOWN error code to the client") + + Errors.UNKNOWN_SERVER_ERROR + + case other => + error(s"Appending metadata message for group ${group.groupId} generation $generationId failed " + + s"due to unexpected error: ${status.error.exceptionName}") + + other + } } responseCallback(responseError) @@ -395,26 +424,7 @@ class GroupMetadataManager(brokerId: Int, debug(s"Offset commit $filteredOffsetMetadata from group ${group.groupId}, consumer $consumerId " + s"with generation ${group.generationId} failed when appending to log due to ${status.error.exceptionName}") - // transform the log append error code to the corresponding the commit status error code - status.error match { - case Errors.UNKNOWN_TOPIC_OR_PARTITION - | Errors.NOT_ENOUGH_REPLICAS - | Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND => - Errors.COORDINATOR_NOT_AVAILABLE - - case Errors.NOT_LEADER_OR_FOLLOWER - | Errors.KAFKA_STORAGE_ERROR => - Errors.NOT_COORDINATOR - - case Errors.MESSAGE_TOO_LARGE - | Errors.RECORD_LIST_TOO_LARGE - | Errors.INVALID_FETCH_SIZE => - Errors.INVALID_COMMIT_OFFSET_SIZE - - // We may see INVALID_TXN_STATE or INVALID_PID_MAPPING here due to transaction verification. - // They can be returned without mapping to a new error. - case other => other - } + maybeConvertError(status.error) } } From 7cad5676752aadc1ad9a781507130522e073f49d Mon Sep 17 00:00:00 2001 From: Justine Date: Tue, 12 Dec 2023 15:54:12 -0800 Subject: [PATCH 24/24] rename method --- .../scala/kafka/coordinator/group/GroupCoordinator.scala | 2 +- .../kafka/coordinator/group/GroupMetadataManager.scala | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala index 0f241079365dd..aba205372b685 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupCoordinator.scala @@ -920,7 +920,7 @@ private[group] class GroupCoordinator( verificationGuard: VerificationGuard ): Unit = { if (error != Errors.NONE) { - val finalError = GroupMetadataManager.maybeConvertError(error) + val finalError = GroupMetadataManager.maybeConvertOffsetCommitError(error) responseCallback(offsetMetadata.map { case (k, _) => k -> finalError }) } else { doTxnCommitOffsets(group, memberId, groupInstanceId, generationId, producerId, producerEpoch, diff --git a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala index f96d8edc828ea..31af1d81a2218 100644 --- a/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala +++ b/core/src/main/scala/kafka/coordinator/group/GroupMetadataManager.scala @@ -26,7 +26,7 @@ import java.util.concurrent.locks.ReentrantLock import java.util.concurrent.ConcurrentHashMap import com.yammer.metrics.core.Gauge import kafka.common.OffsetAndMetadata -import kafka.coordinator.group.GroupMetadataManager.maybeConvertError +import kafka.coordinator.group.GroupMetadataManager.maybeConvertOffsetCommitError import kafka.server.{LogAppendResult, ReplicaManager, RequestLocal} import kafka.utils.CoreUtils.inLock import kafka.utils.Implicits._ @@ -424,7 +424,7 @@ class GroupMetadataManager(brokerId: Int, debug(s"Offset commit $filteredOffsetMetadata from group ${group.groupId}, consumer $consumerId " + s"with generation ${group.generationId} failed when appending to log due to ${status.error.exceptionName}") - maybeConvertError(status.error) + maybeConvertOffsetCommitError(status.error) } } @@ -1360,7 +1360,7 @@ object GroupMetadataManager { "%X".format(BigInt(1, bytes)) } - def maybeConvertError(error: Errors) : Errors = { + def maybeConvertOffsetCommitError(error: Errors) : Errors = { error match { case Errors.UNKNOWN_TOPIC_OR_PARTITION | Errors.NOT_ENOUGH_REPLICAS