Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
e894d84
Redo verification path
jolshan Nov 16, 2023
2a52318
Fix build issues
jolshan Nov 17, 2023
865078b
Fix tests
jolshan Nov 17, 2023
a20f238
Merge branch 'trunk' of github.com:apache/kafka into kafka-15784
jolshan Nov 17, 2023
9bac9a9
Fix test failures
jolshan Nov 17, 2023
b4a920b
Rewrite GroupCoordinator and GroupMetadataManager to handle checks fo…
jolshan Nov 29, 2023
05c0d28
Merge branch 'trunk' of github.com:apache/kafka into kafka-15784
jolshan Nov 29, 2023
7bc6d06
Update comments and method names
jolshan Nov 30, 2023
a471f04
Fix style issues and passing verification guards
jolshan Dec 4, 2023
7c01682
Clean up GroupCoordinator, GroupMetadataManger, and ReplicaManager code
jolshan Dec 6, 2023
965ec39
remove package private scoping that is not needed
jolshan Dec 6, 2023
54197cb
Merge commit '965ec39460324d77d55c90771f23ac0c9c2ad70b' into kafka-15987
jolshan Dec 28, 2023
e214171
clean ups
jolshan Dec 28, 2023
4b944d5
cleanups
jolshan Dec 28, 2023
95480af
Test fixes
jolshan Jan 3, 2024
1f41590
Merge branch 'trunk' of github.com:apache/kafka into kafka-15987
jolshan Jan 3, 2024
7ae321c
Hide extra parameter away
jolshan Jan 5, 2024
e1fa7d6
Small format fix
jolshan Jan 8, 2024
9d2ba2e
Merge branch 'trunk' of github.com:apache/kafka into kafka-15987
jolshan Jan 8, 2024
5518d80
Merge branch 'trunk' of github.com:apache/kafka into kafka-15987
jolshan Jan 11, 2024
3034e52
Addressing comments
jolshan Jan 23, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse
import org.apache.kafka.common.requests.TransactionResult
import org.apache.kafka.common.utils.{BufferSupplier, Time}
import org.apache.kafka.coordinator.group.runtime.PartitionWriter
import org.apache.kafka.storage.internals.log.VerificationGuard
import org.apache.kafka.storage.internals.log.{AppendOrigin, VerificationGuard}

import java.util
import java.util.concurrent.CompletableFuture
Expand Down Expand Up @@ -215,9 +215,11 @@ class CoordinatorPartitionWriter[T](
verificationGuard: VerificationGuard = VerificationGuard.SENTINEL
): Long = {
var appendResults: Map[TopicPartition, PartitionResponse] = Map.empty
replicaManager.appendForGroup(
replicaManager.appendRecords(
timeout = 0L,
requiredAcks = 1,
internalTopicsAllowed = true,
origin = AppendOrigin.COORDINATOR,
entriesPerPartition = Map(tp -> memoryRecords),
responseCallback = results => appendResults = results,
requestLocal = RequestLocal.NoCaching,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -335,9 +335,11 @@ class GroupMetadataManager(brokerId: Int,
verificationGuards: Map[TopicPartition, VerificationGuard] = Map.empty
): Unit = {
// call replica manager to append the group message
replicaManager.appendForGroup(
replicaManager.appendRecords(
timeout = config.offsetCommitTimeoutMs.toLong,
requiredAcks = config.offsetCommitRequiredAcks,
internalTopicsAllowed = true,
origin = AppendOrigin.COORDINATOR,
entriesPerPartition = records,
delayedProduceLock = Some(group.lock),
responseCallback = callback,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -281,12 +281,12 @@ class TransactionStateManager(brokerId: Int,

inReadLock(stateLock) {
replicaManager.appendRecords(
config.requestTimeoutMs,
TransactionLog.EnforcedRequiredAcks,
timeout = config.requestTimeoutMs,
requiredAcks = TransactionLog.EnforcedRequiredAcks,
internalTopicsAllowed = true,
origin = AppendOrigin.COORDINATOR,
entriesPerPartition = Map(transactionPartition -> tombstoneRecords),
removeFromCacheCallback,
responseCallback = removeFromCacheCallback,
requestLocal = RequestLocal.NoCaching)
}
}
Expand Down
8 changes: 3 additions & 5 deletions core/src/main/scala/kafka/server/KafkaApis.scala
Original file line number Diff line number Diff line change
Expand Up @@ -717,17 +717,15 @@ class KafkaApis(val requestChannel: RequestChannel,
val internalTopicsAllowed = request.header.clientId == AdminUtils.ADMIN_CLIENT_ID

// call the replica manager to append messages to the replicas
replicaManager.appendRecords(
replicaManager.handleProduceAppend(
timeout = produceRequest.timeout.toLong,
requiredAcks = produceRequest.acks,
internalTopicsAllowed = internalTopicsAllowed,
origin = AppendOrigin.CLIENT,
transactionalId = produceRequest.transactionalId,
entriesPerPartition = authorizedRequestInfo,
requestLocal = requestLocal,
responseCallback = sendResponseCallback,
recordValidationStatsCallback = processingStatsCallback,
transactionalId = produceRequest.transactionalId()
)
requestLocal = requestLocal)

// if the request is put into the purgatory, it will have a held reference and hence cannot be garbage collected;
// hence we clear its data here in order to let GC reclaim its memory since it is already appended to log
Expand Down
8 changes: 4 additions & 4 deletions core/src/main/scala/kafka/server/KafkaRequestHandler.scala
Original file line number Diff line number Diff line change
Expand Up @@ -67,17 +67,17 @@ object KafkaRequestHandler {
if (requestChannel == null || currentRequest == null) {
if (!bypassThreadCheck)
throw new IllegalStateException("Attempted to reschedule to request handler thread from non-request handler thread.")
T => asyncCompletionCallback(requestLocal, T)
t => asyncCompletionCallback(requestLocal, t)
} else {
T => {
t => {
if (threadCurrentRequest.get() == currentRequest) {
// If the callback is actually executed on the same request thread, we can directly execute
// it without re-scheduling it.
asyncCompletionCallback(requestLocal, T)
asyncCompletionCallback(requestLocal, t)
} else {
// The requestChannel and request are captured in this lambda, so when it's executed on the callback thread
// we can re-schedule the original callback on a request thread and update the metrics accordingly.
requestChannel.sendCallbackRequest(RequestChannel.CallbackRequest(newRequestLocal => asyncCompletionCallback(newRequestLocal, T), currentRequest))
requestChannel.sendCallbackRequest(RequestChannel.CallbackRequest(newRequestLocal => asyncCompletionCallback(newRequestLocal, t), currentRequest))
}
}
}
Expand Down
Loading