diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 3f2df5a32b650..5d4ad4eb4b019 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -322,14 +322,16 @@ - - + + files="(ConsumerGroupMember|GroupMetadataManager).java"/> + files="(RecordHelpersTest|GroupMetadataManagerTest).java"/> + = 4; + } + + /** + * Starting from version 9 of the JoinGroup API, static members are able to + * skip running the assignor based on the `SkipAssignment` field. We leverage + * this to tell the leader that it is the leader of the group but by skipping + * running the assignor while the group is in stable state. + * Notes: + * 1) This allows the leader to continue monitoring metadata changes for the + * group. Note that any metadata changes happening while the static leader is + * down won't be noticed. + * 2) The assignors are not idempotent nor free from side effects. This is why + * we skip entirely the assignment step as it could generate a different group + * assignment which would be ignored by the group coordinator because the group + * is the stable state. + * + * Prior to version 9 of the JoinGroup API, we wanted to avoid current leader + * performing trivial assignment while the group is in stable stage, because + * the new assignment in leader's next sync call won't be broadcast by a stable group. + * This could be guaranteed by always returning the old leader id so that the current + * leader won't assume itself as a leader based on the returned message, since the new + * member.id won't match returned leader id, therefore no assignment will be performed. + * + * @param apiVersion The JoinGroupRequest api version. + * + * @return whether the version supports skipping assignment. + */ + + public static boolean supportsSkippingAssignment(short apiVersion) { + return apiVersion >= 9; + } + + /** + * Get the client's join reason. + * + * @param request The JoinGroupRequest. + * + * @return The join reason. + */ + public static String joinReason(JoinGroupRequestData request) { + String joinReason = request.reason(); + if (joinReason == null || joinReason.isEmpty()) { + joinReason = "not provided"; + } + return joinReason; + } + public JoinGroupRequest(JoinGroupRequestData data, short version) { super(ApiKeys.JOIN_GROUP, version); this.data = data; diff --git a/core/src/main/scala/kafka/server/BrokerServer.scala b/core/src/main/scala/kafka/server/BrokerServer.scala index 8641a74824b56..3094ab2359a22 100644 --- a/core/src/main/scala/kafka/server/BrokerServer.scala +++ b/core/src/main/scala/kafka/server/BrokerServer.scala @@ -531,7 +531,12 @@ class BrokerServer( config.consumerGroupHeartbeatIntervalMs, config.consumerGroupMaxSize, config.consumerGroupAssignors, - config.offsetsTopicSegmentBytes + config.offsetsTopicSegmentBytes, + config.groupMaxSize, + config.groupInitialRebalanceDelay, + GroupCoordinatorConfig.GENERIC_GROUP_NEW_MEMBER_JOIN_TIMEOUT_MS, + config.groupMinSessionTimeoutMs, + config.groupMaxSessionTimeoutMs ) val timer = new SystemTimerReaper( "group-coordinator-reaper", diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupCoordinatorConfig.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupCoordinatorConfig.java index f4b23a429fabc..6bba39bc16dab 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupCoordinatorConfig.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupCoordinatorConfig.java @@ -25,6 +25,11 @@ */ public class GroupCoordinatorConfig { + /** + * The timeout used to wait for a new member in milliseconds. + */ + public static final int GENERIC_GROUP_NEW_MEMBER_JOIN_TIMEOUT_MS = 5 * 60 * 1000; + /** * The number of threads or event loops running. */ @@ -56,13 +61,43 @@ public class GroupCoordinatorConfig { */ public final int offsetsTopicSegmentBytes; + /** + * The generic group maximum size. + */ + public final int genericGroupMaxSize; + + /** + * The delay in milliseconds introduced for the first rebalance of a generic group. + */ + public final int genericGroupInitialRebalanceDelayMs; + + /** + * The timeout used to wait for a new member in milliseconds. + */ + public final int genericGroupNewMemberJoinTimeoutMs; + + /** + * The generic group minimum session timeout. + */ + public final int genericGroupMinSessionTimeoutMs; + + /** + * The generic group maximum session timeout. + */ + public final int genericGroupMaxSessionTimeoutMs; + public GroupCoordinatorConfig( int numThreads, int consumerGroupSessionTimeoutMs, int consumerGroupHeartbeatIntervalMs, int consumerGroupMaxSize, List consumerGroupAssignors, - int offsetsTopicSegmentBytes + int offsetsTopicSegmentBytes, + int genericGroupMaxSize, + int genericGroupInitialRebalanceDelayMs, + int genericGroupNewMemberJoinTimeoutMs, + int genericGroupMinSessionTimeoutMs, + int genericGroupMaxSessionTimeoutMs ) { this.numThreads = numThreads; this.consumerGroupSessionTimeoutMs = consumerGroupSessionTimeoutMs; @@ -70,5 +105,10 @@ public GroupCoordinatorConfig( this.consumerGroupMaxSize = consumerGroupMaxSize; this.consumerGroupAssignors = consumerGroupAssignors; this.offsetsTopicSegmentBytes = offsetsTopicSegmentBytes; + this.genericGroupMaxSize = genericGroupMaxSize; + this.genericGroupInitialRebalanceDelayMs = genericGroupInitialRebalanceDelayMs; + this.genericGroupNewMemberJoinTimeoutMs = genericGroupNewMemberJoinTimeoutMs; + this.genericGroupMinSessionTimeoutMs = genericGroupMinSessionTimeoutMs; + this.genericGroupMaxSessionTimeoutMs = genericGroupMaxSessionTimeoutMs; } } diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupCoordinatorService.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupCoordinatorService.java index 324a3e8a05780..571ea6e2b9a11 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupCoordinatorService.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupCoordinatorService.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.coordinator.group; +import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.TopicConfig; import org.apache.kafka.common.errors.InvalidFetchSizeException; @@ -123,10 +124,10 @@ public GroupCoordinatorService build() { throw new IllegalArgumentException("Writer must be set."); if (loader == null) throw new IllegalArgumentException("Loader must be set."); - if (timer == null) - throw new IllegalArgumentException("Timer must be set."); if (time == null) throw new IllegalArgumentException("Time must be set."); + if (timer == null) + throw new IllegalArgumentException("Timer must be set."); String logPrefix = String.format("GroupCoordinator id=%d", nodeId); LogContext logContext = new LogContext(String.format("[%s] ", logPrefix)); @@ -150,6 +151,7 @@ public GroupCoordinatorService build() { .withPartitionWriter(writer) .withLoader(loader) .withCoordinatorBuilderSupplier(supplier) + .withTime(time) .build(); return new GroupCoordinatorService( @@ -286,9 +288,33 @@ public CompletableFuture joinGroup( return FutureUtils.failedFuture(Errors.COORDINATOR_NOT_AVAILABLE.exception()); } - return FutureUtils.failedFuture(Errors.UNSUPPORTED_VERSION.exception( - "This API is not implemented yet." - )); + CompletableFuture responseFuture = new CompletableFuture<>(); + + if (!isGroupIdNotEmpty(request.groupId())) { + responseFuture.complete(new JoinGroupResponseData() + .setMemberId(request.memberId()) + .setErrorCode(Errors.INVALID_GROUP_ID.code())); + + return responseFuture; + } + + runtime.scheduleWriteOperation("generic-group-join", + topicPartitionFor(request.groupId()), + coordinator -> coordinator.genericGroupJoin(context, request, responseFuture) + ).exceptionally(exception -> { + if (!(exception instanceof KafkaException)) { + log.error("JoinGroup request {} hit an unexpected exception: {}", + request, exception.getMessage()); + } + + if (!responseFuture.isDone()) { + responseFuture.complete(new JoinGroupResponseData() + .setErrorCode(Errors.forException(exception).code())); + } + return null; + }); + + return responseFuture; } /** @@ -599,4 +625,8 @@ public void shutdown() { Utils.closeQuietly(runtime, "coordinator runtime"); log.info("Shutdown complete."); } + + private static boolean isGroupIdNotEmpty(String groupId) { + return groupId != null && !groupId.isEmpty(); + } } diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java index c0804ff6bdc0f..da5010eac0313 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupMetadataManager.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.coordinator.group; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.ApiException; import org.apache.kafka.common.errors.FencedMemberEpochException; @@ -28,6 +29,13 @@ import org.apache.kafka.common.errors.UnsupportedAssignorException; import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; import org.apache.kafka.common.message.ConsumerGroupHeartbeatResponseData; +import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocol; +import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection; +import org.apache.kafka.common.message.JoinGroupRequestData; +import org.apache.kafka.common.message.JoinGroupResponseData; +import org.apache.kafka.common.message.SyncGroupResponseData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.JoinGroupRequest; import org.apache.kafka.common.requests.RequestContext; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; @@ -51,7 +59,11 @@ import org.apache.kafka.coordinator.group.generated.ConsumerGroupTargetAssignmentMemberValue; import org.apache.kafka.coordinator.group.generated.ConsumerGroupTargetAssignmentMetadataKey; import org.apache.kafka.coordinator.group.generated.ConsumerGroupTargetAssignmentMetadataValue; +import org.apache.kafka.coordinator.group.generated.GroupMetadataKey; +import org.apache.kafka.coordinator.group.generated.GroupMetadataValue; import org.apache.kafka.coordinator.group.generic.GenericGroup; +import org.apache.kafka.coordinator.group.generic.GenericGroupMember; +import org.apache.kafka.coordinator.group.generic.GenericGroupState; import org.apache.kafka.coordinator.group.runtime.CoordinatorResult; import org.apache.kafka.coordinator.group.runtime.CoordinatorTimer; import org.apache.kafka.image.MetadataDelta; @@ -71,10 +83,17 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; +import static org.apache.kafka.common.protocol.Errors.COORDINATOR_NOT_AVAILABLE; +import static org.apache.kafka.common.protocol.Errors.NOT_COORDINATOR; +import static org.apache.kafka.common.protocol.Errors.UNKNOWN_SERVER_ERROR; +import static org.apache.kafka.common.requests.JoinGroupRequest.UNKNOWN_MEMBER_ID; +import static org.apache.kafka.coordinator.group.Group.GroupType.CONSUMER; +import static org.apache.kafka.coordinator.group.Group.GroupType.GENERIC; import static org.apache.kafka.coordinator.group.RecordHelpers.newCurrentAssignmentRecord; import static org.apache.kafka.coordinator.group.RecordHelpers.newCurrentAssignmentTombstoneRecord; import static org.apache.kafka.coordinator.group.RecordHelpers.newGroupEpochRecord; @@ -82,6 +101,11 @@ import static org.apache.kafka.coordinator.group.RecordHelpers.newMemberSubscriptionRecord; import static org.apache.kafka.coordinator.group.RecordHelpers.newMemberSubscriptionTombstoneRecord; import static org.apache.kafka.coordinator.group.RecordHelpers.newTargetAssignmentTombstoneRecord; +import static org.apache.kafka.coordinator.group.generic.GenericGroupState.COMPLETING_REBALANCE; +import static org.apache.kafka.coordinator.group.generic.GenericGroupState.DEAD; +import static org.apache.kafka.coordinator.group.generic.GenericGroupState.EMPTY; +import static org.apache.kafka.coordinator.group.generic.GenericGroupState.PREPARING_REBALANCE; +import static org.apache.kafka.coordinator.group.generic.GenericGroupState.STABLE; /** * The GroupMetadataManager manages the metadata of all generic and consumer groups. It holds @@ -98,13 +122,19 @@ public static class Builder { private LogContext logContext = null; private SnapshotRegistry snapshotRegistry = null; private Time time = null; - private CoordinatorTimer timer = null; + private CoordinatorTimer timer = null; private List assignors = null; private int consumerGroupMaxSize = Integer.MAX_VALUE; - private int consumerGroupSessionTimeoutMs = 45000; private int consumerGroupHeartbeatIntervalMs = 5000; private int consumerGroupMetadataRefreshIntervalMs = Integer.MAX_VALUE; + private TopicPartition topicPartition = null; private MetadataImage metadataImage = null; + private int consumerGroupSessionTimeoutMs = 45000; + private int genericGroupMaxSize = Integer.MAX_VALUE; + private int genericGroupInitialRebalanceDelayMs = 3000; + private int genericGroupNewMemberJoinTimeoutMs = 5 * 60 * 1000; + private int genericGroupMinSessionTimeoutMs; + private int genericGroupMaxSessionTimeoutMs; Builder withLogContext(LogContext logContext) { this.logContext = logContext; @@ -121,7 +151,7 @@ Builder withTime(Time time) { return this; } - Builder withTimer(CoordinatorTimer timer) { + Builder withTimer(CoordinatorTimer timer) { this.timer = timer; return this; } @@ -156,6 +186,36 @@ Builder withMetadataImage(MetadataImage metadataImage) { return this; } + Builder withTopicPartition(TopicPartition tp) { + this.topicPartition = tp; + return this; + } + + Builder withGenericGroupMaxSize(int genericGroupMaxSize) { + this.genericGroupMaxSize = genericGroupMaxSize; + return this; + } + + Builder withGenericGroupInitialRebalanceDelayMs(int genericGroupInitialRebalanceDelayMs) { + this.genericGroupInitialRebalanceDelayMs = genericGroupInitialRebalanceDelayMs; + return this; + } + + Builder withGenericGroupNewMemberJoinTimeoutMs(int genericGroupNewMemberJoinTimeoutMs) { + this.genericGroupNewMemberJoinTimeoutMs = genericGroupNewMemberJoinTimeoutMs; + return this; + } + + Builder withGenericGroupMinSessionTimeoutMs(int genericGroupMinSessionTimeoutMs) { + this.genericGroupMinSessionTimeoutMs = genericGroupMinSessionTimeoutMs; + return this; + } + + Builder withGenericGroupMaxSessionTimeoutMs(int genericGroupMaxSessionTimeoutMs) { + this.genericGroupMaxSessionTimeoutMs = genericGroupMaxSessionTimeoutMs; + return this; + } + GroupMetadataManager build() { if (logContext == null) logContext = new LogContext(); if (snapshotRegistry == null) snapshotRegistry = new SnapshotRegistry(logContext); @@ -167,7 +227,12 @@ GroupMetadataManager build() { if (assignors == null || assignors.isEmpty()) throw new IllegalArgumentException("Assignors must be set before building."); + if (topicPartition == null) { + throw new IllegalStateException("TopicPartition must be set before building."); + } + return new GroupMetadataManager( + topicPartition, snapshotRegistry, logContext, time, @@ -177,11 +242,26 @@ GroupMetadataManager build() { consumerGroupMaxSize, consumerGroupSessionTimeoutMs, consumerGroupHeartbeatIntervalMs, - consumerGroupMetadataRefreshIntervalMs + consumerGroupMetadataRefreshIntervalMs, + genericGroupMaxSize, + genericGroupInitialRebalanceDelayMs, + genericGroupNewMemberJoinTimeoutMs, + genericGroupMinSessionTimeoutMs, + genericGroupMaxSessionTimeoutMs ); } } + /** + * The topic partition associated with the metadata manager. + */ + private final TopicPartition topicPartition; + + /** + * The log context. + */ + private final LogContext logContext; + /** * The logger. */ @@ -200,7 +280,7 @@ GroupMetadataManager build() { /** * The system timer. */ - private final CoordinatorTimer timer; + private final CoordinatorTimer timer; /** * The supported partition assignors keyed by their name. @@ -247,24 +327,66 @@ GroupMetadataManager build() { */ private MetadataImage metadataImage; + /** + * An empty result returned to the state machine. This means that + * there are no records to append to the log. + * + * Package private for testing. + */ + static final CoordinatorResult EMPTY_RESULT = + new CoordinatorResult<>(Collections.emptyList(), CompletableFuture.completedFuture(null)); + + /** + * The maximum number of members allowed in a single generic group. + */ + private final int genericGroupMaxSize; + + /** + * Initial rebalance delay for members joining a generic group. + */ + private final int genericGroupInitialRebalanceDelayMs; + + /** + * The timeout used to wait for a new member in milliseconds. + */ + private final int genericGroupNewMemberJoinTimeoutMs; + + /** + * The group minimum session timeout. + */ + private final int genericGroupMinSessionTimeoutMs; + + /** + * The group maximum session timeout. + */ + private final int genericGroupMaxSessionTimeoutMs; + private GroupMetadataManager( + TopicPartition topicPartition, SnapshotRegistry snapshotRegistry, LogContext logContext, Time time, - CoordinatorTimer timer, + CoordinatorTimer timer, List assignors, MetadataImage metadataImage, int consumerGroupMaxSize, int consumerGroupSessionTimeoutMs, int consumerGroupHeartbeatIntervalMs, - int consumerGroupMetadataRefreshIntervalMs + int consumerGroupMetadataRefreshIntervalMs, + int genericGroupMaxSize, + int genericGroupInitialRebalanceDelayMs, + int genericGroupNewMemberJoinTimeoutMs, + int genericGroupMinSessionTimeoutMs, + int genericGroupMaxSessionTimeoutMs ) { + this.logContext = logContext; this.log = logContext.logger(GroupMetadataManager.class); this.snapshotRegistry = snapshotRegistry; this.time = time; this.timer = timer; this.metadataImage = metadataImage; this.assignors = assignors.stream().collect(Collectors.toMap(PartitionAssignor::name, Function.identity())); + this.topicPartition = topicPartition; this.defaultAssignor = assignors.get(0); this.groups = new TimelineHashMap<>(snapshotRegistry, 0); this.groupsByTopics = new TimelineHashMap<>(snapshotRegistry, 0); @@ -272,6 +394,11 @@ private GroupMetadataManager( this.consumerGroupSessionTimeoutMs = consumerGroupSessionTimeoutMs; this.consumerGroupHeartbeatIntervalMs = consumerGroupHeartbeatIntervalMs; this.consumerGroupMetadataRefreshIntervalMs = consumerGroupMetadataRefreshIntervalMs; + this.genericGroupMaxSize = genericGroupMaxSize; + this.genericGroupInitialRebalanceDelayMs = genericGroupInitialRebalanceDelayMs; + this.genericGroupNewMemberJoinTimeoutMs = genericGroupNewMemberJoinTimeoutMs; + this.genericGroupMinSessionTimeoutMs = genericGroupMinSessionTimeoutMs; + this.genericGroupMaxSessionTimeoutMs = genericGroupMaxSessionTimeoutMs; } /** @@ -309,7 +436,7 @@ ConsumerGroup getOrMaybeCreateConsumerGroup( groups.put(groupId, consumerGroup); return consumerGroup; } else { - if (group.type() == Group.GroupType.CONSUMER) { + if (group.type() == CONSUMER) { return (ConsumerGroup) group; } else { // We don't support upgrading/downgrading between protocols at the moment so @@ -319,6 +446,45 @@ ConsumerGroup getOrMaybeCreateConsumerGroup( } } + /** + * Gets or maybe creates a generic group. + * + * @param groupId The group id. + * @param createIfNotExists A boolean indicating whether the group should be + * created if it does not exist. + * + * @return A GenericGroup. + * @throws UnknownMemberIdException if the group does not exist and createIfNotExists is false. + * @throws GroupIdNotFoundException if the group is not a generic group. + * + * Package private for testing. + */ + GenericGroup getOrMaybeCreateGenericGroup( + String groupId, + boolean createIfNotExists + ) throws UnknownMemberIdException, GroupIdNotFoundException { + Group group = groups.get(groupId); + + if (group == null && !createIfNotExists) { + throw new UnknownMemberIdException(String.format("Generic group %s not found.", groupId)); + } + + if (group == null) { + GenericGroup genericGroup = new GenericGroup(logContext, groupId, GenericGroupState.EMPTY, time); + groups.put(groupId, genericGroup); + return genericGroup; + } else { + if (group.type() == GENERIC) { + return (GenericGroup) group; + } else { + // We don't support upgrading/downgrading between protocols at the moment so + // we throw an exception if a group exists with the wrong type. + throw new GroupIdNotFoundException(String.format("Group %s is not a generic group.", + groupId)); + } + } + } + /** * Removes the group. * @@ -798,7 +964,7 @@ private void scheduleConsumerGroupSessionTimeout( ConsumerGroupMember member = group.getOrMaybeCreateMember(memberId, false); log.info("[GroupId {}] Member {} fenced from the group because its session expired.", groupId, memberId); - return consumerGroupFenceMember(group, member); + return new CoordinatorResult<>(consumerGroupFenceMember(group, member)); } catch (GroupIdNotFoundException ex) { log.debug("[GroupId {}] Could not fence {} because the group does not exist.", groupId, memberId); @@ -807,7 +973,7 @@ private void scheduleConsumerGroupSessionTimeout( groupId, memberId); } - return Collections.emptyList(); + return new CoordinatorResult<>(Collections.emptyList()); }); } @@ -848,12 +1014,12 @@ private void scheduleConsumerGroupRevocationTimeout( member.memberEpoch() != expectedMemberEpoch) { log.debug("[GroupId {}] Ignoring revocation timeout for {} because the member " + "state does not match the expected state.", groupId, memberId); - return Collections.emptyList(); + return new CoordinatorResult<>(Collections.emptyList()); } log.info("[GroupId {}] Member {} fenced from the group because " + "it failed to revoke partitions within {}ms.", groupId, memberId, revocationTimeoutMs); - return consumerGroupFenceMember(group, member); + return new CoordinatorResult<>(consumerGroupFenceMember(group, member)); } catch (GroupIdNotFoundException ex) { log.debug("[GroupId {}] Could not fence {}} because the group does not exist.", groupId, memberId); @@ -862,7 +1028,7 @@ private void scheduleConsumerGroupRevocationTimeout( groupId, memberId); } - return Collections.emptyList(); + return new CoordinatorResult<>(Collections.emptyList()); }); } @@ -1233,6 +1399,17 @@ public void onLoaded() { case GENERIC: GenericGroup genericGroup = (GenericGroup) group; log.info("Loaded generic group {} with {} members.", groupId, genericGroup.allMembers().size()); + genericGroup.allMembers().forEach(member -> { + log.debug("Loaded member {} in generic group {}.", member.memberId(), groupId); + rescheduleGenericGroupMemberHeartbeat(genericGroup, member); + }); + + if (genericGroup.size() > genericGroupMaxSize) { + // In case the max size config has changed. + prepareRebalance(genericGroup, "Freshly-loaded group " + groupId + + " (size " + genericGroup.size() + ") is over capacity " + genericGroupMaxSize + + ". Rebalancing in order to give a chance for consumers to commit offsets"); + } break; } }); @@ -1245,4 +1422,1305 @@ public static String consumerGroupSessionTimeoutKey(String groupId, String membe public static String consumerGroupRevocationTimeoutKey(String groupId, String memberId) { return "revocation-timeout-" + groupId + "-" + memberId; } + + /** + * Replays GroupMetadataKey/Value to update the soft state of + * the generic group. + * + * @param key A GroupMetadataKey key. + * @param value A GroupMetadataValue record. + */ + public void replay( + GroupMetadataKey key, + GroupMetadataValue value + ) { + String groupId = key.group(); + + if (value == null) { + // Tombstone. Group should be removed. + removeGroup(groupId); + } else { + List loadedMembers = new ArrayList<>(); + for (GroupMetadataValue.MemberMetadata member : value.members()) { + int rebalanceTimeout = member.rebalanceTimeout() == -1 ? + member.sessionTimeout() : member.rebalanceTimeout(); + + JoinGroupRequestProtocolCollection supportedProtocols = new JoinGroupRequestProtocolCollection(); + supportedProtocols.add(new JoinGroupRequestProtocol() + .setName(value.protocol()) + .setMetadata(member.subscription())); + + GenericGroupMember loadedMember = new GenericGroupMember( + member.memberId(), + Optional.ofNullable(member.groupInstanceId()), + member.clientId(), + member.clientHost(), + rebalanceTimeout, + member.sessionTimeout(), + value.protocolType(), + supportedProtocols, + member.assignment() + ); + + loadedMembers.add(loadedMember); + } + + String protocolType = value.protocolType(); + + GenericGroup genericGroup = new GenericGroup( + this.logContext, + groupId, + loadedMembers.isEmpty() ? EMPTY : STABLE, + time, + value.generation(), + protocolType == null || protocolType.isEmpty() ? Optional.empty() : Optional.of(protocolType), + Optional.ofNullable(value.protocol()), + Optional.ofNullable(value.leader()), + value.currentStateTimestamp() == -1 ? Optional.empty() : Optional.of(value.currentStateTimestamp()) + ); + + loadedMembers.forEach(member -> genericGroup.add(member, null)); + groups.put(groupId, genericGroup); + + genericGroup.setSubscribedTopics( + genericGroup.computeSubscribedTopics() + ); + } + } + + /** + * Handle a JoinGroupRequest. + * + * @param context The request context. + * @param request The actual JoinGroup request. + * + * @return The result that contains records to append if the join group phase completes. + */ + public CoordinatorResult genericGroupJoin( + RequestContext context, + JoinGroupRequestData request, + CompletableFuture responseFuture + ) { + CoordinatorResult result = EMPTY_RESULT; + + String groupId = request.groupId(); + String memberId = request.memberId(); + int sessionTimeoutMs = request.sessionTimeoutMs(); + + if (sessionTimeoutMs < genericGroupMinSessionTimeoutMs || + sessionTimeoutMs > genericGroupMaxSessionTimeoutMs + ) { + responseFuture.complete(new JoinGroupResponseData() + .setMemberId(memberId) + .setErrorCode(Errors.INVALID_SESSION_TIMEOUT.code()) + ); + } else { + boolean isUnknownMember = memberId.equals(UNKNOWN_MEMBER_ID); + // Group is created if it does not exist and the member id is UNKNOWN. if member + // is specified but group does not exist, request is rejected with GROUP_ID_NOT_FOUND + GenericGroup group; + boolean isNewGroup = !groups.containsKey(groupId); + try { + group = getOrMaybeCreateGenericGroup(groupId, isUnknownMember); + } catch (Throwable t) { + responseFuture.complete(new JoinGroupResponseData() + .setMemberId(memberId) + .setErrorCode(Errors.forException(t).code()) + ); + return EMPTY_RESULT; + } + + if (!acceptJoiningMember(group, memberId)) { + group.remove(memberId); + responseFuture.complete(new JoinGroupResponseData() + .setMemberId(UNKNOWN_MEMBER_ID) + .setErrorCode(Errors.GROUP_MAX_SIZE_REACHED.code()) + ); + } else if (isUnknownMember) { + result = genericGroupJoinNewMember( + context, + request, + group, + responseFuture + ); + } else { + result = genericGroupJoinExistingMember( + context, + request, + group, + responseFuture + ); + } + + if (isNewGroup && result == EMPTY_RESULT) { + // If there are no records to append and if a group was newly created, we need to append + // records to the log to commit the group to the timeline data structure. + CompletableFuture appendFuture = new CompletableFuture<>(); + appendFuture.whenComplete((__, t) -> { + if (t != null) { + // We failed to write the empty group metadata. This will revert the snapshot, removing + // the newly created group. + log.warn("Failed to write empty metadata for group {}: {}", group.groupId(), t.getMessage()); + + responseFuture.complete(new JoinGroupResponseData() + .setErrorCode(appendGroupMetadataErrorToResponseError(Errors.forException(t)).code())); + } + }); + + List records = Collections.singletonList( + RecordHelpers.newEmptyGroupMetadataRecord(group, metadataImage.features().metadataVersion()) + ); + + return new CoordinatorResult<>(records, appendFuture); + } + } + return result; + } + + /** + * Attempt to complete join group phase. We do not complete + * the join group phase if this is the initial rebalance. + * + * @param group The group. + * + * @return The coordinator result that will be appended to the log. + */ + private CoordinatorResult maybeCompleteJoinPhase(GenericGroup group) { + if (group.isInState(PREPARING_REBALANCE) && + group.hasAllMembersJoined() && + group.previousState() != EMPTY + ) { + return completeGenericGroupJoin(group); + } + + return EMPTY_RESULT; + } + + /** + * Handle a new member generic group join. + * + * @param context The request context. + * @param request The join group request. + * @param group The group to add the member. + * @param responseFuture The response future to complete. + * + * @return The coordinator result that will be appended to the log. + */ + private CoordinatorResult genericGroupJoinNewMember( + RequestContext context, + JoinGroupRequestData request, + GenericGroup group, + CompletableFuture responseFuture + ) { + if (group.isInState(DEAD)) { + // If the group is marked as dead, it means some other thread has just removed the group + // from the coordinator metadata; it is likely that the group has migrated to some other + // coordinator OR the group is in a transient unstable phase. Let the member retry + // finding the correct coordinator and rejoin. + responseFuture.complete(new JoinGroupResponseData() + .setMemberId(UNKNOWN_MEMBER_ID) + .setErrorCode(Errors.COORDINATOR_NOT_AVAILABLE.code()) + ); + } else if (!group.supportsProtocols(request.protocolType(), request.protocols())) { + responseFuture.complete(new JoinGroupResponseData() + .setMemberId(UNKNOWN_MEMBER_ID) + .setErrorCode(Errors.INCONSISTENT_GROUP_PROTOCOL.code()) + ); + } else { + Optional groupInstanceId = Optional.ofNullable(request.groupInstanceId()); + String newMemberId = group.generateMemberId(context.clientId(), groupInstanceId); + + if (groupInstanceId.isPresent()) { + return genericGroupJoinNewStaticMember( + context, + request, + group, + newMemberId, + responseFuture + ); + } else { + return genericGroupJoinNewDynamicMember( + context, + request, + group, + newMemberId, + responseFuture + ); + } + } + + return EMPTY_RESULT; + } + + /** + * Handle new static member join. If there was an existing member id for the group instance id, + * replace that member. Otherwise, add the member and rebalance. + * + * @param context The request context. + * @param request The join group request. + * @param group The group to add the member. + * @param newMemberId The newly generated member id. + * @param responseFuture The response future to complete. + * + * @return The coordinator result that will be appended to the log. + */ + private CoordinatorResult genericGroupJoinNewStaticMember( + RequestContext context, + JoinGroupRequestData request, + GenericGroup group, + String newMemberId, + CompletableFuture responseFuture + ) { + String groupInstanceId = request.groupInstanceId(); + String existingMemberId = group.staticMemberId(groupInstanceId); + if (existingMemberId != null) { + log.info("Static member with groupInstanceId={} and unknown member id joins " + + "group {} in {} state. Replacing previously mapped member {} with this groupInstanceId.", + groupInstanceId, group.groupId(), group.currentState(), existingMemberId); + + return updateStaticMemberAndRebalance( + context, + request, + group, + existingMemberId, + newMemberId, + responseFuture + ); + } else { + log.info("Static member with groupInstanceId={} and unknown member id joins " + + "group {} in {} state. Created a new member id {} for this member and added to the group.", + groupInstanceId, group.groupId(), group.currentState(), newMemberId); + + return addMemberAndRebalance(context, request, group, newMemberId, responseFuture); + } + } + + /** + * Handle a new dynamic member join. If the member id field is required, the group metadata manager + * will add the new member id to the pending members and respond with MEMBER_ID_REQUIRED along with + * the new member id for the client to join with. + * + * Otherwise, add the new member to the group and rebalance. + * + * @param context The request context. + * @param request The join group request. + * @param group The group to add the member. + * @param newMemberId The newly generated member id. + * @param responseFuture The response future to complete. + * + * @return The coordinator result that will be appended to the log. + */ + private CoordinatorResult genericGroupJoinNewDynamicMember( + RequestContext context, + JoinGroupRequestData request, + GenericGroup group, + String newMemberId, + CompletableFuture responseFuture + ) { + if (JoinGroupRequest.requiresKnownMemberId(context.apiVersion())) { + // If member id required, register the member in the pending member list and send + // back a response to call for another join group request with allocated member id. + log.info("Dynamic member with unknown member id joins group {} in {} state. " + + "Created a new member id {} and requesting the member to rejoin with this id.", + group.groupId(), group.currentState(), newMemberId); + + group.addPendingMember(newMemberId); + String genericGroupHeartbeatKey = genericGroupHeartbeatKey(group.groupId(), newMemberId); + + timer.schedule( + genericGroupHeartbeatKey, + request.sessionTimeoutMs(), + TimeUnit.MILLISECONDS, + false, + () -> expireGenericGroupMemberHeartbeat(group, newMemberId) + ); + + responseFuture.complete(new JoinGroupResponseData() + .setMemberId(newMemberId) + .setErrorCode(Errors.MEMBER_ID_REQUIRED.code()) + ); + } else { + log.info("Dynamic member with unknown member id joins group {} in state {}. " + + "Created a new member id {} and added the member to the group.", + group.groupId(), group.currentState(), newMemberId); + + return addMemberAndRebalance(context, request, group, newMemberId, responseFuture); + } + + return EMPTY_RESULT; + } + + /** + * Handle a join group request for an existing member. + * + * @param context The request context. + * @param request The join group request. + * @param group The group to add the member. + * @param responseFuture The response future to complete. + * + * @return The coordinator result that will be appended to the log. + */ + private CoordinatorResult genericGroupJoinExistingMember( + RequestContext context, + JoinGroupRequestData request, + GenericGroup group, + CompletableFuture responseFuture + ) { + String memberId = request.memberId(); + String groupInstanceId = request.groupInstanceId(); + + if (group.isInState(DEAD)) { + // If the group is marked as dead, it means the group was recently removed the group + // from the coordinator metadata; it is likely that the group has migrated to some other + // coordinator OR the group is in a transient unstable phase. Let the member retry + // finding the correct coordinator and rejoin. + responseFuture.complete(new JoinGroupResponseData() + .setMemberId(memberId) + .setErrorCode(Errors.COORDINATOR_NOT_AVAILABLE.code()) + ); + } else if (!group.supportsProtocols(request.protocolType(), request.protocols())) { + responseFuture.complete(new JoinGroupResponseData() + .setMemberId(memberId) + .setErrorCode(Errors.INCONSISTENT_GROUP_PROTOCOL.code()) + ); + } else if (group.isPendingMember(memberId)) { + // A rejoining pending member will be accepted. Note that pending member cannot be a static member. + if (groupInstanceId != null) { + throw new IllegalStateException("Received unexpected JoinGroup with groupInstanceId=" + + groupInstanceId + " for pending member with memberId=" + memberId); + } + + log.debug("Pending dynamic member with id {} joins group {} in {} state. Adding to the group now.", + memberId, group.groupId(), group.currentState()); + + return addMemberAndRebalance( + context, + request, + group, + memberId, + responseFuture + ); + } else { + Optional memberError = validateExistingMember( + group, + memberId, + groupInstanceId, + "join-group" + ); + + if (memberError.isPresent()) { + responseFuture.complete(new JoinGroupResponseData() + .setMemberId(memberId) + .setErrorCode(memberError.get().code()) + ); + } else { + GenericGroupMember member = group.member(memberId); + if (group.isInState(PREPARING_REBALANCE)) { + return updateMemberThenRebalanceOrCompleteJoin( + request, + group, + member, + "Member " + member.memberId() + " is joining group during " + group.stateAsString() + + "; client reason: " + JoinGroupRequest.joinReason(request), + responseFuture + ); + } else if (group.isInState(COMPLETING_REBALANCE)) { + if (member.matches(request.protocols())) { + // Member is joining with the same metadata (which could be because it failed to + // receive the initial JoinGroup response), so just return current group information + // for the current generation. + responseFuture.complete(new JoinGroupResponseData() + .setMembers(group.isLeader(memberId) ? + group.currentGenericGroupMembers() : Collections.emptyList()) + .setMemberId(memberId) + .setGenerationId(group.generationId()) + .setProtocolName(group.protocolName().orElse(null)) + .setProtocolType(group.protocolType().orElse(null)) + .setLeader(group.leaderOrNull()) + .setSkipAssignment(false) + ); + } else { + // Member has changed metadata, so force a rebalance + return updateMemberThenRebalanceOrCompleteJoin( + request, + group, + member, + "Updating metadata for member " + memberId + " during " + group.stateAsString() + + "; client reason: " + JoinGroupRequest.joinReason(request), + responseFuture + ); + } + } else if (group.isInState(STABLE)) { + if (group.isLeader(memberId)) { + // Force a rebalance if the leader sends JoinGroup; + // This allows the leader to trigger rebalances for changes affecting assignment + // which do not affect the member metadata (such as topic metadata changes for the consumer) + return updateMemberThenRebalanceOrCompleteJoin( + request, + group, + member, + "Leader " + memberId + " re-joining group during " + group.stateAsString() + + "; client reason: " + JoinGroupRequest.joinReason(request), + responseFuture + ); + } else if (!member.matches(request.protocols())) { + return updateMemberThenRebalanceOrCompleteJoin( + request, + group, + member, + "Updating metadata for member " + memberId + " during " + group.stateAsString() + + "; client reason: " + JoinGroupRequest.joinReason(request), + responseFuture + ); + } else { + // For followers with no actual change to their metadata, just return group information + // for the current generation which will allow them to issue SyncGroup. + responseFuture.complete(new JoinGroupResponseData() + .setMembers(Collections.emptyList()) + .setMemberId(memberId) + .setGenerationId(group.generationId()) + .setProtocolName(group.protocolName().orElse(null)) + .setProtocolType(group.protocolType().orElse(null)) + .setLeader(group.leaderOrNull()) + .setSkipAssignment(false) + ); + } + } else { + // Group reached unexpected (Empty) state. Let the joining member reset their generation and rejoin. + log.warn("Attempt to add rejoining member {} of group {} in unexpected group state {}", + memberId, group.groupId(), group.stateAsString()); + + responseFuture.complete(new JoinGroupResponseData() + .setMemberId(memberId) + .setErrorCode(Errors.UNKNOWN_MEMBER_ID.code()) + ); + } + } + } + + return EMPTY_RESULT; + } + + /** + * Complete the join group phase. Remove all dynamic members that have not rejoined + * during this stage and proceed with the next generation for this group. The generation id + * is incremented and the group transitions to CompletingRebalance state if there is at least + * one member. + * + * If the group is in Empty state, append a new group metadata record to the log. Otherwise, + * complete all members' join group response futures and wait for sync requests from members. + * + * @param group The group that is completing the join group phase. + * + * @return The coordinator result that will be appended to the log. + */ + private CoordinatorResult completeGenericGroupJoin( + GenericGroup group + ) { + timer.cancel(genericGroupJoinKey(group.groupId())); + String groupId = group.groupId(); + + Map notYetRejoinedDynamicMembers = + group.notYetRejoinedMembers().entrySet().stream() + .filter(entry -> !entry.getValue().isStaticMember()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + if (!notYetRejoinedDynamicMembers.isEmpty()) { + notYetRejoinedDynamicMembers.values().forEach(failedMember -> { + group.remove(failedMember.memberId()); + timer.cancel(genericGroupHeartbeatKey(group.groupId(), failedMember.memberId())); + }); + + log.info("Group {} removed dynamic members who haven't joined: {}", + groupId, notYetRejoinedDynamicMembers.keySet()); + } + + if (group.isInState(DEAD)) { + log.info("Group {} is dead, skipping rebalance stage.", groupId); + } else if (!group.maybeElectNewJoinedLeader() && !group.allMembers().isEmpty()) { + // If all members are not rejoining, we will postpone the completion + // of rebalance preparing stage, and send out another delayed operation + // until session timeout removes all the non-responsive members. + log.error("Group {} could not complete rebalance because no members rejoined.", groupId); + + timer.schedule( + genericGroupJoinKey(groupId), + group.rebalanceTimeoutMs(), + TimeUnit.MILLISECONDS, + false, + () -> completeGenericGroupJoin(group) + ); + + return EMPTY_RESULT; + } else { + group.initNextGeneration(); + if (group.isInState(EMPTY)) { + log.info("Group {} with generation {} is now empty ({}-{})", + groupId, group.generationId(), topicPartition.topic(), topicPartition.partition()); + + CompletableFuture appendFuture = new CompletableFuture<>(); + appendFuture.whenComplete((__, t) -> { + if (t != null) { + // We failed to write the empty group metadata. If the broker fails before another rebalance, + // the previous generation written to the log will become active again (and most likely timeout). + // This should be safe since there are no active members in an empty generation, so we just warn. + log.warn("Failed to write empty metadata for group {}: {}", group.groupId(), t.getMessage()); + } + }); + + List records = Collections.singletonList( + RecordHelpers.newGroupMetadataRecord(group, metadataImage.features().metadataVersion()) + ); + + return new CoordinatorResult<>(records, appendFuture); + + } else { + log.info("Stabilized group {} generation {} ({}) with {} members", + groupId, group.generationId(), topicPartition, group.size()); + + // Complete the awaiting join group response future for all the members after rebalancing + group.allMembers().forEach(member -> { + List members = Collections.emptyList(); + if (group.isLeader(member.memberId())) { + members = group.currentGenericGroupMembers(); + } + + JoinGroupResponseData response = new JoinGroupResponseData() + .setMembers(members) + .setMemberId(member.memberId()) + .setGenerationId(group.generationId()) + .setProtocolName(group.protocolName().orElse(null)) + .setProtocolType(group.protocolType().orElse(null)) + .setLeader(group.leaderOrNull()) + .setSkipAssignment(false) + .setErrorCode(Errors.NONE.code()); + + group.completeJoinFuture(member, response); + rescheduleGenericGroupMemberHeartbeat(group, member); + member.setIsNew(false); + + group.addPendingSyncMember(member.memberId()); + }); + + schedulePendingSync(group); + } + } + + return EMPTY_RESULT; + } + + /** + * Wait for sync requests for the group. + * + * @param group The group. + */ + private void schedulePendingSync(GenericGroup group) { + timer.schedule(genericGroupSyncKey(group.groupId()), + group.rebalanceTimeoutMs(), + TimeUnit.MILLISECONDS, + false, + () -> expirePendingSync(group, group.generationId())); + } + + /** + * Invoked when the heartbeat operation is expired from the timer. Possibly remove the member and + * try complete the join phase. + * + * @param group The group. + * @param memberId The member id. + * + * @return The coordinator result that will be appended to the log. + */ + private CoordinatorResult expireGenericGroupMemberHeartbeat( + GenericGroup group, + String memberId + ) { + if (group.isInState(DEAD)) { + log.info("Received notification of heartbeat expiration for member {} after group {} " + + "had already been unloaded or deleted.", + memberId, group.groupId()); + } else if (group.isPendingMember(memberId)) { + log.info("Pending member {} in group {} has been removed after session timeout expiration.", + memberId, group.groupId()); + + return removePendingMemberAndUpdateGenericGroup(group, memberId); + } else if (!group.hasMemberId(memberId)) { + log.debug("Member {} has already been removed from the group.", memberId); + } else { + GenericGroupMember member = group.member(memberId); + if (!member.hasSatisfiedHeartbeat()) { + log.info("Member {} in group {} has failed, removing it from the group.", + member.memberId(), group.groupId()); + + return removeMemberAndUpdateGenericGroup( + group, + member, + "removing member " + member.memberId() + " on heartbeat expiration." + ); + } + } + return EMPTY_RESULT; + } + + /** + * Invoked when the heartbeat key is expired from the timer. Possibly remove the member + * from the group and try to complete the join phase. + * + * @param group The group. + * @param member The member. + * @param reason The reason for removing the member. + * + * @return The coordinator result that will be appended to the log. + */ + private CoordinatorResult removeMemberAndUpdateGenericGroup( + GenericGroup group, + GenericGroupMember member, + String reason + ) { + // New members may timeout with a pending JoinGroup while the group is still rebalancing, so we have + // to invoke the response future before removing the member. We return UNKNOWN_MEMBER_ID so + // that the consumer will retry the JoinGroup request if it is still active. + group.completeJoinFuture(member, new JoinGroupResponseData() + .setMemberId(UNKNOWN_MEMBER_ID) + .setErrorCode(Errors.UNKNOWN_MEMBER_ID.code()) + ); + group.remove(member.memberId()); + + if (group.isInState(STABLE) || group.isInState(COMPLETING_REBALANCE)) { + return maybePrepareRebalanceOrCompleteJoin(group, reason); + } else if (group.isInState(PREPARING_REBALANCE) && group.hasAllMembersJoined()) { + return completeGenericGroupJoin(group); + } + + return EMPTY_RESULT; + } + + /** + * Remove a pending member from the group and possibly complete the join phase. + * + * @param group The group. + * @param memberId The member id. + * + * @return The coordinator result that will be appended to the log. + */ + private CoordinatorResult removePendingMemberAndUpdateGenericGroup( + GenericGroup group, + String memberId + ) { + group.remove(memberId); + + if (group.isInState(PREPARING_REBALANCE) && group.hasAllMembersJoined()) { + return completeGenericGroupJoin(group); + } + + return EMPTY_RESULT; + } + + /** + * Update an existing member. Then begin a rebalance or complete the join phase. + * + * @param request The join group request. + * @param group The group to add the member. + * @param member The member. + * @param joinReason The client reason for the join request. + * @param responseFuture The response future to complete. + * + * @return The coordinator result that will be appended to the log. + */ + private CoordinatorResult updateMemberThenRebalanceOrCompleteJoin( + JoinGroupRequestData request, + GenericGroup group, + GenericGroupMember member, + String joinReason, + CompletableFuture responseFuture + ) { + group.updateMember( + member, + request.protocols(), + request.rebalanceTimeoutMs(), + request.sessionTimeoutMs(), + responseFuture + ); + + return maybePrepareRebalanceOrCompleteJoin(group, joinReason); + } + + /** + * We are validating two things: + * 1. If `groupInstanceId` is present, then it exists and is mapped to `memberId` + * 2. The `memberId` exists in the group + * + * @param group The generic group. + * @param memberId The member id. + * @param groupInstanceId The group instance id. + * @param operation The API operation. + * + * @return the error. + */ + private Optional validateExistingMember( + GenericGroup group, + String memberId, + String groupInstanceId, + String operation + ) { + if (groupInstanceId == null) { + if (!group.hasMemberId(memberId)) { + return Optional.of(Errors.UNKNOWN_MEMBER_ID); + } else { + return Optional.empty(); + } + } + + String existingMemberId = group.staticMemberId(groupInstanceId); + if (existingMemberId == null) { + return Optional.of(Errors.UNKNOWN_MEMBER_ID); + } + + if (!existingMemberId.equals(memberId)) { + log.info("Request memberId={} for static member with groupInstanceId={} " + + "is fenced by existing memberId={} during operation {}", + memberId, groupInstanceId, existingMemberId, operation); + + return Optional.of(Errors.FENCED_INSTANCE_ID); + } + + return Optional.empty(); + } + + /** + * Add a member and rebalance. + * + * @param context The request context. + * @param request The join group request. + * @param group The group to add the member. + * @param memberId The member id. + * @param responseFuture The response future to complete. + * + * @return The coordinator result that will be appended to the log. + */ + private CoordinatorResult addMemberAndRebalance( + RequestContext context, + JoinGroupRequestData request, + GenericGroup group, + String memberId, + CompletableFuture responseFuture + ) { + Optional groupInstanceId = Optional.ofNullable(request.groupInstanceId()); + GenericGroupMember member = new GenericGroupMember( + memberId, + groupInstanceId, + context.clientId(), + context.clientAddress().toString(), + request.rebalanceTimeoutMs(), + request.sessionTimeoutMs(), + request.protocolType(), + request.protocols() + ); + + member.setIsNew(true); + + // Update the newMemberAdded flag to indicate that the initial rebalance can be further delayed + if (group.isInState(PREPARING_REBALANCE) && group.previousState() == EMPTY) { + group.setNewMemberAdded(true); + } + + group.add(member, responseFuture); + + // The session timeout does not affect new members since they do not have their memberId and + // cannot send heartbeats. Furthermore, we cannot detect disconnects because sockets are muted + // while the JoinGroup request is parked. If the client does disconnect (e.g. because of a request + // timeout during a long rebalance), they may simply retry which will lead to a lot of defunct + // members in the rebalance. To prevent this going on indefinitely, we time out JoinGroup requests + // for new members. If the new member is still there, we expect it to retry. + rescheduleGenericGroupMemberHeartbeat(group, member, genericGroupNewMemberJoinTimeoutMs); + + return maybePrepareRebalanceOrCompleteJoin(group, "Adding new member " + memberId + " with group instance id " + + request.groupInstanceId() + "; client reason: " + JoinGroupRequest.joinReason(request)); + } + + /** + * Prepare a rebalance if the group is in a valid state. Otherwise, try + * to complete the join phase. + * + * @param group The group to rebalance. + * @param reason The reason for the rebalance. + * + * @return The coordinator result that will be appended to the log. + */ + private CoordinatorResult maybePrepareRebalanceOrCompleteJoin( + GenericGroup group, + String reason + ) { + if (group.canRebalance()) { + return prepareRebalance(group, reason); + } else { + return maybeCompleteJoinPhase(group); + } + } + + /** + * Prepare a rebalance. + * + * @param group The group to rebalance. + * @param reason The reason for the rebalance. + * + * @return The coordinator result that will be appended to the log. + * + * Package private for testing. + */ + CoordinatorResult prepareRebalance( + GenericGroup group, + String reason + ) { + // If any members are awaiting sync, cancel their request and have them rejoin. + if (group.isInState(COMPLETING_REBALANCE)) { + resetAndPropagateAssignmentWithError(group, Errors.REBALANCE_IN_PROGRESS); + } + + // If a sync expiration is pending, cancel it. + removeSyncExpiration(group); + + boolean isInitialRebalance = group.isInState(EMPTY); + if (isInitialRebalance) { + // The group is new. Provide more time for the members to join. + int delayMs = genericGroupInitialRebalanceDelayMs; + int remainingMs = Math.max(group.rebalanceTimeoutMs() - genericGroupInitialRebalanceDelayMs, 0); + + timer.schedule( + genericGroupJoinKey(group.groupId()), + delayMs, + TimeUnit.MILLISECONDS, + false, + () -> tryCompleteInitialRebalanceElseSchedule(group, delayMs, remainingMs) + ); + } + + group.transitionTo(PREPARING_REBALANCE); + + log.info("Preparing to rebalance group {} in state {} with old generation {} ({}-{}) (reason: {})", + group.groupId(), group.currentState(), group.generationId(), + topicPartition.topic(), topicPartition.partition(), reason); + + return isInitialRebalance ? EMPTY_RESULT : maybeCompleteJoinElseSchedule(group); + } + + /** + * Try to complete the join phase. Otherwise, schedule a new join operation. + * + * @param group The group. + * + * @return The coordinator result that will be appended to the log. + */ + private CoordinatorResult maybeCompleteJoinElseSchedule( + GenericGroup group + ) { + String genericGroupJoinKey = genericGroupJoinKey(group.groupId()); + if (group.hasAllMembersJoined()) { + // All members have joined. Proceed to sync phase. + return completeGenericGroupJoin(group); + } else { + timer.schedule( + genericGroupJoinKey, + group.rebalanceTimeoutMs(), + TimeUnit.MILLISECONDS, + false, + () -> completeGenericGroupJoin(group) + ); + return EMPTY_RESULT; + } + } + + /** + * Try to complete the join phase of the initial rebalance. + * Otherwise, extend the rebalance. + * + * @param group The group under initial rebalance. + * + * @return The coordinator result that will be appended to the log. + */ + private CoordinatorResult tryCompleteInitialRebalanceElseSchedule( + GenericGroup group, + int delayMs, + int remainingMs + ) { + if (group.newMemberAdded() && remainingMs != 0) { + // A new member was added. Extend the delay. + group.setNewMemberAdded(false); + int newDelayMs = Math.min(genericGroupInitialRebalanceDelayMs, remainingMs); + int newRemainingMs = Math.max(remainingMs - delayMs, 0); + + timer.schedule( + genericGroupJoinKey(group.groupId()), + newDelayMs, + TimeUnit.MILLISECONDS, + false, + () -> tryCompleteInitialRebalanceElseSchedule(group, newDelayMs, newRemainingMs) + ); + } else { + // No more time remaining. Complete the join phase. + return completeGenericGroupJoin(group); + } + + return EMPTY_RESULT; + } + + /** + * Reset assignment for all members and propagate the error to all members in the group. + * + * @param group The group. + * @param error The error to propagate. + */ + private void resetAndPropagateAssignmentWithError(GenericGroup group, Errors error) { + if (!group.isInState(COMPLETING_REBALANCE)) { + throw new IllegalStateException("Group " + group.groupId() + " must be in " + COMPLETING_REBALANCE.name() + + " state but is in " + group.currentState() + "."); + } + + group.allMembers().forEach(member -> member.setAssignment(GenericGroupMember.EMPTY_ASSIGNMENT)); + propagateAssignment(group, error); + } + + /** + * Propagate assignment and error to all members. + * + * @param group The group. + * @param error The error to propagate. + */ + private void propagateAssignment(GenericGroup group, Errors error) { + Optional protocolName = Optional.empty(); + Optional protocolType = Optional.empty(); + if (error == Errors.NONE) { + protocolName = group.protocolName(); + protocolType = group.protocolType(); + } + + for (GenericGroupMember member : group.allMembers()) { + if (!member.hasAssignment() && error == Errors.NONE) { + log.warn("Sending empty assignment to member {} of {} for " + "generation {} with no errors", + member.memberId(), group.groupId(), group.generationId()); + } + + if (group.completeSyncFuture(member, + new SyncGroupResponseData() + .setProtocolName(protocolName.orElse(null)) + .setProtocolType(protocolType.orElse(null)) + .setAssignment(member.assignment()) + .setErrorCode(error.code()))) { + + // Reset the session timeout for members after propagating the member's assignment. + // This is because if any member's session expired while we were still awaiting either + // the leader sync group or the append future, its expiration will be ignored and no + // future heartbeat expectations will not be scheduled. + rescheduleGenericGroupMemberHeartbeat(group, member); + } + } + } + + /** + * Complete and schedule next heartbeat. + * + * @param group The group. + * @param member The member. + */ + private void rescheduleGenericGroupMemberHeartbeat( + GenericGroup group, + GenericGroupMember member + ) { + rescheduleGenericGroupMemberHeartbeat(group, member, member.sessionTimeoutMs()); + } + + /** + * Reschedule the heartbeat. + * + * @param group The group. + * @param member The member. + * @param timeoutMs The timeout for the new heartbeat. + */ + private void rescheduleGenericGroupMemberHeartbeat( + GenericGroup group, + GenericGroupMember member, + long timeoutMs + ) { + String genericGroupHeartbeatKey = genericGroupHeartbeatKey(group.groupId(), member.memberId()); + + // Reschedule the next heartbeat expiration deadline + timer.schedule(genericGroupHeartbeatKey, + timeoutMs, + TimeUnit.MILLISECONDS, + false, + () -> expireGenericGroupMemberHeartbeat(group, member.memberId())); + } + + /** + * Remove the sync key from the timer and clear all pending sync members from the group. + * Invoked when a new rebalance is triggered. + * + * @param group The group. + */ + private void removeSyncExpiration(GenericGroup group) { + group.clearPendingSyncMembers(); + timer.cancel(genericGroupSyncKey(group.groupId())); + } + + /** + * Expire pending sync. + * + * @param group The group. + * @param generationId The generation when the pending sync was originally scheduled. + * + * @return The coordinator result that will be appended to the log. + * */ + private CoordinatorResult expirePendingSync( + GenericGroup group, + int generationId + ) { + if (generationId != group.generationId()) { + log.error("Received unexpected notification of sync expiration for {} with an old " + + "generation {} while the group has {}.", group.groupId(), generationId, group.generationId()); + } else { + if (group.isInState(DEAD) || group.isInState(EMPTY) || group.isInState(PREPARING_REBALANCE)) { + log.error("Received unexpected notification of sync expiration after group {} already " + + "transitioned to {} state.", group.groupId(), group.stateAsString()); + } else if (group.isInState(COMPLETING_REBALANCE) || group.isInState(STABLE)) { + if (!group.hasReceivedSyncFromAllMembers()) { + Set pendingSyncMembers = new HashSet<>(group.allPendingSyncMembers()); + pendingSyncMembers.forEach(memberId -> { + group.remove(memberId); + timer.cancel(genericGroupHeartbeatKey(group.groupId(), memberId)); + }); + + log.debug("Group {} removed members who haven't sent their sync requests: {}", + group.groupId(), pendingSyncMembers); + + return prepareRebalance(group, "Removing " + pendingSyncMembers + " on pending sync request expiration"); + } + } + } + + return EMPTY_RESULT; + } + + /** + * Checks whether the group can accept a joining member. + * + * @param group The group. + * @param memberId The member. + * + * @return whether the group can accept a joining member. + */ + private boolean acceptJoiningMember(GenericGroup group, String memberId) { + switch (group.currentState()) { + case EMPTY: + case DEAD: + // Always accept the request when the group is empty or dead + return true; + case PREPARING_REBALANCE: + // An existing member is accepted if it is already awaiting. New members are accepted + // up to the max group size. Note that the number of awaiting members is used here + // for two reasons: + // 1) the group size is not reliable as it could already be above the max group size + // if the max group size was reduced. + // 2) using the number of awaiting members allows to kick out the last rejoining + // members of the group. + return (group.hasMemberId(memberId) && group.member(memberId).isAwaitingJoin()) || + group.numAwaitingJoinResponse() < genericGroupMaxSize; + case COMPLETING_REBALANCE: + case STABLE: + // An existing member is accepted. New members are accepted up to the max group size. + // Note that the group size is used here. When the group transitions to CompletingRebalance, + // members who haven't rejoined are removed. + return group.hasMemberId(memberId) || group.size() < genericGroupMaxSize; + default: + throw new IllegalStateException("Unknown group state: " + group.stateAsString()); + } + } + + /** + * Update a static member and rebalance. + * + * @param context The request context. + * @param request The join group request. + * @param group The group of the static member. + * @param oldMemberId The existing static member id. + * @param newMemberId The new joining static member id. + * @param responseFuture The response future to complete. + * + * @return The coordinator result that will be appended to the log. + */ + private CoordinatorResult updateStaticMemberAndRebalance( + RequestContext context, + JoinGroupRequestData request, + GenericGroup group, + String oldMemberId, + String newMemberId, + CompletableFuture responseFuture + ) { + String currentLeader = group.leaderOrNull(); + GenericGroupMember newMember = group.replaceStaticMember(request.groupInstanceId(), oldMemberId, newMemberId); + + // Heartbeat of old member id will expire without effect since the group no longer contains that member id. + // New heartbeat shall be scheduled with new member id. + rescheduleGenericGroupMemberHeartbeat(group, newMember); + + int oldRebalanceTimeoutMs = newMember.rebalanceTimeoutMs(); + int oldSessionTimeoutMs = newMember.sessionTimeoutMs(); + JoinGroupRequestProtocolCollection oldProtocols = newMember.supportedProtocols(); + + group.updateMember( + newMember, + request.protocols(), + request.rebalanceTimeoutMs(), + request.sessionTimeoutMs(), + responseFuture + ); + + if (group.isInState(STABLE)) { + // Check if group's selected protocol of next generation will change, if not, simply store group to persist + // the updated static member, if yes, rebalance should be triggered to keep the group's assignment + // and selected protocol consistent + String groupInstanceId = request.groupInstanceId(); + String selectedProtocolForNextGeneration = group.selectProtocol(); + if (group.protocolName().orElse("").equals(selectedProtocolForNextGeneration)) { + log.info("Static member which joins during Stable stage and doesn't affect " + + "the selected protocol will not trigger a rebalance."); + + CompletableFuture appendFuture = new CompletableFuture<>(); + appendFuture.whenComplete((__, t) -> { + if (t != null) { + log.warn("Failed to persist metadata for group {} static member {} with " + + "group instance id {} due to {}. Reverting to old member id {}.", + group.groupId(), newMemberId, groupInstanceId, t.getMessage(), oldMemberId); + + // Failed to persist the member id of the given static member, revert the update of the static member in the group. + group.updateMember(newMember, oldProtocols, oldRebalanceTimeoutMs, oldSessionTimeoutMs, null); + GenericGroupMember oldMember = group.replaceStaticMember(groupInstanceId, newMemberId, oldMemberId); + rescheduleGenericGroupMemberHeartbeat(group, oldMember); + + responseFuture.complete( + new JoinGroupResponseData() + .setMembers(Collections.emptyList()) + .setMemberId(UNKNOWN_MEMBER_ID) + .setGenerationId(group.generationId()) + .setProtocolName(group.protocolName().orElse(null)) + .setProtocolType(group.protocolType().orElse(null)) + .setLeader(currentLeader) + .setSkipAssignment(false) + .setErrorCode(appendGroupMetadataErrorToResponseError(Errors.forException(t)).code())); + + } else if (JoinGroupRequest.supportsSkippingAssignment(context.apiVersion())) { + boolean isLeader = group.isLeader(newMemberId); + + group.completeJoinFuture(newMember, new JoinGroupResponseData() + .setMembers(isLeader ? group.currentGenericGroupMembers() : Collections.emptyList()) + .setMemberId(newMemberId) + .setGenerationId(group.generationId()) + .setProtocolName(group.protocolName().orElse(null)) + .setProtocolType(group.protocolType().orElse(null)) + .setLeader(group.leaderOrNull()) + .setSkipAssignment(isLeader) + ); + } else { + group.completeJoinFuture(newMember, new JoinGroupResponseData() + .setMembers(Collections.emptyList()) + .setMemberId(newMemberId) + .setGenerationId(group.generationId()) + .setProtocolName(group.protocolName().orElse(null)) + .setProtocolType(group.protocolType().orElse(null)) + .setLeader(currentLeader) + .setSkipAssignment(false) + ); + } + }); + + List records = Collections.singletonList( + RecordHelpers.newGroupMetadataRecord(group, metadataImage.features().metadataVersion()) + ); + + return new CoordinatorResult<>(records, appendFuture); + } else { + return maybePrepareRebalanceOrCompleteJoin( + group, + "Group's selectedProtocol will change because static member " + + newMember.memberId() + " with instance id " + groupInstanceId + + " joined with change of protocol; client reason: " + JoinGroupRequest.joinReason(request) + ); + } + } else if (group.isInState(COMPLETING_REBALANCE)) { + // if the group is in after-sync stage, upon getting a new join-group of a known static member + // we should still trigger a new rebalance, since the old member may already be sent to the leader + // for assignment, and hence when the assignment gets back there would be a mismatch of the old member id + // with the new replaced member id. As a result the new member id would not get any assignment. + return prepareRebalance( + group, + "Updating metadata for static member " + newMember.memberId() + " with instance id " + + request.groupInstanceId() + "; client reason: " + JoinGroupRequest.joinReason(request) + ); + } else if (group.isInState(EMPTY) || group.isInState(DEAD)) { + throw new IllegalStateException("Group " + group.groupId() + " was not supposed to be in the state " + + group.stateAsString() + " when the unknown static member " + request.groupInstanceId() + " rejoins."); + + } + return EMPTY_RESULT; + } + + // Visible for testing + static Errors appendGroupMetadataErrorToResponseError(Errors appendError) { + switch (appendError) { + case UNKNOWN_TOPIC_OR_PARTITION: + case NOT_ENOUGH_REPLICAS: + return COORDINATOR_NOT_AVAILABLE; + + case NOT_LEADER_OR_FOLLOWER: + case KAFKA_STORAGE_ERROR: + return NOT_COORDINATOR; + + case MESSAGE_TOO_LARGE: + case RECORD_LIST_TOO_LARGE: + case INVALID_FETCH_SIZE: + return UNKNOWN_SERVER_ERROR; + + default: + return appendError; + } + } + + /** + * Generate a heartbeat key for the timer. + * + * Package private for testing. + * + * @param groupId The group id. + * @param memberId The member id. + * + * @return the heartbeat key. + */ + static String genericGroupHeartbeatKey(String groupId, String memberId) { + return "heartbeat-" + groupId + "-" + memberId; + } + + /** + * Generate a join key for the timer. + * + * Package private for testing. + * + * @param groupId The group id. + * + * @return the join key. + */ + static String genericGroupJoinKey(String groupId) { + return "join-" + groupId; + } + + /** + * Generate a sync key for the timer. + * + * Package private for testing. + * + * @param groupId The group id. + * + * @return the sync key. + */ + static String genericGroupSyncKey(String groupId) { + return "sync-" + groupId; + } } diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/RecordHelpers.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/RecordHelpers.java index fc041e3351fef..863be1070a414 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/RecordHelpers.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/RecordHelpers.java @@ -38,6 +38,7 @@ import org.apache.kafka.server.common.MetadataVersion; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -436,6 +437,36 @@ public static Record newGroupMetadataTombstoneRecord( ); } + /** + * Creates an empty GroupMetadata record. + * + * @param group The generic group. + * @param metadataVersion The metadata version. + * @return The record. + */ + public static Record newEmptyGroupMetadataRecord( + GenericGroup group, + MetadataVersion metadataVersion + ) { + return new Record( + new ApiMessageAndVersion( + new GroupMetadataKey() + .setGroup(group.groupId()), + (short) 2 + ), + new ApiMessageAndVersion( + new GroupMetadataValue() + .setProtocol(null) + .setProtocolType("") + .setGeneration(0) + .setLeader(null) + .setCurrentStateTimestamp(group.currentStateTimestampOrDefault()) + .setMembers(Collections.emptyList()), + metadataVersion.groupMetadataValueVersion() + ) + ); + } + private static List toTopicPartitions( Map> topicPartitions ) { diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/ReplicatedGroupCoordinator.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/ReplicatedGroupCoordinator.java index 40fa0061ab9b2..854bed1060b2e 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/ReplicatedGroupCoordinator.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/ReplicatedGroupCoordinator.java @@ -16,8 +16,11 @@ */ package org.apache.kafka.coordinator.group; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; import org.apache.kafka.common.message.ConsumerGroupHeartbeatResponseData; +import org.apache.kafka.common.message.JoinGroupRequestData; +import org.apache.kafka.common.message.JoinGroupResponseData; import org.apache.kafka.common.protocol.ApiMessage; import org.apache.kafka.common.requests.RequestContext; import org.apache.kafka.common.utils.LogContext; @@ -34,6 +37,8 @@ import org.apache.kafka.coordinator.group.generated.ConsumerGroupTargetAssignmentMemberValue; import org.apache.kafka.coordinator.group.generated.ConsumerGroupTargetAssignmentMetadataKey; import org.apache.kafka.coordinator.group.generated.ConsumerGroupTargetAssignmentMetadataValue; +import org.apache.kafka.coordinator.group.generated.GroupMetadataKey; +import org.apache.kafka.coordinator.group.generated.GroupMetadataValue; import org.apache.kafka.coordinator.group.runtime.Coordinator; import org.apache.kafka.coordinator.group.runtime.CoordinatorBuilder; import org.apache.kafka.coordinator.group.runtime.CoordinatorResult; @@ -43,6 +48,8 @@ import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.timeline.SnapshotRegistry; +import java.util.concurrent.CompletableFuture; + /** * The group coordinator replicated state machine that manages the metadata of all generic and * consumer groups. It holds the hard and the soft state of the groups. This class has two kinds @@ -59,8 +66,9 @@ public static class Builder implements CoordinatorBuilder timer; + private CoordinatorTimer timer; public Builder( GroupCoordinatorConfig config @@ -86,7 +94,7 @@ public CoordinatorBuilder withTime( @Override public CoordinatorBuilder withTimer( - CoordinatorTimer timer + CoordinatorTimer timer ) { this.timer = timer; return this; @@ -100,6 +108,14 @@ public CoordinatorBuilder withSnapshotRegist return this; } + @Override + public CoordinatorBuilder withTopicPartition( + TopicPartition topicPartition + ) { + this.topicPartition = topicPartition; + return this; + } + @Override public ReplicatedGroupCoordinator build() { if (logContext == null) logContext = new LogContext(); @@ -111,6 +127,8 @@ public ReplicatedGroupCoordinator build() { throw new IllegalArgumentException("Time must be set."); if (timer == null) throw new IllegalArgumentException("Timer must be set."); + if (topicPartition == null) + throw new IllegalArgumentException("TopicPartition must be set."); return new ReplicatedGroupCoordinator( new GroupMetadataManager.Builder() @@ -121,6 +139,11 @@ public ReplicatedGroupCoordinator build() { .withAssignors(config.consumerGroupAssignors) .withConsumerGroupMaxSize(config.consumerGroupMaxSize) .withConsumerGroupHeartbeatInterval(config.consumerGroupHeartbeatIntervalMs) + .withTopicPartition(topicPartition) + .withGenericGroupInitialRebalanceDelayMs(config.genericGroupInitialRebalanceDelayMs) + .withGenericGroupNewMemberJoinTimeoutMs(config.genericGroupNewMemberJoinTimeoutMs) + .withGenericGroupMinSessionTimeoutMs(config.genericGroupMinSessionTimeoutMs) + .withGenericGroupMaxSessionTimeoutMs(config.genericGroupMaxSessionTimeoutMs) .build() ); } @@ -158,6 +181,18 @@ public CoordinatorResult consumerGro return groupMetadataManager.consumerGroupHeartbeat(context, request); } + public CoordinatorResult genericGroupJoin( + RequestContext context, + JoinGroupRequestData request, + CompletableFuture responseFuture + ) { + return groupMetadataManager.genericGroupJoin( + context, + request, + responseFuture + ); + } + /** * The coordinator has been loaded. This is used to apply any * post loading operations (e.g. registering timers). @@ -204,6 +239,13 @@ public void replay(Record record) throws RuntimeException { ApiMessageAndVersion value = record.value(); switch (key.version()) { + case 2: + groupMetadataManager.replay( + (GroupMetadataKey) key.message(), + (GroupMetadataValue) messageOrNull(value) + ); + break; + case 3: groupMetadataManager.replay( (ConsumerGroupMetadataKey) key.message(), diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/generic/GenericGroup.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/generic/GenericGroup.java index fdd16526de123..c6036374d78c1 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/generic/GenericGroup.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/generic/GenericGroup.java @@ -17,6 +17,7 @@ package org.apache.kafka.coordinator.group.generic; import org.apache.kafka.clients.consumer.internals.ConsumerProtocol; +import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection; import org.apache.kafka.common.message.JoinGroupResponseData; import org.apache.kafka.common.message.ListGroupsResponseData; import org.apache.kafka.common.message.SyncGroupResponseData; @@ -99,6 +100,11 @@ public class GenericGroup implements Group { */ private GenericGroupState state; + /** + * The previous group state. + */ + private GenericGroupState previousState; + /** * The timestamp of when the group transitioned * to its current state. @@ -108,22 +114,22 @@ public class GenericGroup implements Group { /** * The protocol type used for rebalance. */ - private Optional protocolType = Optional.empty(); + private Optional protocolType; /** * The protocol name used for rebalance. */ - private Optional protocolName = Optional.empty(); + private Optional protocolName; /** * The generation id. */ - private int generationId = 0; + private int generationId; /** * The id of the group's leader. */ - private Optional leaderId = Optional.empty(); + private Optional leaderId; /** * The members of the group. @@ -173,13 +179,42 @@ public GenericGroup( String groupId, GenericGroupState initialState, Time time + ) { + this( + logContext, + groupId, + initialState, + time, + 0, + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.of(time.milliseconds()) + ); + } + + public GenericGroup( + LogContext logContext, + String groupId, + GenericGroupState initialState, + Time time, + int generationId, + Optional protocolType, + Optional protocolName, + Optional leaderId, + Optional currentStateTimestamp ) { Objects.requireNonNull(logContext); this.log = logContext.logger(GenericGroup.class); this.groupId = Objects.requireNonNull(groupId); this.state = Objects.requireNonNull(initialState); + this.previousState = DEAD; this.time = Objects.requireNonNull(time); - this.currentStateTimestamp = Optional.of(time.milliseconds()); + this.generationId = generationId; + this.protocolType = protocolType; + this.protocolName = protocolName; + this.leaderId = leaderId; + this.currentStateTimestamp = currentStateTimestamp; } /** @@ -234,7 +269,18 @@ public Optional protocolType() { * @return the current group state. */ public GenericGroupState currentState() { - return state; + return this.state; + } + + public GenericGroupState previousState() { + return this.previousState; + } + + /** + * @return true if a new member was added. + */ + public boolean newMemberAdded() { + return this.newMemberAdded; } /** @@ -298,6 +344,24 @@ public long currentStateTimestampOrDefault() { return currentStateTimestamp.orElse(-1L); } + /** + * Sets newMemberAdded. + * + * @param value the value to set. + */ + public void setNewMemberAdded(boolean value) { + this.newMemberAdded = value; + } + + /** + * Sets subscribedTopics. + * + * @param subscribedTopics the value to set. + */ + public void setSubscribedTopics(Optional> subscribedTopics) { + this.subscribedTopics = subscribedTopics; + } + /** * @return whether the group is using the consumer protocol. */ @@ -434,7 +498,7 @@ public boolean maybeElectNewJoinedLeader() { * @param groupInstanceId the group instance id. * @param oldMemberId the old member id. * @param newMemberId the new member id that will replace the old member id. - * @return the old member. + * @return the member with the new id. */ public GenericGroupMember replaceStaticMember( String groupInstanceId, @@ -484,7 +548,7 @@ public GenericGroupMember replaceStaticMember( } staticMembers.put(groupInstanceId, newMemberId); - return removedMember; + return newMember; } /** @@ -515,7 +579,7 @@ public boolean addPendingMember(String memberId) { /** * @return number of members that are pending join. */ - public int numPending() { + public int numPendingJoinMembers() { return pendingJoinMembers.size(); } @@ -704,6 +768,7 @@ public boolean canRebalance() { */ public void transitionTo(GenericGroupState groupState) { assertValidTransition(groupState); + previousState = state; state = groupState; currentStateTimestamp = Optional.of(time.milliseconds()); } @@ -782,6 +847,7 @@ private Set candidateProtocols() { * protocol can be supported if it is supported by all members. * * @param member the member to check. + * * @return a boolean based on the condition mentioned above. */ public boolean supportsProtocols(GenericGroupMember member) { @@ -797,6 +863,26 @@ public boolean supportsProtocols(GenericGroupMember member) { * * @param memberProtocolType the member protocol type. * @param memberProtocols the set of protocol names. + * + * @return a boolean based on the condition mentioned above. + */ + public boolean supportsProtocols( + String memberProtocolType, + JoinGroupRequestProtocolCollection memberProtocols + ) { + return supportsProtocols( + memberProtocolType, + GenericGroupMember.plainProtocolSet(memberProtocols) + ); + } + + /** + * Checks whether at least one of the given protocols can be supported. A + * protocol can be supported if it is supported by all members. + * + * @param memberProtocolType the member protocol type. + * @param memberProtocols the set of protocol names. + * * @return a boolean based on the condition mentioned above. */ public boolean supportsProtocols(String memberProtocolType, Set memberProtocols) { @@ -838,7 +924,7 @@ public boolean isSubscribedToTopic(String topic) { * * @return the subscribed topics or None based on the condition above. */ - Optional> computeSubscribedTopics() { + public Optional> computeSubscribedTopics() { if (!protocolType.isPresent()) { return Optional.empty(); } @@ -854,6 +940,9 @@ Optional> computeSubscribedTopics() { try { Set allSubscribedTopics = new HashSet<>(); members.values().forEach(member -> { + // The consumer protocol is parsed with V0 which is the based prefix of all versions. + // This way the consumer group manager does not depend on any specific existing or + // future versions of the consumer protocol. VO must prefix all new versions. ByteBuffer buffer = ByteBuffer.wrap(member.metadata(protocolName.get())); ConsumerProtocol.deserializeVersion(buffer); allSubscribedTopics.addAll(new HashSet<>( @@ -882,7 +971,7 @@ Optional> computeSubscribedTopics() { */ public void updateMember( GenericGroupMember member, - List protocols, + JoinGroupRequestProtocolCollection protocols, int rebalanceTimeoutMs, int sessionTimeoutMs, CompletableFuture future diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/generic/GenericGroupMember.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/generic/GenericGroupMember.java index 9af962a6bbd16..9878e2d4afed8 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/generic/GenericGroupMember.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/generic/GenericGroupMember.java @@ -17,14 +17,15 @@ package org.apache.kafka.coordinator.group.generic; +import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocol; +import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection; import org.apache.kafka.common.message.JoinGroupResponseData; import org.apache.kafka.common.message.SyncGroupResponseData; -import java.util.List; +import java.util.HashSet; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; /** * This class encapsulates a generic group member's metadata. @@ -93,7 +94,7 @@ public class GenericGroupMember { /** * The list of supported protocols. */ - private List supportedProtocols; + private JoinGroupRequestProtocolCollection supportedProtocols; /** * The assignment stored by the client assignor. @@ -115,16 +116,6 @@ public class GenericGroupMember { */ private boolean isNew = false; - /** - * This variable is used to track heartbeat completion through the delayed - * heartbeat purgatory. When scheduling a new heartbeat expiration, we set - * this value to `false`. Upon receiving the heartbeat (or any other event - * indicating the liveness of the client), we set it to `true` so that the - * delayed heartbeat can be completed. - */ - private boolean heartbeatSatisfied = false; - - public GenericGroupMember( String memberId, Optional groupInstanceId, @@ -133,7 +124,7 @@ public GenericGroupMember( int rebalanceTimeoutMs, int sessionTimeoutMs, String protocolType, - List supportedProtocols + JoinGroupRequestProtocolCollection supportedProtocols ) { this( memberId, @@ -156,7 +147,7 @@ public GenericGroupMember( int rebalanceTimeoutMs, int sessionTimeoutMs, String protocolType, - List supportedProtocols, + JoinGroupRequestProtocolCollection supportedProtocols, byte[] assignment ) { this.memberId = memberId; @@ -195,16 +186,14 @@ public boolean isAwaitingSync() { * Get the metadata corresponding to the provided protocol. */ public byte[] metadata(String protocolName) { - Optional match = supportedProtocols.stream() - .filter(protocol -> protocol.name().equals(protocolName)) - .findFirst(); - - if (match.isPresent()) { - return match.get().metadata(); - } else { - throw new IllegalArgumentException("Member does not support protocol " + - protocolName); + for (JoinGroupRequestProtocol protocol : supportedProtocols) { + if (protocol.name().equals(protocolName)) { + return protocol.metadata(); + } } + + throw new IllegalArgumentException("Member does not support protocol " + + protocolName); } /** @@ -215,52 +204,63 @@ public byte[] metadata(String protocolName) { */ public boolean hasSatisfiedHeartbeat() { if (isNew) { - // New members can be expired while awaiting join, so we have to check this first - return heartbeatSatisfied; - } else if (isAwaitingJoin() || isAwaitingSync()) { - // Members that are awaiting a rebalance automatically satisfy expected heartbeats - return true; + // New members can be expired even while awaiting join, so we check this first + return false; } else { - // Otherwise, we require the next heartbeat - return heartbeatSatisfied; + // Members that are awaiting a rebalance automatically satisfy expected heartbeats + return isAwaitingJoin() || isAwaitingSync(); } } /** * Compare the given list of protocols with the member's supported protocols. + * * @param protocols list of protocols to match. * @return true if the given list matches the member's list of supported protocols, * false otherwise. */ - public boolean matches(List protocols) { + public boolean matches(JoinGroupRequestProtocolCollection protocols) { return protocols.equals(this.supportedProtocols); } /** * Vote for one of the potential group protocols. This takes into account the protocol preference as * indicated by the order of supported protocols and returns the first one also contained in the set - * @param candidates the protocol names that this member can vote for + * + * @param candidates The protocol names that this member can vote for * @return the first supported protocol that matches one of the candidates */ public String vote(Set candidates) { - Optional match = supportedProtocols.stream() - .filter(protocol -> candidates.contains(protocol.name())) - .findFirst(); - - if (match.isPresent()) { - return match.get().name(); - } else { - throw new IllegalArgumentException("Member does not support any of the candidate protocols"); + for (JoinGroupRequestProtocol protocol : supportedProtocols) { + if (candidates.contains(protocol.name())) { + return protocol.name(); + } } + + throw new IllegalArgumentException("Member does not support any of the candidate protocols"); } /** * Transform protocols into their respective names. + * * @param supportedProtocols list of supported protocols. * @return a set of protocol names from the given list of supported protocols. */ - public static Set plainProtocolSet(List supportedProtocols) { - return supportedProtocols.stream().map(Protocol::name).collect(Collectors.toSet()); + public static Set plainProtocolSet( + JoinGroupRequestProtocolCollection supportedProtocols + ) { + Set protocolNames = new HashSet<>(); + for (JoinGroupRequestProtocol protocol : supportedProtocols) { + protocolNames.add(protocol.name()); + } + return protocolNames; + } + + /** + * @return whether the member has an assignment set. + */ + public boolean hasAssignment() { + return assignment != null && assignment.length > 0; } /** @@ -315,7 +315,7 @@ public String protocolType() { /** * @return the list of supported protocols. */ - public List supportedProtocols() { + public JoinGroupRequestProtocolCollection supportedProtocols() { return supportedProtocols; } @@ -347,13 +347,6 @@ public boolean isNew() { return isNew; } - /** - * @return true if the existing heartbeat was satisfied, false otherwise. - */ - public boolean heartBeatSatisfied() { - return heartbeatSatisfied; - } - /** * @param value the new rebalance timeout in milliseconds. */ @@ -371,7 +364,7 @@ public void setSessionTimeoutMs(int value) { /** * @param value the new list of supported protocols. */ - public void setSupportedProtocols(List value) { + public void setSupportedProtocols(JoinGroupRequestProtocolCollection value) { this.supportedProtocols = value; } @@ -403,13 +396,6 @@ public void setIsNew(boolean value) { this.isNew = value; } - /** - * @param value whether the heartbeat was satisfied. - */ - public void setHeartBeatSatisfied(boolean value) { - this.heartbeatSatisfied = value; - } - @Override public String toString() { return "GenericGroupMember(" + diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/generic/Protocol.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/generic/Protocol.java deleted file mode 100644 index 3f27ad19d9c8d..0000000000000 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/generic/Protocol.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kafka.coordinator.group.generic; - -import java.util.Arrays; -import java.util.Objects; - -/** - * This class represents a protocol that is supported by a - * {@link GenericGroupMember}. - */ -public class Protocol { - - /** - * Represents an empty metadata. - */ - public static final byte[] EMPTY_METADATA = new byte[0]; - - /** - * The name of the protocol. - */ - private final String name; - - /** - * The protocol's metadata. - */ - private final byte[] metadata; - - public Protocol(String name, byte[] metadata) { - this.name = Objects.requireNonNull(name); - this.metadata = metadata; - } - - /** - * @return the name of the protocol. - */ - public String name() { - return this.name; - } - - /** - * @return the metadata stored inside the protocol. - */ - public byte[] metadata() { - return this.metadata; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Protocol protocol = (Protocol) o; - - if (!name.equals(protocol.name)) return false; - return Arrays.equals(metadata, protocol.metadata); - } - - @Override - public int hashCode() { - int result = name.hashCode(); - result = 31 * result + Arrays.hashCode(metadata); - return result; - } - - @Override - public String toString() { - return "Protocol(name=" + name + - ", metadata=" + Arrays.toString(metadata) + - ")"; - } -} diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/runtime/CoordinatorBuilder.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/runtime/CoordinatorBuilder.java index 8c9cf4315445b..dae9c6d62a36e 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/runtime/CoordinatorBuilder.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/runtime/CoordinatorBuilder.java @@ -16,10 +16,12 @@ */ package org.apache.kafka.coordinator.group.runtime; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.timeline.SnapshotRegistry; + /** * A builder to build a {@link Coordinator} replicated state machine. * @@ -51,6 +53,16 @@ CoordinatorBuilder withLogContext( LogContext logContext ); + /** + * Sets the topic partition. + * @param topicPartition The topic partition. + * + * @return The builder. + */ + CoordinatorBuilder withTopicPartition( + TopicPartition topicPartition + ); + /** * Sets the time. * @@ -70,7 +82,7 @@ CoordinatorBuilder withTime( * @return The builder. */ CoordinatorBuilder withTimer( - CoordinatorTimer timer + CoordinatorTimer timer ); /** diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/runtime/CoordinatorResult.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/runtime/CoordinatorResult.java index ff49bc95657c6..92907797581cc 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/runtime/CoordinatorResult.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/runtime/CoordinatorResult.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Objects; +import java.util.concurrent.CompletableFuture; /** * The result of an operation applied to a state machine. The result @@ -37,6 +38,11 @@ public class CoordinatorResult { */ private final T response; + /** + * The future to complete once the records are committed. + */ + private final CompletableFuture appendFuture; + /** * Constructs a Result with records and a response. * @@ -46,9 +52,38 @@ public class CoordinatorResult { public CoordinatorResult( List records, T response + ) { + this(records, response, null); + } + + /** + * Constructs a Result with records and an append-future. + * + * @param records A non-null list of records. + * @param appendFuture The future to complete once the records are committed. + */ + public CoordinatorResult( + List records, + CompletableFuture appendFuture + ) { + this(records, null, appendFuture); + } + + /** + * Constructs a Result with records, a response, and an append-future. + * + * @param records A non-null list of records. + * @param response A response. + * @param appendFuture The future to complete once the records are committed. + */ + public CoordinatorResult( + List records, + T response, + CompletableFuture appendFuture ) { this.records = Objects.requireNonNull(records); this.response = response; + this.appendFuture = appendFuture; } /** @@ -59,7 +94,7 @@ public CoordinatorResult( public CoordinatorResult( List records ) { - this(records, null); + this(records, null, null); } /** @@ -76,28 +111,47 @@ public T response() { return response; } + /** + * @return The append-future. + */ + public CompletableFuture appendFuture() { + return appendFuture; + } + + /** + * If the append-future exists, this means + * that the in-memory state was already updated. + * + * @return Whether to replay the records. + */ + public boolean replayRecords() { + return appendFuture == null; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - CoordinatorResult result = (CoordinatorResult) o; + CoordinatorResult that = (CoordinatorResult) o; - if (!records.equals(result.records)) return false; - return response.equals(result.response); + if (!Objects.equals(records, that.records)) return false; + if (!Objects.equals(response, that.response)) return false; + return Objects.equals(appendFuture, that.appendFuture); } @Override public int hashCode() { - int result = records.hashCode(); - result = 31 * result + response.hashCode(); + int result = records != null ? records.hashCode() : 0; + result = 31 * result + (response != null ? response.hashCode() : 0); + result = 31 * result + (appendFuture != null ? appendFuture.hashCode() : 0); return result; } - @Override public String toString() { - return "Result(records=" + records + + return "CoordinatorResult(records=" + records + ", response=" + response + + ", appendFuture=" + appendFuture + ")"; } } diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/runtime/CoordinatorRuntime.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/runtime/CoordinatorRuntime.java index 815988931966e..c2ef5ec994e99 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/runtime/CoordinatorRuntime.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/runtime/CoordinatorRuntime.java @@ -85,7 +85,7 @@ public static class Builder, U> { private PartitionWriter partitionWriter; private CoordinatorLoader loader; private CoordinatorBuilderSupplier coordinatorBuilderSupplier; - private Time time; + private Time time = Time.SYSTEM; private Timer timer; public Builder withLogPrefix(String logPrefix) { @@ -231,7 +231,7 @@ boolean canTransitionFrom(CoordinatorState state) { * * When a timer fails with an unexpected exception, the timer is rescheduled with a backoff. */ - class EventBasedCoordinatorTimer implements CoordinatorTimer { + class EventBasedCoordinatorTimer implements CoordinatorTimer { /** * The logger. */ @@ -258,7 +258,7 @@ public void schedule( long delay, TimeUnit unit, boolean retry, - TimeoutOperation operation + TimeoutOperation operation ) { // The TimerTask wraps the TimeoutOperation into a CoordinatorWriteEvent. When the TimerTask // expires, the event is pushed to the queue of the coordinator runtime to be executed. This @@ -277,7 +277,7 @@ public void run() { } // Execute the timeout operation. - return new CoordinatorResult<>(operation.generateRecords(), null); + return operation.generateRecords(); }); // If the write event fails, it is rescheduled with a small backoff except if retry @@ -505,6 +505,7 @@ private void transitionTo( .withSnapshotRegistry(snapshotRegistry) .withTime(time) .withTimer(timer) + .withTopicPartition(tp) .build(); break; @@ -654,7 +655,9 @@ public void run() { // is put into the deferred event queue. try { // Apply the records to the state machine. - result.records().forEach(context.coordinator::replay); + if (result.replayRecords()) { + result.records().forEach(context.coordinator::replay); + } // Write the records to the log and update the last written // offset. @@ -685,9 +688,13 @@ public void run() { */ @Override public void complete(Throwable exception) { + CompletableFuture appendFuture = result != null ? result.appendFuture() : null; + if (exception == null) { + if (appendFuture != null) result.appendFuture().complete(result.response()); future.complete(result.response()); } else { + if (appendFuture != null) result.appendFuture().completeExceptionally(exception); future.completeExceptionally(exception); } } diff --git a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/runtime/CoordinatorTimer.java b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/runtime/CoordinatorTimer.java index ff718259b2f0f..834db48bd29ae 100644 --- a/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/runtime/CoordinatorTimer.java +++ b/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/runtime/CoordinatorTimer.java @@ -18,13 +18,12 @@ import org.apache.kafka.common.KafkaException; -import java.util.List; import java.util.concurrent.TimeUnit; /** * An interface to schedule and cancel operations. */ -public interface CoordinatorTimer { +public interface CoordinatorTimer { /** * Generates the records needed to implement this timeout write operation. In general, * this operation should not modify the hard state of the coordinator. That modifications @@ -36,8 +35,8 @@ public interface CoordinatorTimer { * * @param The record type. */ - interface TimeoutOperation { - List generateRecords() throws KafkaException; + interface TimeoutOperation { + CoordinatorResult generateRecords() throws KafkaException; } /** @@ -51,7 +50,7 @@ interface TimeoutOperation { * be retried on failure. * @param operation The operation to perform upon expiration. */ - void schedule(String key, long delay, TimeUnit unit, boolean retry, TimeoutOperation operation); + void schedule(String key, long delay, TimeUnit unit, boolean retry, TimeoutOperation operation); /** * Remove an operation corresponding to a given key. diff --git a/group-coordinator/src/main/resources/common/message/GroupMetadataValue.json b/group-coordinator/src/main/resources/common/message/GroupMetadataValue.json index 8405fbb991853..6045f77812266 100644 --- a/group-coordinator/src/main/resources/common/message/GroupMetadataValue.json +++ b/group-coordinator/src/main/resources/common/message/GroupMetadataValue.json @@ -38,7 +38,7 @@ { "name": "groupInstanceId", "versions": "3+", "type": "string", "default": "null", "nullableVersions": "3+", "ignorable": true}, { "name": "clientId", "versions": "0+", "type": "string" }, { "name": "clientHost", "versions": "0+", "type": "string" }, - { "name": "rebalanceTimeout", "versions": "1+", "type": "int32", "ignorable": true}, + { "name": "rebalanceTimeout", "versions": "1+", "type": "int32", "default": -1, "ignorable": true}, { "name": "sessionTimeout", "versions": "0+", "type": "int32" }, { "name": "subscription", "versions": "0+", "type": "bytes" }, { "name": "assignment", "versions": "0+", "type": "bytes" } diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupCoordinatorConfigTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupCoordinatorConfigTest.java index e54b6229c3759..9025a1ffb5d4b 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupCoordinatorConfigTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupCoordinatorConfigTest.java @@ -34,7 +34,12 @@ public void testConfigs() { 10, 55, Collections.singletonList(assignor), - 2222 + 2222, + 60, + 3000, + 5 * 60 * 1000, + 120, + 10 * 60 * 1000 ); assertEquals(10, config.numThreads); @@ -43,5 +48,10 @@ public void testConfigs() { assertEquals(55, config.consumerGroupMaxSize); assertEquals(Collections.singletonList(assignor), config.consumerGroupAssignors); assertEquals(2222, config.offsetsTopicSegmentBytes); + assertEquals(60, config.genericGroupMaxSize); + assertEquals(3000, config.genericGroupInitialRebalanceDelayMs); + assertEquals(5 * 60 * 1000, config.genericGroupNewMemberJoinTimeoutMs); + assertEquals(120, config.genericGroupMinSessionTimeoutMs); + assertEquals(10 * 60 * 1000, config.genericGroupMaxSessionTimeoutMs); } } diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupCoordinatorServiceTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupCoordinatorServiceTest.java index 29851e4db3702..f2586ab8fcadd 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupCoordinatorServiceTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupCoordinatorServiceTest.java @@ -29,8 +29,17 @@ import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; import org.apache.kafka.common.message.ConsumerGroupHeartbeatResponseData; +import org.apache.kafka.common.message.JoinGroupRequestData; +import org.apache.kafka.common.message.JoinGroupResponseData; +import org.apache.kafka.common.network.ClientInformation; +import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.RequestContext; +import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.security.auth.KafkaPrincipal; +import org.apache.kafka.common.security.auth.SecurityProtocol; +import org.apache.kafka.common.utils.BufferSupplier; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.coordinator.group.assignor.RangeAssignor; @@ -43,6 +52,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentMatchers; +import java.net.InetAddress; import java.util.Collections; import java.util.OptionalInt; import java.util.Properties; @@ -52,10 +62,13 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Stream; +import static org.apache.kafka.common.requests.JoinGroupRequest.UNKNOWN_MEMBER_ID; import static org.apache.kafka.coordinator.group.TestUtil.requestContext; import static org.apache.kafka.test.TestUtils.assertFutureThrows; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -75,7 +88,12 @@ private GroupCoordinatorConfig createConfig() { 5, Integer.MAX_VALUE, Collections.singletonList(new RangeAssignor()), - 1000 + 1000, + Integer.MAX_VALUE, + 3000, + 5 * 60 * 1000, + 120, + 10 * 5 * 1000 ); } @@ -271,4 +289,113 @@ public void testOnResignation() { 10 ); } + + @Test + public void testJoinGroup() { + CoordinatorRuntime runtime = mockRuntime(); + GroupCoordinatorService service = new GroupCoordinatorService( + new LogContext(), + createConfig(), + runtime + ); + + JoinGroupRequestData request = new JoinGroupRequestData() + .setGroupId("foo"); + + service.startup(() -> 1); + + when(runtime.scheduleWriteOperation( + ArgumentMatchers.eq("generic-group-join"), + ArgumentMatchers.eq(new TopicPartition("__consumer_offsets", 0)), + ArgumentMatchers.any() + )).thenReturn(CompletableFuture.completedFuture( + new JoinGroupResponseData() + )); + + CompletableFuture responseFuture = service.joinGroup( + requestContext(ApiKeys.JOIN_GROUP), + request, + BufferSupplier.NO_CACHING + ); + + assertFalse(responseFuture.isDone()); + } + + @Test + public void testJoinGroupWithException() throws Exception { + CoordinatorRuntime runtime = mockRuntime(); + GroupCoordinatorService service = new GroupCoordinatorService( + new LogContext(), + createConfig(), + runtime + ); + + JoinGroupRequestData request = new JoinGroupRequestData() + .setGroupId("foo"); + + service.startup(() -> 1); + + when(runtime.scheduleWriteOperation( + ArgumentMatchers.eq("generic-group-join"), + ArgumentMatchers.eq(new TopicPartition("__consumer_offsets", 0)), + ArgumentMatchers.any() + )).thenReturn(FutureUtils.failedFuture(new IllegalStateException())); + + CompletableFuture future = service.joinGroup( + requestContext(ApiKeys.JOIN_GROUP), + request, + BufferSupplier.NO_CACHING + ); + + assertEquals( + new JoinGroupResponseData() + .setErrorCode(Errors.UNKNOWN_SERVER_ERROR.code()), + future.get(5, TimeUnit.SECONDS) + ); + } + + @Test + public void testJoinGroupInvalidGroupId() throws Exception { + CoordinatorRuntime runtime = mockRuntime(); + GroupCoordinatorService service = new GroupCoordinatorService( + new LogContext(), + createConfig(), + runtime + ); + + service.startup(() -> 1); + + JoinGroupRequestData request = new JoinGroupRequestData() + .setGroupId(null) + .setMemberId(UNKNOWN_MEMBER_ID); + + RequestContext context = new RequestContext( + new RequestHeader( + ApiKeys.JOIN_GROUP, + ApiKeys.JOIN_GROUP.latestVersion(), + "client", + 0 + ), + "1", + InetAddress.getLoopbackAddress(), + KafkaPrincipal.ANONYMOUS, + ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT), + SecurityProtocol.PLAINTEXT, + ClientInformation.EMPTY, + false + ); + + CompletableFuture response = service.joinGroup( + context, + request, + BufferSupplier.NO_CACHING + ); + + assertTrue(response.isDone()); + JoinGroupResponseData expectedResponse = new JoinGroupResponseData() + .setErrorCode(Errors.INVALID_GROUP_ID.code()) + .setMemberId(UNKNOWN_MEMBER_ID); + + assertEquals(expectedResponse, response.get()); + } } diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java index 12d53c92efa1c..af4cb709b2801 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/GroupMetadataManagerTest.java @@ -16,6 +16,9 @@ */ package org.apache.kafka.coordinator.group; +import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor; +import org.apache.kafka.clients.consumer.internals.ConsumerProtocol; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.FencedMemberEpochException; import org.apache.kafka.common.errors.GroupIdNotFoundException; @@ -23,9 +26,12 @@ import org.apache.kafka.common.errors.InvalidRequestException; import org.apache.kafka.common.errors.UnknownMemberIdException; import org.apache.kafka.common.errors.UnknownServerException; +import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; import org.apache.kafka.common.errors.UnsupportedAssignorException; import org.apache.kafka.common.message.ConsumerGroupHeartbeatRequestData; import org.apache.kafka.common.message.ConsumerGroupHeartbeatResponseData; +import org.apache.kafka.common.message.JoinGroupRequestData; +import org.apache.kafka.common.message.JoinGroupResponseData; import org.apache.kafka.common.metadata.PartitionRecord; import org.apache.kafka.common.metadata.RemoveTopicRecord; import org.apache.kafka.common.metadata.TopicRecord; @@ -33,12 +39,15 @@ import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ApiMessage; +import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.RequestContext; import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.coordinator.group.MockCoordinatorTimer.ExpiredTimeout; +import org.apache.kafka.coordinator.group.MockCoordinatorTimer.ScheduledTimeout; import org.apache.kafka.coordinator.group.assignor.AssignmentSpec; import org.apache.kafka.coordinator.group.assignor.GroupAssignment; import org.apache.kafka.coordinator.group.assignor.MemberAssignment; @@ -60,6 +69,10 @@ import org.apache.kafka.coordinator.group.generated.ConsumerGroupTargetAssignmentMemberValue; import org.apache.kafka.coordinator.group.generated.ConsumerGroupTargetAssignmentMetadataKey; import org.apache.kafka.coordinator.group.generated.ConsumerGroupTargetAssignmentMetadataValue; +import org.apache.kafka.coordinator.group.generated.GroupMetadataKey; +import org.apache.kafka.coordinator.group.generated.GroupMetadataValue; +import org.apache.kafka.coordinator.group.generic.GenericGroup; +import org.apache.kafka.coordinator.group.generic.GenericGroupMember; import org.apache.kafka.coordinator.group.runtime.CoordinatorResult; import org.apache.kafka.image.MetadataDelta; import org.apache.kafka.image.MetadataImage; @@ -67,8 +80,11 @@ import org.apache.kafka.image.TopicImage; import org.apache.kafka.image.TopicsImage; import org.apache.kafka.server.common.ApiMessageAndVersion; +import org.apache.kafka.server.common.MetadataVersion; import org.apache.kafka.timeline.SnapshotRegistry; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.opentest4j.AssertionFailedError; import java.net.InetAddress; @@ -80,13 +96,30 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.apache.kafka.common.utils.Utils.mkSet; +import static org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocol; +import static org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection; +import static org.apache.kafka.common.requests.JoinGroupRequest.UNKNOWN_MEMBER_ID; import static org.apache.kafka.coordinator.group.AssignmentTestUtil.mkAssignment; import static org.apache.kafka.coordinator.group.AssignmentTestUtil.mkTopicAssignment; +import static org.apache.kafka.coordinator.group.GroupMetadataManager.appendGroupMetadataErrorToResponseError; import static org.apache.kafka.coordinator.group.GroupMetadataManager.consumerGroupRevocationTimeoutKey; import static org.apache.kafka.coordinator.group.GroupMetadataManager.consumerGroupSessionTimeoutKey; +import static org.apache.kafka.coordinator.group.GroupMetadataManager.EMPTY_RESULT; +import static org.apache.kafka.coordinator.group.GroupMetadataManager.genericGroupHeartbeatKey; +import static org.apache.kafka.coordinator.group.generic.GenericGroupState.COMPLETING_REBALANCE; +import static org.apache.kafka.coordinator.group.generic.GenericGroupState.DEAD; +import static org.apache.kafka.coordinator.group.generic.GenericGroupState.EMPTY; +import static org.apache.kafka.coordinator.group.generic.GenericGroupState.PREPARING_REBALANCE; +import static org.apache.kafka.coordinator.group.generic.GenericGroupState.STABLE; import static org.junit.jupiter.api.AssertionFailureBuilder.assertionFailure; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -95,6 +128,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -231,14 +265,20 @@ public List build(TopicsImage topicsImage) { static class GroupMetadataManagerTestContext { static class Builder { final private MockTime time = new MockTime(); - final private MockCoordinatorTimer timer = new MockCoordinatorTimer<>(time); + final private MockCoordinatorTimer timer = new MockCoordinatorTimer<>(time); final private LogContext logContext = new LogContext(); final private SnapshotRegistry snapshotRegistry = new SnapshotRegistry(logContext); + final private TopicPartition groupMetadataTopicPartition = new TopicPartition("topic", 0); private MetadataImage metadataImage; - private List assignors; + private List assignors = Collections.singletonList(new MockPartitionAssignor("range")); private List consumerGroupBuilders = new ArrayList<>(); private int consumerGroupMaxSize = Integer.MAX_VALUE; private int consumerGroupMetadataRefreshIntervalMs = Integer.MAX_VALUE; + private int genericGroupMaxSize = Integer.MAX_VALUE; + private int genericGroupInitialRebalanceDelayMs = 3000; + private int genericGroupNewMemberJoinTimeoutMs = 5 * 60 * 1000; + private int genericGroupMinSessionTimeoutMs = 10; + private int genericGroupMaxSessionTimeoutMs = 10 * 60 * 1000; public Builder withMetadataImage(MetadataImage metadataImage) { this.metadataImage = metadataImage; @@ -265,6 +305,26 @@ public Builder withConsumerGroupMetadataRefreshIntervalMs(int consumerGroupMetad return this; } + public Builder withGenericGroupMaxSize(int genericGroupMaxSize) { + this.genericGroupMaxSize = genericGroupMaxSize; + return this; + } + + public Builder withGenericGroupInitialRebalanceDelayMs(int genericGroupInitialRebalanceDelayMs) { + this.genericGroupInitialRebalanceDelayMs = genericGroupInitialRebalanceDelayMs; + return this; + } + + public Builder withGenericGroupMinSessionTimeoutMs(int genericGroupMinSessionTimeoutMs) { + this.genericGroupMinSessionTimeoutMs = genericGroupMinSessionTimeoutMs; + return this; + } + + public Builder withGenericGroupMaxSessionTimeoutMs(int genericGroupMaxSessionTimeoutMs) { + this.genericGroupMaxSessionTimeoutMs = genericGroupMaxSessionTimeoutMs; + return this; + } + public GroupMetadataManagerTestContext build() { if (metadataImage == null) metadataImage = MetadataImage.EMPTY; if (assignors == null) assignors = Collections.emptyList(); @@ -274,6 +334,7 @@ public GroupMetadataManagerTestContext build() { timer, snapshotRegistry, new GroupMetadataManager.Builder() + .withTopicPartition(groupMetadataTopicPartition) .withSnapshotRegistry(snapshotRegistry) .withLogContext(logContext) .withTime(time) @@ -284,7 +345,14 @@ public GroupMetadataManagerTestContext build() { .withConsumerGroupMaxSize(consumerGroupMaxSize) .withAssignors(assignors) .withConsumerGroupMetadataRefreshIntervalMs(consumerGroupMetadataRefreshIntervalMs) - .build() + .withGenericGroupMaxSize(genericGroupMaxSize) + .withGenericGroupMinSessionTimeoutMs(genericGroupMinSessionTimeoutMs) + .withGenericGroupMaxSessionTimeoutMs(genericGroupMaxSessionTimeoutMs) + .withGenericGroupInitialRebalanceDelayMs(genericGroupInitialRebalanceDelayMs) + .withGenericGroupNewMemberJoinTimeoutMs(genericGroupNewMemberJoinTimeoutMs) + .build(), + genericGroupInitialRebalanceDelayMs, + genericGroupNewMemberJoinTimeoutMs ); consumerGroupBuilders.forEach(builder -> { @@ -298,23 +366,29 @@ public GroupMetadataManagerTestContext build() { } final MockTime time; - final MockCoordinatorTimer timer; + final MockCoordinatorTimer timer; final SnapshotRegistry snapshotRegistry; final GroupMetadataManager groupMetadataManager; + final int genericGroupInitialRebalanceDelayMs; + final int genericGroupNewMemberJoinTimeoutMs; long lastCommittedOffset = 0L; long lastWrittenOffset = 0L; public GroupMetadataManagerTestContext( MockTime time, - MockCoordinatorTimer timer, + MockCoordinatorTimer timer, SnapshotRegistry snapshotRegistry, - GroupMetadataManager groupMetadataManager + GroupMetadataManager groupMetadataManager, + int genericGroupInitialRebalanceDelayMs, + int genericGroupNewMemberJoinTimeoutMs ) { this.time = time; this.timer = timer; this.snapshotRegistry = snapshotRegistry; this.groupMetadataManager = groupMetadataManager; + this.genericGroupInitialRebalanceDelayMs = genericGroupInitialRebalanceDelayMs; + this.genericGroupNewMemberJoinTimeoutMs = genericGroupNewMemberJoinTimeoutMs; } public void commit() { @@ -376,19 +450,28 @@ public CoordinatorResult consumerGro return result; } - public List> sleep(long ms) { + public List> sleep(long ms) { time.sleep(ms); - List> timeouts = timer.poll(); - timeouts.forEach(timeout -> timeout.records.forEach(this::replay)); + List> timeouts = timer.poll(); + timeouts.forEach(timeout -> { + if (timeout.result.replayRecords()) { + timeout.result.records().forEach(this::replay); + } + }); return timeouts; } - public MockCoordinatorTimer.ScheduledTimeout assertSessionTimeout( + public void sleepAndAssertEmptyResult(long ms) { + List> timeouts = sleep(ms); + timeouts.forEach(timeout -> assertEquals(EMPTY_RESULT, timeout.result)); + } + + public ScheduledTimeout assertSessionTimeout( String groupId, String memberId, long delayMs ) { - MockCoordinatorTimer.ScheduledTimeout timeout = + ScheduledTimeout timeout = timer.timeout(consumerGroupSessionTimeoutKey(groupId, memberId)); assertNotNull(timeout); assertEquals(time.milliseconds() + delayMs, timeout.deadlineMs); @@ -399,17 +482,17 @@ public void assertNoSessionTimeout( String groupId, String memberId ) { - MockCoordinatorTimer.ScheduledTimeout timeout = + ScheduledTimeout timeout = timer.timeout(consumerGroupSessionTimeoutKey(groupId, memberId)); assertNull(timeout); } - public MockCoordinatorTimer.ScheduledTimeout assertRevocationTimeout( + public ScheduledTimeout assertRevocationTimeout( String groupId, String memberId, long delayMs ) { - MockCoordinatorTimer.ScheduledTimeout timeout = + ScheduledTimeout timeout = timer.timeout(consumerGroupRevocationTimeoutKey(groupId, memberId)); assertNotNull(timeout); assertEquals(time.milliseconds() + delayMs, timeout.deadlineMs); @@ -420,11 +503,141 @@ public void assertNoRevocationTimeout( String groupId, String memberId ) { - MockCoordinatorTimer.ScheduledTimeout timeout = + ScheduledTimeout timeout = timer.timeout(consumerGroupRevocationTimeoutKey(groupId, memberId)); assertNull(timeout); } + GenericGroup createGenericGroup(String groupId) { + return groupMetadataManager.getOrMaybeCreateGenericGroup(groupId, true); + } + + public CoordinatorResult sendGenericGroupJoin( + JoinGroupRequestData request, + CompletableFuture responseFuture + ) { + return sendGenericGroupJoin(request, responseFuture, false); + } + + public CoordinatorResult sendGenericGroupJoin( + JoinGroupRequestData request, + CompletableFuture responseFuture, + boolean requireKnownMemberId + ) { + return sendGenericGroupJoin(request, responseFuture, requireKnownMemberId, false); + } + + public CoordinatorResult sendGenericGroupJoin( + JoinGroupRequestData request, + CompletableFuture responseFuture, + boolean requireKnownMemberId, + boolean supportSkippingAssignment + ) { + // requireKnownMemberId is true: version >= 4 (See JoinGroupRequest#requiresKnownMemberId()) + // supportSkippingAssignment is true: version >= 9 (See JoinGroupRequest#supportsSkippingAssignment()) + short joinGroupVersion = 3; + + if (requireKnownMemberId) { + joinGroupVersion = 4; + if (supportSkippingAssignment) { + joinGroupVersion = ApiKeys.JOIN_GROUP.latestVersion(); + } + } + + RequestContext context = new RequestContext( + new RequestHeader( + ApiKeys.JOIN_GROUP, + joinGroupVersion, + "client", + 0 + ), + "1", + InetAddress.getLoopbackAddress(), + KafkaPrincipal.ANONYMOUS, + ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT), + SecurityProtocol.PLAINTEXT, + ClientInformation.EMPTY, + false + ); + + return groupMetadataManager.genericGroupJoin( + context, + request, + responseFuture + ); + } + + public JoinGroupResponseData joinGenericGroupAsDynamicMemberAndCompleteJoin( + JoinGroupRequestData request + ) throws ExecutionException, InterruptedException { + boolean requireKnownMemberId = true; + String newMemberId = request.memberId(); + + if (request.memberId().equals(UNKNOWN_MEMBER_ID)) { + // Since member id is required, we need another round to get the successful join group result. + CompletableFuture firstJoinFuture = new CompletableFuture<>(); + sendGenericGroupJoin( + request, + firstJoinFuture, + requireKnownMemberId + ); + assertTrue(firstJoinFuture.isDone()); + assertEquals(Errors.MEMBER_ID_REQUIRED.code(), firstJoinFuture.get().errorCode()); + newMemberId = firstJoinFuture.get().memberId(); + } + + // Second round + CompletableFuture secondJoinFuture = new CompletableFuture<>(); + JoinGroupRequestData secondRequest = new JoinGroupRequestData() + .setGroupId(request.groupId()) + .setMemberId(newMemberId) + .setProtocolType(request.protocolType()) + .setProtocols(request.protocols()) + .setSessionTimeoutMs(request.sessionTimeoutMs()) + .setRebalanceTimeoutMs(request.rebalanceTimeoutMs()) + .setReason(request.reason()); + + sendGenericGroupJoin( + secondRequest, + secondJoinFuture, + requireKnownMemberId + ); + + List> timeouts = sleep(genericGroupInitialRebalanceDelayMs); + assertEquals(1, timeouts.size()); + assertTrue(secondJoinFuture.isDone()); + assertEquals(Errors.NONE.code(), secondJoinFuture.get().errorCode()); + return secondJoinFuture.get(); + } + + public JoinGroupResponseData joinGenericGroupAndCompleteJoin( + JoinGroupRequestData request, + boolean requireKnownMemberId, + boolean supportSkippingAssignment + ) throws ExecutionException, InterruptedException { + if (requireKnownMemberId && request.groupInstanceId().isEmpty()) { + return joinGenericGroupAsDynamicMemberAndCompleteJoin(request); + } + + try { + CompletableFuture responseFuture = new CompletableFuture<>(); + sendGenericGroupJoin( + request, + responseFuture, + requireKnownMemberId, + supportSkippingAssignment + ); + + sleep(genericGroupInitialRebalanceDelayMs); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.NONE.code(), responseFuture.get().errorCode()); + return responseFuture.get(); + } catch (Exception e) { + fail("Failed to due: " + e.getMessage()); + } + return null; + } + private ApiMessage messageOrNull(ApiMessageAndVersion apiMessageAndVersion) { if (apiMessageAndVersion == null) { return null; @@ -444,6 +657,13 @@ private void replay( } switch (key.version()) { + case GroupMetadataKey.HIGHEST_SUPPORTED_VERSION: + groupMetadataManager.replay( + (GroupMetadataKey) key.message(), + (GroupMetadataValue) messageOrNull(value) + ); + break; + case ConsumerGroupMemberMetadataKey.HIGHEST_SUPPORTED_VERSION: groupMetadataManager.replay( (ConsumerGroupMemberMetadataKey) key.message(), @@ -2574,18 +2794,20 @@ public void testSessionTimeoutExpiration() { context.assertSessionTimeout(groupId, memberId, 45000); // Advance time past the session timeout. - List> timeouts = context.sleep(45000 + 1); + List> timeouts = context.sleep(45000 + 1); // Verify the expired timeout. assertEquals( - Collections.singletonList(new MockCoordinatorTimer.ExpiredTimeout( + Collections.singletonList(new ExpiredTimeout( consumerGroupSessionTimeoutKey(groupId, memberId), - Arrays.asList( - RecordHelpers.newCurrentAssignmentTombstoneRecord(groupId, memberId), - RecordHelpers.newTargetAssignmentTombstoneRecord(groupId, memberId), - RecordHelpers.newMemberSubscriptionTombstoneRecord(groupId, memberId), - RecordHelpers.newGroupSubscriptionMetadataRecord(groupId, Collections.emptyMap()), - RecordHelpers.newGroupEpochRecord(groupId, 2) + new CoordinatorResult<>( + Arrays.asList( + RecordHelpers.newCurrentAssignmentTombstoneRecord(groupId, memberId), + RecordHelpers.newTargetAssignmentTombstoneRecord(groupId, memberId), + RecordHelpers.newMemberSubscriptionTombstoneRecord(groupId, memberId), + RecordHelpers.newGroupSubscriptionMetadataRecord(groupId, Collections.emptyMap()), + RecordHelpers.newGroupEpochRecord(groupId, 2) + ) ) )), timeouts @@ -2797,7 +3019,7 @@ public void testRevocationTimeoutLifecycle() { // Verify that there is a revocation timeout. Keep a reference // to the timeout for later. - MockCoordinatorTimer.ScheduledTimeout scheduledTimeout = + ScheduledTimeout scheduledTimeout = context.assertRevocationTimeout(groupId, memberId1, 90000); assertEquals( @@ -2833,7 +3055,7 @@ public void testRevocationTimeoutLifecycle() { // Execute the scheduled revocation timeout captured earlier to simulate a // stale timeout. This should be a no-op. - assertEquals(Collections.emptyList(), scheduledTimeout.operation.generateRecords()); + assertEquals(Collections.emptyList(), scheduledTimeout.operation.generateRecords().records()); } @Test @@ -2957,17 +3179,19 @@ public void testRevocationTimeoutExpiration() { ); // Advance time past the revocation timeout. - List> timeouts = context.sleep(10000 + 1); + List> timeouts = context.sleep(10000 + 1); // Verify the expired timeout. assertEquals( - Collections.singletonList(new MockCoordinatorTimer.ExpiredTimeout( + Collections.singletonList(new ExpiredTimeout( consumerGroupRevocationTimeoutKey(groupId, memberId1), - Arrays.asList( - RecordHelpers.newCurrentAssignmentTombstoneRecord(groupId, memberId1), - RecordHelpers.newTargetAssignmentTombstoneRecord(groupId, memberId1), - RecordHelpers.newMemberSubscriptionTombstoneRecord(groupId, memberId1), - RecordHelpers.newGroupEpochRecord(groupId, 3) + new CoordinatorResult<>( + Arrays.asList( + RecordHelpers.newCurrentAssignmentTombstoneRecord(groupId, memberId1), + RecordHelpers.newTargetAssignmentTombstoneRecord(groupId, memberId1), + RecordHelpers.newMemberSubscriptionTombstoneRecord(groupId, memberId1), + RecordHelpers.newGroupEpochRecord(groupId, 3) + ) ) )), timeouts @@ -3035,143 +3259,2270 @@ public void testOnLoaded() { assertNotNull(context.timer.timeout(consumerGroupRevocationTimeoutKey("foo", "foo-1"))); } - private void assertUnorderedListEquals( - List expected, - List actual - ) { - assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + @Test + public void testGenerateRecordsOnNewGroup() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture, true); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.MEMBER_ID_REQUIRED.code(), responseFuture.get().errorCode()); + + GenericGroup group = context.createGenericGroup("group-id"); + + assertEquals( + Collections.singletonList(RecordHelpers.newEmptyGroupMetadataRecord(group, MetadataVersion.latest())), + result.records() + ); } - private void assertResponseEquals( - ConsumerGroupHeartbeatResponseData expected, - ConsumerGroupHeartbeatResponseData actual - ) { - if (!responseEquals(expected, actual)) { - assertionFailure() - .expected(expected) - .actual(actual) - .buildAndThrow(); - } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testReplayGroupMetadataRecords(boolean useDefaultRebalanceTimeout) { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + + byte[] subscription = ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("foo"))).array(); + List members = new ArrayList<>(); + List expectedMembers = new ArrayList<>(); + JoinGroupRequestProtocolCollection expectedProtocols = new JoinGroupRequestProtocolCollection(0); + expectedProtocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(subscription)); + + IntStream.range(0, 2).forEach(i -> { + members.add(new GroupMetadataValue.MemberMetadata() + .setMemberId("member-" + i) + .setGroupInstanceId("group-instance-id-" + i) + .setSubscription(subscription) + .setAssignment(new byte[]{2}) + .setClientId("client-" + i) + .setClientHost("host-" + i) + .setSessionTimeout(4000) + .setRebalanceTimeout(useDefaultRebalanceTimeout ? -1 : 9000) + ); + + expectedMembers.add(new GenericGroupMember( + "member-" + i, + Optional.of("group-instance-id-" + i), + "client-" + i, + "host-" + i, + useDefaultRebalanceTimeout ? 4000 : 9000, + 4000, + "consumer", + expectedProtocols, + new byte[]{2} + )); + }); + + Record groupMetadataRecord = newGroupMetadataRecord("group-id", + new GroupMetadataValue() + .setMembers(members) + .setGeneration(1) + .setLeader("member-0") + .setProtocolType("consumer") + .setProtocol("range") + .setCurrentStateTimestamp(context.time.milliseconds()), + MetadataVersion.latest()); + + context.replay(groupMetadataRecord); + GenericGroup group = context.groupMetadataManager.getOrMaybeCreateGenericGroup("group-id", false); + + GenericGroup expectedGroup = new GenericGroup( + new LogContext(), + "group-id", + STABLE, + context.time, + 1, + Optional.of("consumer"), + Optional.of("range"), + Optional.of("member-0"), + Optional.of(context.time.milliseconds()) + ); + expectedMembers.forEach(expectedGroup::add); + + assertEquals(expectedGroup.groupId(), group.groupId()); + assertEquals(expectedGroup.generationId(), group.generationId()); + assertEquals(expectedGroup.protocolType(), group.protocolType()); + assertEquals(expectedGroup.protocolName(), group.protocolName()); + assertEquals(expectedGroup.leaderOrNull(), group.leaderOrNull()); + assertEquals(expectedGroup.currentState(), group.currentState()); + assertEquals(expectedGroup.currentStateTimestampOrDefault(), group.currentStateTimestampOrDefault()); + assertEquals(expectedGroup.currentGenericGroupMembers(), group.currentGenericGroupMembers()); } - private boolean responseEquals( - ConsumerGroupHeartbeatResponseData expected, - ConsumerGroupHeartbeatResponseData actual - ) { - if (expected.throttleTimeMs() != actual.throttleTimeMs()) return false; - if (expected.errorCode() != actual.errorCode()) return false; - if (!Objects.equals(expected.errorMessage(), actual.errorMessage())) return false; - if (!Objects.equals(expected.memberId(), actual.memberId())) return false; - if (expected.memberEpoch() != actual.memberEpoch()) return false; - if (expected.shouldComputeAssignment() != actual.shouldComputeAssignment()) return false; - if (expected.heartbeatIntervalMs() != actual.heartbeatIntervalMs()) return false; - // Unordered comparison of the assignments. - return responseAssignmentEquals(expected.assignment(), actual.assignment()); + @Test + public void testOnLoadedExceedGroupMaxSizeTriggersRebalance() { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withGenericGroupMaxSize(1) + .build(); + + byte[] subscription = ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("foo"))).array(); + List members = new ArrayList<>(); + + IntStream.range(0, 2).forEach(i -> { + members.add(new GroupMetadataValue.MemberMetadata() + .setMemberId("member-" + i) + .setGroupInstanceId("group-instance-id-" + i) + .setSubscription(subscription) + .setAssignment(new byte[]{2}) + .setClientId("client-" + i) + .setClientHost("host-" + i) + .setSessionTimeout(4000) + .setRebalanceTimeout(9000) + ); + }); + + Record groupMetadataRecord = newGroupMetadataRecord("group-id", + new GroupMetadataValue() + .setMembers(members) + .setGeneration(1) + .setLeader("member-0") + .setProtocolType("consumer") + .setProtocol("range") + .setCurrentStateTimestamp(context.time.milliseconds()), + MetadataVersion.latest()); + + context.replay(groupMetadataRecord); + context.groupMetadataManager.onLoaded(); + GenericGroup group = context.groupMetadataManager.getOrMaybeCreateGenericGroup("group-id", false); + + assertTrue(group.isInState(PREPARING_REBALANCE)); + assertEquals(2, group.size()); } - private boolean responseAssignmentEquals( - ConsumerGroupHeartbeatResponseData.Assignment expected, - ConsumerGroupHeartbeatResponseData.Assignment actual - ) { - if (expected == actual) return true; - if (expected == null) return false; - if (actual == null) return false; + @Test + public void testOnLoadedSchedulesGenericGroupMemberHeartbeats() { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); - if (!Objects.equals(fromAssignment(expected.pendingTopicPartitions()), fromAssignment(actual.pendingTopicPartitions()))) - return false; + byte[] subscription = ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("foo"))).array(); + List members = new ArrayList<>(); + + IntStream.range(0, 2).forEach(i -> { + members.add(new GroupMetadataValue.MemberMetadata() + .setMemberId("member-" + i) + .setGroupInstanceId("group-instance-id-" + i) + .setSubscription(subscription) + .setAssignment(new byte[]{2}) + .setClientId("client-" + i) + .setClientHost("host-" + i) + .setSessionTimeout(4000) + .setRebalanceTimeout(9000) + ); + }); - return Objects.equals(fromAssignment(expected.assignedTopicPartitions()), fromAssignment(actual.assignedTopicPartitions())); + Record groupMetadataRecord = newGroupMetadataRecord("group-id", + new GroupMetadataValue() + .setMembers(members) + .setGeneration(1) + .setLeader("member-0") + .setProtocolType("consumer") + .setProtocol("range") + .setCurrentStateTimestamp(context.time.milliseconds()), + MetadataVersion.latest()); + + context.replay(groupMetadataRecord); + context.groupMetadataManager.onLoaded(); + + IntStream.range(0, 2).forEach(i -> { + ScheduledTimeout timeout = context.timer.timeout( + genericGroupHeartbeatKey("group-id", "member-1")); + + assertNotNull(timeout); + assertEquals(context.time.milliseconds() + 4000, timeout.deadlineMs); + }); } - private Map> fromAssignment( - List assignment - ) { - if (assignment == null) return null; + @Test + public void testJoinGroupShouldReceiveErrorIfGroupOverMaxSize() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withGenericGroupMaxSize(10) + .build(); + context.createGenericGroup("group-id"); - Map> assignmentMap = new HashMap<>(); - assignment.forEach(topicPartitions -> { - assignmentMap.put(topicPartitions.topicId(), new HashSet<>(topicPartitions.partitions())); + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .withReason("exceed max group size") + .build(); + + IntStream.range(0, 10).forEach(i -> { + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + assertFalse(responseFuture.isDone()); + assertTrue(result.records().isEmpty()); }); - return assignmentMap; + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.GROUP_MAX_SIZE_REACHED.code(), responseFuture.get(5, TimeUnit.SECONDS).errorCode()); } - private void assertRecordsEquals( - List expectedRecords, - List actualRecords - ) { - try { - assertEquals(expectedRecords.size(), actualRecords.size()); + @Test + public void testDynamicMembersJoinGroupWithMaxSizeAndRequiredKnownMember() { + boolean requiredKnownMemberId = true; + int groupMaxSize = 10; + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withGenericGroupMaxSize(groupMaxSize) + .withGenericGroupInitialRebalanceDelayMs(50) + .build(); - for (int i = 0; i < expectedRecords.size(); i++) { - Record expectedRecord = expectedRecords.get(i); - Record actualRecord = actualRecords.get(i); - assertRecordEquals(expectedRecord, actualRecord); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + // First round of join requests. Generate member ids. All requests will be accepted + // as the group is still Empty. + List> firstRoundFutures = new ArrayList<>(); + IntStream.range(0, groupMaxSize + 1).forEach(i -> { + CompletableFuture responseFuture = new CompletableFuture<>(); + firstRoundFutures.add(responseFuture); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture, requiredKnownMemberId); + assertTrue(responseFuture.isDone()); + JoinGroupResponseData response = null; + try { + response = responseFuture.get(); + } catch (Exception ignored) { } - } catch (AssertionFailedError e) { - assertionFailure() - .expected(expectedRecords) - .actual(actualRecords) - .buildAndThrow(); - } - } + assertNotNull(response); + assertEquals(Errors.MEMBER_ID_REQUIRED.code(), response.errorCode()); + assertTrue(result.records().isEmpty()); + }); - private void assertRecordEquals( - Record expected, - Record actual - ) { - try { - assertApiMessageAndVersionEquals(expected.key(), actual.key()); - assertApiMessageAndVersionEquals(expected.value(), actual.value()); - } catch (AssertionFailedError e) { - assertionFailure() - .expected(expected) - .actual(actual) - .buildAndThrow(); - } + List memberIds = verifyGenericGroupJoinResponses(firstRoundFutures, 0, Errors.MEMBER_ID_REQUIRED); + assertEquals(groupMaxSize + 1, memberIds.size()); + assertEquals(0, group.size()); + assertTrue(group.isInState(EMPTY)); + assertEquals(groupMaxSize + 1, group.numPendingJoinMembers()); + + // Second round of join requests with the generated member ids. + // One of them will fail, reaching group max size. + List> secondRoundFutures = new ArrayList<>(); + memberIds.forEach(memberId -> { + CompletableFuture responseFuture = new CompletableFuture<>(); + secondRoundFutures.add(responseFuture); + CoordinatorResult result = context.sendGenericGroupJoin( + request.setMemberId(memberId), + responseFuture, + requiredKnownMemberId + ); + assertTrue(result.records().isEmpty()); + }); + + // Advance clock by group initial rebalance delay to complete first inital delayed join. + // This will extend the initial rebalance as new members have joined. + assertNoOrEmptyResult(context.sleep(50)); + // Advance clock by group initial rebalance delay to complete second inital delayed join. + // Since there are no new members that joined since the previous delayed join, + // the join group phase will complete. + assertNoOrEmptyResult(context.sleep(50)); + + verifyGenericGroupJoinResponses(secondRoundFutures, groupMaxSize, Errors.GROUP_MAX_SIZE_REACHED); + assertEquals(groupMaxSize, group.size()); + assertEquals(0, group.numPendingJoinMembers()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); + + // Members that were accepted can rejoin while others are rejected in CompletingRebalance state. + List> thirdRoundFutures = new ArrayList<>(); + memberIds.forEach(memberId -> { + CompletableFuture responseFuture = new CompletableFuture<>(); + thirdRoundFutures.add(responseFuture); + CoordinatorResult result = context.sendGenericGroupJoin( + request.setMemberId(memberId), + responseFuture, + requiredKnownMemberId + ); + assertTrue(result.records().isEmpty()); + }); + + verifyGenericGroupJoinResponses(thirdRoundFutures, groupMaxSize, Errors.GROUP_MAX_SIZE_REACHED); } - private void assertApiMessageAndVersionEquals( - ApiMessageAndVersion expected, - ApiMessageAndVersion actual - ) { - if (expected == actual) return; + @Test + public void testDynamicMembersJoinGroupWithMaxSizeAndNotRequiredKnownMember() { + boolean requiredKnownMemberId = false; + int groupMaxSize = 10; + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withGenericGroupMaxSize(groupMaxSize) + .withGenericGroupInitialRebalanceDelayMs(50) + .build(); - assertEquals(expected.version(), actual.version()); + GenericGroup group = context.createGenericGroup("group-id"); - if (actual.message() instanceof ConsumerGroupCurrentMemberAssignmentValue) { - // The order of the topics stored in ConsumerGroupCurrentMemberAssignmentValue is not - // always guaranteed. Therefore, we need a special comparator. - ConsumerGroupCurrentMemberAssignmentValue expectedValue = - (ConsumerGroupCurrentMemberAssignmentValue) expected.message(); - ConsumerGroupCurrentMemberAssignmentValue actualValue = - (ConsumerGroupCurrentMemberAssignmentValue) actual.message(); + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); - assertEquals(expectedValue.memberEpoch(), actualValue.memberEpoch()); - assertEquals(expectedValue.previousMemberEpoch(), actualValue.previousMemberEpoch()); - assertEquals(expectedValue.targetMemberEpoch(), actualValue.targetMemberEpoch()); - assertEquals(expectedValue.error(), actualValue.error()); - assertEquals(expectedValue.metadataVersion(), actualValue.metadataVersion()); - assertEquals(expectedValue.metadataBytes(), actualValue.metadataBytes()); + // First round of join requests. This will trigger a rebalance. + List> firstRoundFutures = new ArrayList<>(); + IntStream.range(0, groupMaxSize + 1).forEach(i -> { + CompletableFuture responseFuture = new CompletableFuture<>(); + firstRoundFutures.add(responseFuture); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture, requiredKnownMemberId); + assertTrue(result.records().isEmpty()); + }); - // We transform those to Maps before comparing them. - assertEquals(fromTopicPartitions(expectedValue.assignedPartitions()), - fromTopicPartitions(actualValue.assignedPartitions())); - assertEquals(fromTopicPartitions(expectedValue.partitionsPendingRevocation()), - fromTopicPartitions(actualValue.partitionsPendingRevocation())); - assertEquals(fromTopicPartitions(expectedValue.partitionsPendingAssignment()), - fromTopicPartitions(actualValue.partitionsPendingAssignment())); - } else { - assertEquals(expected.message(), actual.message()); - } + assertEquals(groupMaxSize, group.size()); + assertEquals(groupMaxSize, group.numAwaitingJoinResponse()); + assertTrue(group.isInState(PREPARING_REBALANCE)); + + // Advance clock by group initial rebalance delay to complete first inital delayed join. + // This will extend the initial rebalance as new members have joined. + assertNoOrEmptyResult(context.sleep(50)); + // Advance clock by group initial rebalance delay to complete second inital delayed join. + // Since there are no new members that joined since the previous delayed join, + // we will complete the rebalance. + assertNoOrEmptyResult(context.sleep(50)); + + List memberIds = verifyGenericGroupJoinResponses(firstRoundFutures, groupMaxSize, Errors.GROUP_MAX_SIZE_REACHED); + + // Members that were accepted can rejoin while others are rejected in CompletingRebalance state. + List> secondRoundFutures = new ArrayList<>(); + memberIds.forEach(memberId -> { + CompletableFuture responseFuture = new CompletableFuture<>(); + secondRoundFutures.add(responseFuture); + CoordinatorResult result = context.sendGenericGroupJoin( + request.setMemberId(memberId), + responseFuture, + requiredKnownMemberId + ); + assertTrue(result.records().isEmpty()); + }); + + verifyGenericGroupJoinResponses(secondRoundFutures, 10, Errors.GROUP_MAX_SIZE_REACHED); + assertEquals(groupMaxSize, group.size()); + assertEquals(0, group.numAwaitingJoinResponse()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); } - private Map> fromTopicPartitions( - List assignment - ) { - Map> assignmentMap = new HashMap<>(); - assignment.forEach(topicPartitions -> { - assignmentMap.put(topicPartitions.topicId(), new HashSet<>(topicPartitions.partitions())); + @Test + public void testStaticMembersJoinGroupWithMaxSize() { + int groupMaxSize = 10; + + List groupInstanceIds = IntStream.range(0, groupMaxSize + 1) + .mapToObj(i -> "instance-id-" + i) + .collect(Collectors.toList()); + + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withGenericGroupMaxSize(groupMaxSize) + .withGenericGroupInitialRebalanceDelayMs(50) + .build(); + + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + // First round of join requests. This will trigger a rebalance. + List> firstRoundFutures = new ArrayList<>(); + groupInstanceIds.forEach(instanceId -> { + CompletableFuture responseFuture = new CompletableFuture<>(); + firstRoundFutures.add(responseFuture); + CoordinatorResult result = context.sendGenericGroupJoin(request.setGroupInstanceId(instanceId), responseFuture); + assertTrue(result.records().isEmpty()); }); - return assignmentMap; + + assertEquals(groupMaxSize, group.size()); + assertEquals(groupMaxSize, group.numAwaitingJoinResponse()); + assertTrue(group.isInState(PREPARING_REBALANCE)); + + // Advance clock by group initial rebalance delay to complete first inital delayed join. + // This will extend the initial rebalance as new members have joined. + assertNoOrEmptyResult(context.sleep(50)); + // Advance clock by group initial rebalance delay to complete second inital delayed join. + // Since there are no new members that joined since the previous delayed join, + // we will complete the rebalance. + assertNoOrEmptyResult(context.sleep(50)); + + List memberIds = verifyGenericGroupJoinResponses(firstRoundFutures, groupMaxSize, Errors.GROUP_MAX_SIZE_REACHED); + + // Members which were accepted can rejoin, others are rejected, while + // completing rebalance + List> secondRoundFutures = new ArrayList<>(); + IntStream.range(0, groupMaxSize + 1).forEach(i -> { + CompletableFuture responseFuture = new CompletableFuture<>(); + secondRoundFutures.add(responseFuture); + CoordinatorResult result = context.sendGenericGroupJoin( + request.setMemberId(memberIds.get(i)) + .setGroupInstanceId(groupInstanceIds.get(i)), + responseFuture + ); + assertTrue(result.records().isEmpty()); + }); + + verifyGenericGroupJoinResponses(secondRoundFutures, groupMaxSize, Errors.GROUP_MAX_SIZE_REACHED); + assertEquals(groupMaxSize, group.size()); + assertEquals(0, group.numAwaitingJoinResponse()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); } -} + + @Test + public void testDynamicMembersCanRejoinGroupWithMaxSizeWhileRebalancing() { + boolean requiredKnownMemberId = true; + int groupMaxSize = 10; + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withGenericGroupMaxSize(groupMaxSize) + .withGenericGroupInitialRebalanceDelayMs(50) + .build(); + + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + // First round of join requests. Generate member ids. + List> firstRoundFutures = new ArrayList<>(); + IntStream.range(0, groupMaxSize + 1).forEach(i -> { + CompletableFuture responseFuture = new CompletableFuture<>(); + firstRoundFutures.add(responseFuture); + CoordinatorResult result = context.sendGenericGroupJoin( + request, + responseFuture, + requiredKnownMemberId + ); + assertTrue(result.records().isEmpty()); + }); + + assertEquals(0, group.size()); + assertEquals(groupMaxSize + 1, group.numPendingJoinMembers()); + assertTrue(group.isInState(EMPTY)); + + List memberIds = verifyGenericGroupJoinResponses(firstRoundFutures, 0, Errors.MEMBER_ID_REQUIRED); + assertEquals(groupMaxSize + 1, memberIds.size()); + + // Second round of join requests with the generated member ids. + // One of them will fail, reaching group max size. + memberIds.forEach(memberId -> { + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin( + request.setMemberId(memberId), + responseFuture, + requiredKnownMemberId + ); + assertTrue(result.records().isEmpty()); + }); + + assertEquals(groupMaxSize, group.size()); + assertEquals(groupMaxSize, group.numAwaitingJoinResponse()); + assertTrue(group.isInState(PREPARING_REBALANCE)); + + // Members can rejoin while rebalancing + List> thirdRoundFutures = new ArrayList<>(); + memberIds.forEach(memberId -> { + CompletableFuture responseFuture = new CompletableFuture<>(); + thirdRoundFutures.add(responseFuture); + CoordinatorResult result = context.sendGenericGroupJoin( + request.setMemberId(memberId), + responseFuture, + requiredKnownMemberId + ); + assertTrue(result.records().isEmpty()); + }); + + // Advance clock by group initial rebalance delay to complete first inital delayed join. + // This will extend the initial rebalance as new members have joined. + assertNoOrEmptyResult(context.sleep(50)); + // Advance clock by group initial rebalance delay to complete second inital delayed join. + // Since there are no new members that joined since the previous delayed join, + // we will complete the rebalance. + assertNoOrEmptyResult(context.sleep(50)); + + verifyGenericGroupJoinResponses(thirdRoundFutures, groupMaxSize, Errors.GROUP_MAX_SIZE_REACHED); + assertEquals(groupMaxSize, group.size()); + assertEquals(0, group.numAwaitingJoinResponse()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); + } + + @Test + public void testLastJoiningMembersAreKickedOutWhenRejoiningGroupWithMaxSize() { + int groupMaxSize = 10; + + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withGenericGroupMaxSize(groupMaxSize) + .withGenericGroupInitialRebalanceDelayMs(50) + .build(); + + // Create a group and add members that exceed the group max size. + GenericGroup group = context.createGenericGroup("group-id"); + + List memberIds = IntStream.range(0, groupMaxSize + 2) + .mapToObj(i -> group.generateMemberId("client-id", Optional.empty())) + .collect(Collectors.toList()); + + memberIds.forEach(memberId -> { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(new byte[0])); + + group.add( + new GenericGroupMember( + memberId, + Optional.empty(), + "client-id", + "client-host", + 10000, + 5000, + "consumer", + protocols + ) + ); + }); + + context.groupMetadataManager.prepareRebalance(group, "test"); + + List> responseFutures = new ArrayList<>(); + memberIds.forEach(memberId -> { + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(memberId) + .withDefaultProtocolTypeAndProtocols() + .withRebalanceTimeoutMs(10000) + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + responseFutures.add(responseFuture); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + assertTrue(result.records().isEmpty()); + }); + + assertEquals(groupMaxSize, group.size()); + assertEquals(groupMaxSize, group.numAwaitingJoinResponse()); + assertTrue(group.isInState(PREPARING_REBALANCE)); + + // Advance clock by rebalance timeout to complete join phase. + assertNoOrEmptyResult(context.sleep(10000)); + + verifyGenericGroupJoinResponses(responseFutures, groupMaxSize, Errors.GROUP_MAX_SIZE_REACHED); + + assertEquals(groupMaxSize, group.size()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); + + memberIds.subList(groupMaxSize, groupMaxSize + 2) + .forEach(memberId -> assertFalse(group.hasMemberId(memberId))); + + memberIds.subList(0, groupMaxSize) + .forEach(memberId -> assertTrue(group.hasMemberId(memberId))); + } + + @Test + public void testJoinGroupSessionTimeoutTooSmall() throws Exception { + int minSessionTimeout = 50; + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withGenericGroupMinSessionTimeoutMs(minSessionTimeout) + .build(); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withSessionTimeoutMs(minSessionTimeout - 1) + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + assertTrue(responseFuture.isDone()); + assertTrue(result.records().isEmpty()); + assertEquals(Errors.INVALID_SESSION_TIMEOUT.code(), responseFuture.get(5, TimeUnit.SECONDS).errorCode()); + } + + @Test + public void testJoinGroupSessionTimeoutTooLarge() throws Exception { + int maxSessionTimeout = 50; + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withGenericGroupMaxSessionTimeoutMs(maxSessionTimeout) + .build(); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withSessionTimeoutMs(maxSessionTimeout + 1) + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.INVALID_SESSION_TIMEOUT.code(), responseFuture.get().errorCode()); + } + + @Test + public void testJoinGroupUnknownMemberNewGroup() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId("member-id") + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.UNKNOWN_MEMBER_ID.code(), responseFuture.get().errorCode()); + + // Static member + request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId("member-id") + .withGroupInstanceId("group-instance-id") + .build(); + + responseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin(request, responseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.UNKNOWN_MEMBER_ID.code(), responseFuture.get().errorCode()); + } + + @Test + public void testGenericGroupJoinInconsistentProtocolType() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + context.createGenericGroup("group-id"); + + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(0); + protocols.add(new JoinGroupRequestProtocol().setName("range")); + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + context.joinGenericGroupAsDynamicMemberAndCompleteJoin(request); + + request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withProtocolType("connect") + .withProtocols(protocols) + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.INCONSISTENT_GROUP_PROTOCOL.code(), responseFuture.get().errorCode()); + } + + @Test + public void testJoinGroupWithEmptyProtocolType() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + context.createGenericGroup("group-id"); + + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(0); + protocols.add(new JoinGroupRequestProtocol().setName("range")); + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withProtocolType("") + .withProtocols(protocols) + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.INCONSISTENT_GROUP_PROTOCOL.code(), responseFuture.get().errorCode()); + + // Send as static member join. + responseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin(request.setGroupInstanceId("group-instance-id"), responseFuture, true, true); + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.INCONSISTENT_GROUP_PROTOCOL.code(), responseFuture.get().errorCode()); + } + + @Test + public void testJoinGroupWithEmptyGroupProtocol() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + context.createGenericGroup("group-id"); + + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(0); + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withProtocolType("consumer") + .withProtocols(protocols) + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.INCONSISTENT_GROUP_PROTOCOL.code(), responseFuture.get().errorCode()); + } + + @Test + public void testNewMemberJoinExpiration() throws Exception { + // This tests new member expiration during a protracted rebalance. We first create a + // group with one member which uses a large value for session timeout and rebalance timeout. + // We then join with one new member and let the rebalance hang while we await the first member. + // The new member join timeout expires and its JoinGroup request is failed. + + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .withSessionTimeoutMs(5000 + context.genericGroupNewMemberJoinTimeoutMs) + .withRebalanceTimeoutMs(2 * context.genericGroupNewMemberJoinTimeoutMs) + .build(); + + JoinGroupResponseData firstResponse = context.joinGenericGroupAsDynamicMemberAndCompleteJoin(request); + String firstMemberId = firstResponse.memberId(); + assertEquals(firstResponse.leader(), firstMemberId); + assertEquals(Errors.NONE.code(), firstResponse.errorCode()); + + assertNotNull(group); + assertEquals(0, group.allMembers().stream().filter(GenericGroupMember::isNew).count()); + + // Send second join group request for a new dynamic member. + CompletableFuture secondResponseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request + .setSessionTimeoutMs(5000) + .setRebalanceTimeoutMs(5000), + secondResponseFuture + ); + assertTrue(result.records().isEmpty()); + assertFalse(secondResponseFuture.isDone()); + + assertEquals(2, group.allMembers().size()); + assertEquals(1, group.allMembers().stream().filter(GenericGroupMember::isNew).count()); + + GenericGroupMember newMember = group.allMembers().stream().filter(GenericGroupMember::isNew).findFirst().get(); + assertNotEquals(firstMemberId, newMember.memberId()); + + // Advance clock by new member join timeout to expire the second member. + assertNoOrEmptyResult(context.sleep(context.genericGroupNewMemberJoinTimeoutMs)); + + assertTrue(secondResponseFuture.isDone()); + JoinGroupResponseData secondResponse = secondResponseFuture.get(); + + assertEquals(Errors.UNKNOWN_MEMBER_ID.code(), secondResponse.errorCode()); + assertEquals(1, group.allMembers().size()); + assertEquals(0, group.allMembers().stream().filter(GenericGroupMember::isNew).count()); + assertEquals(firstMemberId, group.allMembers().iterator().next().memberId()); + } + + @Test + public void testJoinGroupInconsistentGroupProtocol() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(responseFuture.isDone()); + + JoinGroupRequestProtocolCollection otherProtocols = new JoinGroupRequestProtocolCollection(0); + otherProtocols.add(new JoinGroupRequestProtocol().setName("roundrobin")); + CompletableFuture otherResponseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin(request.setProtocols(otherProtocols), otherResponseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(otherResponseFuture.isDone()); + + assertNoOrEmptyResult(context.sleep(context.genericGroupInitialRebalanceDelayMs)); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.NONE.code(), responseFuture.get().errorCode()); + assertEquals(Errors.INCONSISTENT_GROUP_PROTOCOL.code(), otherResponseFuture.get().errorCode()); + } + + @Test + public void testJoinGroupSecondJoinInconsistentProtocol() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + context.createGenericGroup("group-id"); + + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(0); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("foo"))).array()) + ); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture, true); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.MEMBER_ID_REQUIRED.code(), responseFuture.get(5, TimeUnit.SECONDS).errorCode()); + + // Sending an inconsistent protocol should be refused + String memberId = responseFuture.get(5, TimeUnit.SECONDS).memberId(); + JoinGroupRequestProtocolCollection emptyProtocols = new JoinGroupRequestProtocolCollection(0); + request = request.setMemberId(memberId) + .setProtocols(emptyProtocols); + + responseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin(request, responseFuture, true); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.INCONSISTENT_GROUP_PROTOCOL.code(), responseFuture.get().errorCode()); + + // Sending consistent protocol should be accepted + responseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin(request.setProtocols(protocols), responseFuture, true); + + assertTrue(result.records().isEmpty()); + assertFalse(responseFuture.isDone()); + + assertNoOrEmptyResult(context.sleep(context.genericGroupInitialRebalanceDelayMs)); + + assertTrue(responseFuture.isDone()); + assertEquals(Errors.NONE.code(), responseFuture.get().errorCode()); + } + + @Test + public void testStaticMemberJoinAsFirstMember() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withGroupInstanceId("group-instance-id") + .withDefaultProtocolTypeAndProtocols() + .build(); + + context.joinGenericGroupAndCompleteJoin(request, false, true); + } + + @Test + public void testStaticMemberRejoinWithExplicitUnknownMemberId() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withGroupInstanceId("group-instance-id") + .withDefaultProtocolTypeAndProtocols() + .withSessionTimeoutMs(5000) + .withRebalanceTimeoutMs(5000) + .build(); + + JoinGroupResponseData response = context.joinGenericGroupAndCompleteJoin(request, false, true); + assertEquals(Errors.NONE.code(), response.errorCode()); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin( + request.setMemberId("unknown-member-id"), + responseFuture + ); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.FENCED_INSTANCE_ID.code(), responseFuture.get(5, TimeUnit.SECONDS).errorCode()); + } + + @Test + public void testJoinGroupUnknownConsumerExistingGroup() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .withSessionTimeoutMs(5000) + .withRebalanceTimeoutMs(5000) + .build(); + + JoinGroupResponseData response = context.joinGenericGroupAsDynamicMemberAndCompleteJoin(request); + assertEquals(Errors.NONE.code(), response.errorCode()); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request + .setMemberId("other-member-id"), responseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.UNKNOWN_MEMBER_ID.code(), responseFuture.get().errorCode()); + } + + @Test + public void testJoinGroupUnknownConsumerNewDeadGroup() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + group.transitionTo(DEAD); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.COORDINATOR_NOT_AVAILABLE.code(), responseFuture.get(5, TimeUnit.SECONDS).errorCode()); + } + + @Test + public void testJoinGroupProtocolTypeIsNotProvidedWhenAnErrorOccurs() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId("member-id") + .withDefaultProtocolTypeAndProtocols() + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.UNKNOWN_MEMBER_ID.code(), responseFuture.get().errorCode()); + assertNull(responseFuture.get().protocolType()); + } + + @Test + public void testJoinGroupReturnsTheProtocolType() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + context.createGenericGroup("group-id"); + + // Leader joins + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + CompletableFuture leaderResponseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, leaderResponseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(leaderResponseFuture.isDone()); + + // Member joins + CompletableFuture memberResponseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin(request, memberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(memberResponseFuture.isDone()); + + // Complete join group phase + assertNoOrEmptyResult(context.sleep(context.genericGroupInitialRebalanceDelayMs)); + assertTrue(leaderResponseFuture.isDone()); + assertTrue(memberResponseFuture.isDone()); + + assertEquals(Errors.NONE.code(), leaderResponseFuture.get(5, TimeUnit.SECONDS).errorCode()); + assertEquals("consumer", leaderResponseFuture.get(5, TimeUnit.SECONDS).protocolType()); + assertEquals(Errors.NONE.code(), memberResponseFuture.get(5, TimeUnit.SECONDS).errorCode()); + assertEquals("consumer", memberResponseFuture.get(5, TimeUnit.SECONDS).protocolType()); + } + + @Test + public void testDelayInitialRebalanceByGroupInitialRebalanceDelayOnEmptyGroup() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(responseFuture.isDone()); + + assertNoOrEmptyResult(context.sleep(context.genericGroupInitialRebalanceDelayMs / 2)); + assertFalse(responseFuture.isDone()); + + assertNoOrEmptyResult(context.sleep(context.genericGroupInitialRebalanceDelayMs / 2 + 1)); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.NONE.code(), responseFuture.get(5, TimeUnit.SECONDS).errorCode()); + } + + @Test + public void testResetRebalanceDelayWhenNewMemberJoinsGroupDuringInitialRebalance() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .withRebalanceTimeoutMs(context.genericGroupInitialRebalanceDelayMs * 3) + .build(); + + CompletableFuture firstMemberResponseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, firstMemberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(firstMemberResponseFuture.isDone()); + + assertNoOrEmptyResult(context.sleep(context.genericGroupInitialRebalanceDelayMs - 1)); + CompletableFuture secondMemberResponseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin(request, secondMemberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(secondMemberResponseFuture.isDone()); + assertNoOrEmptyResult(context.sleep(2)); + + // Advance clock past initial rebalance delay and verify futures are not completed. + assertNoOrEmptyResult(context.sleep(context.genericGroupInitialRebalanceDelayMs / 2 + 1)); + assertFalse(firstMemberResponseFuture.isDone()); + assertFalse(secondMemberResponseFuture.isDone()); + + // Advance clock beyond recomputed delay and make sure the futures have completed. + assertNoOrEmptyResult(context.sleep(context.genericGroupInitialRebalanceDelayMs / 2)); + assertTrue(firstMemberResponseFuture.isDone()); + assertTrue(secondMemberResponseFuture.isDone()); + assertEquals(Errors.NONE.code(), firstMemberResponseFuture.get().errorCode()); + assertEquals(Errors.NONE.code(), secondMemberResponseFuture.get().errorCode()); + } + + @Test + public void testDelayRebalanceUptoRebalanceTimeout() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .withRebalanceTimeoutMs(context.genericGroupInitialRebalanceDelayMs * 2) + .build(); + + CompletableFuture firstMemberResponseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, firstMemberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(firstMemberResponseFuture.isDone()); + + CompletableFuture secondMemberResponseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin(request, secondMemberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(secondMemberResponseFuture.isDone()); + + assertNoOrEmptyResult(context.sleep(context.genericGroupInitialRebalanceDelayMs + 1)); + + CompletableFuture thirdMemberResponseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin(request, thirdMemberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(thirdMemberResponseFuture.isDone()); + + // Advance clock right before rebalance timeout. + assertNoOrEmptyResult(context.sleep(context.genericGroupInitialRebalanceDelayMs - 1)); + assertFalse(firstMemberResponseFuture.isDone()); + assertFalse(secondMemberResponseFuture.isDone()); + assertFalse(thirdMemberResponseFuture.isDone()); + + // Advance clock beyond rebalance timeout. + assertNoOrEmptyResult(context.sleep(1)); + assertTrue(firstMemberResponseFuture.isDone()); + assertTrue(secondMemberResponseFuture.isDone()); + assertTrue(thirdMemberResponseFuture.isDone()); + + assertEquals(Errors.NONE.code(), firstMemberResponseFuture.get(5, TimeUnit.SECONDS).errorCode()); + assertEquals(Errors.NONE.code(), secondMemberResponseFuture.get(5, TimeUnit.SECONDS).errorCode()); + assertEquals(Errors.NONE.code(), thirdMemberResponseFuture.get(5, TimeUnit.SECONDS).errorCode()); + } + + @Test + public void testJoinGroupReplaceStaticMember() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withGroupInstanceId("group-instance-id") + .withDefaultProtocolTypeAndProtocols() + .withSessionTimeoutMs(5000) + .build(); + + // Send join group as static member. + CompletableFuture oldMemberResponseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, oldMemberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(oldMemberResponseFuture.isDone()); + assertEquals(1, group.numAwaitingJoinResponse()); + assertEquals(1, group.size()); + + // Replace static member with new member id. Old member id should be fenced. + CompletableFuture newMemberResponseFuture = new CompletableFuture<>(); + context.sendGenericGroupJoin(request, newMemberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(newMemberResponseFuture.isDone()); + assertTrue(oldMemberResponseFuture.isDone()); + assertEquals(Errors.FENCED_INSTANCE_ID.code(), oldMemberResponseFuture.get().errorCode()); + assertEquals(1, group.numAwaitingJoinResponse()); + assertEquals(1, group.size()); + + // Complete join for new member. + assertNoOrEmptyResult(context.sleep(context.genericGroupInitialRebalanceDelayMs)); + assertTrue(newMemberResponseFuture.isDone()); + assertEquals(Errors.NONE.code(), newMemberResponseFuture.get(5, TimeUnit.SECONDS).errorCode()); + assertEquals(0, group.numAwaitingJoinResponse()); + assertEquals(1, group.size()); + } + + @Test + public void testHeartbeatExpirationShouldRemovePendingMember() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .withSessionTimeoutMs(1000) + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture, true); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.MEMBER_ID_REQUIRED.code(), responseFuture.get().errorCode()); + assertEquals(0, group.size()); + assertEquals(1, group.numPendingJoinMembers()); + + // Advance clock by session timeout. Pending member should be removed from group as heartbeat expires. + assertNoOrEmptyResult(context.sleep(1000)); + assertEquals(0, group.numPendingJoinMembers()); + } + + @Test + public void testHeartbeatExpirationShouldRemoveMember() throws Exception { + // Set initial rebalance delay to simulate a long running rebalance. + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .withGenericGroupInitialRebalanceDelayMs(10 * 60 * 1000) + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(responseFuture.isDone()); + assertEquals(1, group.size()); + + String memberId = group.leaderOrNull(); + // Advance clock by new member join timeout. Member should be removed from group as heartbeat expires. + // A group that transitions to Empty after completing join phase will generate records. + List> timeouts = context.sleep(context.genericGroupNewMemberJoinTimeoutMs); + + List expectedRecords = Collections.singletonList(newGroupMetadataRecord("group-id", + new GroupMetadataValue() + .setMembers(Collections.emptyList()) + .setGeneration(1) + .setLeader(null) + .setProtocolType("consumer") + .setProtocol(null) + .setCurrentStateTimestamp(context.time.milliseconds()), + MetadataVersion.latest())); + + assertEquals(1, timeouts.size()); + timeouts.forEach(timeout -> { + assertEquals(genericGroupHeartbeatKey("group-id", memberId), timeout.key); + assertEquals(expectedRecords, timeout.result.records()); + }); + + assertTrue(responseFuture.isDone()); + assertEquals(Errors.UNKNOWN_MEMBER_ID.code(), responseFuture.get().errorCode()); + assertEquals(0, group.size()); + } + + @Test + public void testExistingMemberJoinDeadGroup() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + JoinGroupResponseData response = context.joinGenericGroupAsDynamicMemberAndCompleteJoin(request); + assertEquals(Errors.NONE.code(), response.errorCode()); + String memberId = response.memberId(); + + assertTrue(group.hasMemberId(memberId)); + + group.transitionTo(DEAD); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.COORDINATOR_NOT_AVAILABLE.code(), responseFuture.get().errorCode()); + } + + @Test + public void testJoinGroupExistingPendingMemberWithGroupInstanceIdThrowsException() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture, true); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.MEMBER_ID_REQUIRED.code(), responseFuture.get().errorCode()); + + String memberId = responseFuture.get(5, TimeUnit.SECONDS).memberId(); + + assertThrows(IllegalStateException.class, + () -> context.sendGenericGroupJoin( + request.setMemberId(memberId).setGroupInstanceId("group-instance-id"), + new CompletableFuture<>()) + ); + } + + @Test + public void testJoinGroupExistingMemberUpdatedMetadataTriggersRebalance() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(0); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("foo"))).array())); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withProtocolType("consumer") + .withProtocols(protocols) + .build(); + + JoinGroupResponseData response = context.joinGenericGroupAsDynamicMemberAndCompleteJoin(request); + + assertEquals(Errors.NONE.code(), response.errorCode()); + String memberId = response.memberId(); + GenericGroupMember member = group.member(memberId); + + assertEquals(protocols, member.supportedProtocols()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); + assertEquals(1, group.generationId()); + + protocols = new JoinGroupRequestProtocolCollection(0); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("foo"))).array())); + + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("bar"))).array())); + + // Send updated member metadata. This should trigger a rebalance and complete the join phase. + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin( + request.setMemberId(memberId).setProtocols(protocols), + responseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + + assertTrue(responseFuture.isDone()); + assertEquals(Errors.NONE.code(), response.errorCode()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); + assertEquals(2, group.generationId()); + assertEquals(protocols, member.supportedProtocols()); + } + + @Test + public void testJoinGroupAsExistingLeaderTriggersRebalanceInStableState() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + JoinGroupResponseData response = context.joinGenericGroupAsDynamicMemberAndCompleteJoin(request); + + assertEquals(Errors.NONE.code(), response.errorCode()); + String memberId = response.memberId(); + + assertTrue(group.isInState(COMPLETING_REBALANCE)); + assertTrue(group.isLeader(memberId)); + assertEquals(1, group.generationId()); + + group.transitionTo(STABLE); + // Sending join group as leader should trigger a rebalance. + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin( + request.setMemberId(memberId), + responseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.NONE.code(), responseFuture.get(5, TimeUnit.SECONDS).errorCode()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); + assertEquals(2, group.generationId()); + } + + @Test + public void testJoinGroupAsExistingMemberWithUpdatedMetadataTriggersRebalanceInStableState() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(0); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("foo"))).array())); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withProtocolType("consumer") + .withProtocols(protocols) + .build(); + + JoinGroupResponseData leaderResponse = context.joinGenericGroupAsDynamicMemberAndCompleteJoin(request); + assertEquals(Errors.NONE.code(), leaderResponse.errorCode()); + String leaderId = leaderResponse.leader(); + assertEquals(1, group.generationId()); + + // Member joins. + CompletableFuture memberResponseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, memberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(memberResponseFuture.isDone()); + + // Leader also rejoins. Completes join group phase. + CompletableFuture leaderResponseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin(request.setMemberId(leaderId), leaderResponseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(leaderResponseFuture.isDone()); + assertTrue(memberResponseFuture.isDone()); + assertEquals(Errors.NONE.code(), leaderResponseFuture.get().errorCode()); + assertEquals(Errors.NONE.code(), memberResponseFuture.get().errorCode()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); + assertEquals(2, group.size()); + assertEquals(2, group.generationId()); + + group.transitionTo(STABLE); + + // Member rejoins with updated metadata. This should trigger a rebalance. + String memberId = memberResponseFuture.get().memberId(); + + protocols = new JoinGroupRequestProtocolCollection(0); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("foo"))).array())); + + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("bar"))).array())); + + memberResponseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin(request.setMemberId(memberId).setProtocols(protocols), memberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(memberResponseFuture.isDone()); + + // Leader rejoins. This completes the join group phase. + leaderResponseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin(request.setMemberId(leaderId), leaderResponseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(leaderResponseFuture.isDone()); + assertTrue(memberResponseFuture.isDone()); + assertEquals(Errors.NONE.code(), memberResponseFuture.get(5, TimeUnit.SECONDS).errorCode()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); + assertEquals(3, group.generationId()); + assertEquals(2, group.size()); + } + + @Test + public void testJoinGroupExistingMemberDoesNotTriggerRebalanceInStableState() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + JoinGroupResponseData leaderResponse = context.joinGenericGroupAsDynamicMemberAndCompleteJoin(request); + assertEquals(Errors.NONE.code(), leaderResponse.errorCode()); + String leaderId = leaderResponse.leader(); + assertEquals(1, group.generationId()); + + // Member joins. + CompletableFuture memberResponseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, memberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(memberResponseFuture.isDone()); + + // Leader also rejoins. Completes join group phase. + CompletableFuture leaderResponseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin(request.setMemberId(leaderId), leaderResponseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(leaderResponseFuture.isDone()); + assertTrue(memberResponseFuture.isDone()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); + assertEquals(2, group.size()); + assertEquals(2, group.generationId()); + + String memberId = memberResponseFuture.get(5, TimeUnit.SECONDS).memberId(); + + group.transitionTo(STABLE); + + // Member rejoins with no metadata changes. This does not trigger a rebalance. + memberResponseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin(request.setMemberId(memberId), memberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(memberResponseFuture.isDone()); + assertEquals(Errors.NONE.code(), memberResponseFuture.get().errorCode()); + assertEquals(2, memberResponseFuture.get().generationId()); + assertTrue(group.isInState(STABLE)); + } + + @Test + public void testJoinGroupExistingMemberInEmptyState() throws Exception { + // Existing member joins a group that is in Empty/Dead state. Ask member to rejoin with generation id reset. + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + JoinGroupResponseData response = context.joinGenericGroupAsDynamicMemberAndCompleteJoin(request); + + assertEquals(Errors.NONE.code(), response.errorCode()); + String memberId = response.memberId(); + + assertTrue(group.isInState(COMPLETING_REBALANCE)); + + group.transitionTo(PREPARING_REBALANCE); + group.transitionTo(EMPTY); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request.setMemberId(memberId), responseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.UNKNOWN_MEMBER_ID.code(), responseFuture.get(5, TimeUnit.SECONDS).errorCode()); + assertEquals(-1, responseFuture.get(5, TimeUnit.SECONDS).generationId()); + } + + @Test + public void testCompleteJoinRemoveNotYetRejoinedDynamicMembers() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .withSessionTimeoutMs(1000) + .withRebalanceTimeoutMs(1000) + .build(); + + JoinGroupResponseData leaderResponse = context.joinGenericGroupAsDynamicMemberAndCompleteJoin(request); + assertEquals(Errors.NONE.code(), leaderResponse.errorCode()); + assertEquals(1, group.generationId()); + + // Add new member. This triggers a rebalance. + CompletableFuture memberResponseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, memberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(memberResponseFuture.isDone()); + assertEquals(2, group.size()); + assertTrue(group.isInState(PREPARING_REBALANCE)); + + // Advance clock by rebalance timeout. This will expire the leader as it has not rejoined. + assertNoOrEmptyResult(context.sleep(1000)); + + assertTrue(memberResponseFuture.isDone()); + assertEquals(Errors.NONE.code(), memberResponseFuture.get(5, TimeUnit.SECONDS).errorCode()); + assertEquals(1, group.size()); + assertTrue(group.hasMemberId(memberResponseFuture.get(5, TimeUnit.SECONDS).memberId())); + assertEquals(2, group.generationId()); + } + + @Test + public void testCompleteJoinPhaseInEmptyStateSkipsRebalance() { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .withSessionTimeoutMs(1000) + .withRebalanceTimeoutMs(1000) + .build(); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(responseFuture.isDone()); + + assertEquals(0, group.generationId()); + assertTrue(group.isInState(PREPARING_REBALANCE)); + group.transitionTo(DEAD); + + // Advance clock by initial rebalance delay to complete join phase. + assertNoOrEmptyResult(context.sleep(context.genericGroupInitialRebalanceDelayMs)); + assertEquals(0, group.generationId()); + } + + @Test + public void testCompleteJoinPhaseNoMembersRejoinedExtendsJoinPhase() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withGroupInstanceId("first-instance-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .withSessionTimeoutMs(30000) + .withRebalanceTimeoutMs(10000) + .build(); + + // First member joins group and completes join phase. + JoinGroupResponseData firstMemberResponse = context.joinGenericGroupAndCompleteJoin(request, true, true); + assertEquals(Errors.NONE.code(), firstMemberResponse.errorCode()); + String firstMemberId = firstMemberResponse.memberId(); + + // Second member joins and group goes into rebalancing state. + CompletableFuture secondMemberResponseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin( + request.setGroupInstanceId("second-instance-id"), secondMemberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(secondMemberResponseFuture.isDone()); + + // First static member rejoins and completes join phase. + CompletableFuture firstMemberResponseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin( + request.setMemberId(firstMemberId).setGroupInstanceId("first-instance-id"), + firstMemberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(firstMemberResponseFuture.isDone()); + assertTrue(secondMemberResponseFuture.isDone()); + assertEquals(Errors.NONE.code(), firstMemberResponseFuture.get(5, TimeUnit.SECONDS).errorCode()); + assertEquals(Errors.NONE.code(), secondMemberResponseFuture.get(5, TimeUnit.SECONDS).errorCode()); + assertEquals(2, group.size()); + assertEquals(2, group.generationId()); + + String secondMemberId = secondMemberResponseFuture.get(5, TimeUnit.SECONDS).memberId(); + + // Trigger a rebalance. No members rejoined. + context.groupMetadataManager.prepareRebalance(group, "trigger rebalance"); + + assertEquals(2, group.size()); + assertTrue(group.isInState(PREPARING_REBALANCE)); + assertEquals(0, group.numAwaitingJoinResponse()); + + // Advance clock by rebalance timeout to complete join phase. As long as both members have not + // rejoined, we extend the join phase. + assertNoOrEmptyResult(context.sleep(10000)); + assertEquals(10000, context.timer.timeout("join-group-id").deadlineMs - context.time.milliseconds()); + assertNoOrEmptyResult(context.sleep(10000)); + assertEquals(10000, context.timer.timeout("join-group-id").deadlineMs - context.time.milliseconds()); + + assertTrue(group.isInState(PREPARING_REBALANCE)); + assertEquals(2, group.size()); + assertEquals(2, group.generationId()); + + // Let first and second member rejoin. This should complete the join phase. + firstMemberResponseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin( + request.setMemberId(firstMemberId).setGroupInstanceId("first-instance-id"), + firstMemberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertFalse(firstMemberResponseFuture.isDone()); + assertTrue(group.isInState(PREPARING_REBALANCE)); + assertEquals(2, group.size()); + assertEquals(2, group.generationId()); + + secondMemberResponseFuture = new CompletableFuture<>(); + result = context.sendGenericGroupJoin( + request.setMemberId(secondMemberId).setGroupInstanceId("second-instance-id"), + secondMemberResponseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(firstMemberResponseFuture.isDone()); + assertTrue(secondMemberResponseFuture.isDone()); + assertEquals(Errors.NONE.code(), firstMemberResponseFuture.get(5, TimeUnit.SECONDS).errorCode()); + assertEquals(Errors.NONE.code(), secondMemberResponseFuture.get(5, TimeUnit.SECONDS).errorCode()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); + assertEquals(2, group.size()); + assertEquals(3, group.generationId()); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testReplaceStaticMemberInStableStateNoError( + boolean supportSkippingAssignment + ) throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(0); + + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("foo"))).array()) + ); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withGroupInstanceId("group-instance-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withProtocolType("consumer") + .withProtocols(protocols) + .build(); + + JoinGroupResponseData response = context.joinGenericGroupAndCompleteJoin(request, true, supportSkippingAssignment); + assertEquals(Errors.NONE.code(), response.errorCode()); + String oldMemberId = response.memberId(); + + assertEquals(1, group.size()); + assertEquals(1, group.generationId()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); + + // Simulate successful sync group phase + group.transitionTo(STABLE); + + // Static member rejoins with UNKNOWN_MEMBER_ID. This should update the log with the generated member id. + protocols = new JoinGroupRequestProtocolCollection(0); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("foo"))).array()) + ); + + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("bar"))).array())); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin( + request + .setProtocols(protocols) + .setRebalanceTimeoutMs(7000) + .setSessionTimeoutMs(4500), + responseFuture, + true, + supportSkippingAssignment); + + assertEquals( + Collections.singletonList(RecordHelpers.newGroupMetadataRecord(group, MetadataVersion.latest())), + result.records() + ); + assertFalse(responseFuture.isDone()); + + // Write was successful. + result.appendFuture().complete(null); + assertTrue(responseFuture.isDone()); + + String newMemberId = group.staticMemberId("group-instance-id"); + + JoinGroupResponseData expectedResponse = new JoinGroupResponseData() + .setMembers(Collections.emptyList()) + .setLeader(oldMemberId) + .setMemberId(newMemberId) + .setGenerationId(1) + .setProtocolType("consumer") + .setProtocolName("range") + .setSkipAssignment(supportSkippingAssignment) + .setErrorCode(Errors.NONE.code()); + + if (supportSkippingAssignment) { + expectedResponse + .setMembers(Collections.singletonList( + new JoinGroupResponseData.JoinGroupResponseMember() + .setMemberId(newMemberId) + .setGroupInstanceId("group-instance-id") + .setMetadata(protocols.find("range").metadata()) + )) + .setLeader(newMemberId); + } + + GenericGroupMember updatedMember = group.member(group.staticMemberId("group-instance-id")); + + assertEquals(expectedResponse, responseFuture.get()); + assertEquals(newMemberId, updatedMember.memberId()); + assertEquals(Optional.of("group-instance-id"), updatedMember.groupInstanceId()); + assertEquals(7000, updatedMember.rebalanceTimeoutMs()); + assertEquals(4500, updatedMember.sessionTimeoutMs()); + assertEquals(protocols, updatedMember.supportedProtocols()); + + assertEquals(1, group.size()); + assertEquals(1, group.generationId()); + assertTrue(group.isInState(STABLE)); + } + + @Test + public void testReplaceStaticMemberInStableStateWithUpdatedProtocolTriggersRebalance() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(0); + + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("foo"))).array()) + ); + + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("bar"))).array()) + ); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withGroupInstanceId("group-instance-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withProtocolType("consumer") + .withProtocols(protocols) + .build(); + + JoinGroupResponseData response = context.joinGenericGroupAndCompleteJoin(request, true, true); + assertEquals(Errors.NONE.code(), response.errorCode()); + assertEquals(1, group.size()); + assertEquals(1, group.generationId()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); + + // Simulate successful sync group phase + group.transitionTo(STABLE); + + // Static member rejoins with UNKNOWN_MEMBER_ID. The selected protocol changes and triggers a rebalance. + protocols = new JoinGroupRequestProtocolCollection(0); + + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("bar"))).array()) + ); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request.setProtocols(protocols), responseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.NONE.code(), responseFuture.get(5, TimeUnit.SECONDS).errorCode()); + assertEquals(1, group.size()); + assertEquals(2, group.generationId()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); + } + + @Test + public void testReplaceStaticMemberInStableStateErrors() throws Exception { + // If the append future fails, we need to revert the soft state to the original member. + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(0); + + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("foo"))).array()) + ); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withGroupInstanceId("group-instance-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withProtocolType("consumer") + .withProtocols(protocols) + .build(); + + JoinGroupResponseData response = context.joinGenericGroupAndCompleteJoin(request, false, false); + assertEquals(Errors.NONE.code(), response.errorCode()); + GenericGroupMember oldMember = group.member(response.memberId()); + assertEquals(1, group.size()); + assertEquals(1, group.generationId()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); + + // Simulate successful sync group phase + group.transitionTo(STABLE); + + // Static member rejoins with UNKNOWN_MEMBER_ID but the append fails. This reverts the soft state of the group. + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("bar"))).array())); + + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin( + request + .setProtocols(protocols) + .setRebalanceTimeoutMs(7000) + .setSessionTimeoutMs(6000), + responseFuture, + false, + false); + + assertEquals( + Collections.singletonList(RecordHelpers.newGroupMetadataRecord(group, MetadataVersion.latest())), + result.records() + ); + assertFalse(responseFuture.isDone()); + + // Simulate failed write to log. + result.appendFuture().completeExceptionally(new UnknownTopicOrPartitionException()); + assertTrue(responseFuture.isDone()); + + JoinGroupResponseData expectedResponse = new JoinGroupResponseData() + .setMembers(Collections.emptyList()) + .setLeader(oldMember.memberId()) + .setMemberId(UNKNOWN_MEMBER_ID) + .setGenerationId(1) + .setProtocolType("consumer") + .setProtocolName("range") + .setSkipAssignment(false) + .setErrorCode(Errors.COORDINATOR_NOT_AVAILABLE.code()); + + assertEquals(expectedResponse, responseFuture.get()); + + GenericGroupMember revertedMember = group.member(group.staticMemberId("group-instance-id")); + + assertEquals(oldMember.memberId(), revertedMember.memberId()); + assertEquals(oldMember.groupInstanceId(), revertedMember.groupInstanceId()); + assertEquals(oldMember.rebalanceTimeoutMs(), revertedMember.rebalanceTimeoutMs()); + assertEquals(oldMember.sessionTimeoutMs(), revertedMember.sessionTimeoutMs()); + assertEquals(oldMember.supportedProtocols(), revertedMember.supportedProtocols()); + assertEquals(1, group.size()); + assertEquals(1, group.generationId()); + assertTrue(group.isInState(STABLE)); + } + + @Test + public void testReplaceStaticMemberInCompletingRebalanceStateTriggersRebalance() throws Exception { + GroupMetadataManagerTestContext context = new GroupMetadataManagerTestContext.Builder() + .build(); + GenericGroup group = context.createGenericGroup("group-id"); + + JoinGroupRequestData request = new JoinGroupRequestBuilder() + .withGroupId("group-id") + .withGroupInstanceId("group-instance-id") + .withMemberId(UNKNOWN_MEMBER_ID) + .withDefaultProtocolTypeAndProtocols() + .build(); + + JoinGroupResponseData response = context.joinGenericGroupAndCompleteJoin(request, true, true); + assertEquals(Errors.NONE.code(), response.errorCode()); + + assertEquals(1, group.size()); + assertEquals(1, group.generationId()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); + + // Static member rejoins with UNKNOWN_MEMBER_ID and triggers a rebalance. + CompletableFuture responseFuture = new CompletableFuture<>(); + CoordinatorResult result = context.sendGenericGroupJoin(request, responseFuture); + + assertTrue(result.records().isEmpty()); + assertTrue(responseFuture.isDone()); + assertEquals(Errors.NONE.code(), responseFuture.get(5, TimeUnit.SECONDS).errorCode()); + assertEquals(1, group.size()); + assertEquals(2, group.generationId()); + assertTrue(group.isInState(COMPLETING_REBALANCE)); + } + + @Test + public void testJoinGroupAppendErrorConversion() { + assertEquals(Errors.COORDINATOR_NOT_AVAILABLE, appendGroupMetadataErrorToResponseError(Errors.UNKNOWN_TOPIC_OR_PARTITION)); + assertEquals(Errors.COORDINATOR_NOT_AVAILABLE, appendGroupMetadataErrorToResponseError(Errors.NOT_ENOUGH_REPLICAS)); + + assertEquals(Errors.NOT_COORDINATOR, appendGroupMetadataErrorToResponseError(Errors.NOT_LEADER_OR_FOLLOWER)); + assertEquals(Errors.NOT_COORDINATOR, appendGroupMetadataErrorToResponseError(Errors.KAFKA_STORAGE_ERROR)); + + assertEquals(Errors.UNKNOWN_SERVER_ERROR, appendGroupMetadataErrorToResponseError(Errors.MESSAGE_TOO_LARGE)); + assertEquals(Errors.UNKNOWN_SERVER_ERROR, appendGroupMetadataErrorToResponseError(Errors.RECORD_LIST_TOO_LARGE)); + assertEquals(Errors.UNKNOWN_SERVER_ERROR, appendGroupMetadataErrorToResponseError(Errors.INVALID_FETCH_SIZE)); + + assertEquals(Errors.LEADER_NOT_AVAILABLE, Errors.LEADER_NOT_AVAILABLE); + } + + private void assertUnorderedListEquals( + List expected, + List actual + ) { + assertEquals(new HashSet<>(expected), new HashSet<>(actual)); + } + + private void assertResponseEquals( + ConsumerGroupHeartbeatResponseData expected, + ConsumerGroupHeartbeatResponseData actual + ) { + if (!responseEquals(expected, actual)) { + assertionFailure() + .expected(expected) + .actual(actual) + .buildAndThrow(); + } + } + + private boolean responseEquals( + ConsumerGroupHeartbeatResponseData expected, + ConsumerGroupHeartbeatResponseData actual + ) { + if (expected.throttleTimeMs() != actual.throttleTimeMs()) return false; + if (expected.errorCode() != actual.errorCode()) return false; + if (!Objects.equals(expected.errorMessage(), actual.errorMessage())) return false; + if (!Objects.equals(expected.memberId(), actual.memberId())) return false; + if (expected.memberEpoch() != actual.memberEpoch()) return false; + if (expected.shouldComputeAssignment() != actual.shouldComputeAssignment()) return false; + if (expected.heartbeatIntervalMs() != actual.heartbeatIntervalMs()) return false; + // Unordered comparison of the assignments. + return responseAssignmentEquals(expected.assignment(), actual.assignment()); + } + + private boolean responseAssignmentEquals( + ConsumerGroupHeartbeatResponseData.Assignment expected, + ConsumerGroupHeartbeatResponseData.Assignment actual + ) { + if (expected == actual) return true; + if (expected == null) return false; + if (actual == null) return false; + + if (!Objects.equals(fromAssignment(expected.pendingTopicPartitions()), fromAssignment(actual.pendingTopicPartitions()))) + return false; + + return Objects.equals(fromAssignment(expected.assignedTopicPartitions()), fromAssignment(actual.assignedTopicPartitions())); + } + + private Map> fromAssignment( + List assignment + ) { + if (assignment == null) return null; + + Map> assignmentMap = new HashMap<>(); + assignment.forEach(topicPartitions -> { + assignmentMap.put(topicPartitions.topicId(), new HashSet<>(topicPartitions.partitions())); + }); + return assignmentMap; + } + + private void assertRecordsEquals( + List expectedRecords, + List actualRecords + ) { + try { + assertEquals(expectedRecords.size(), actualRecords.size()); + + for (int i = 0; i < expectedRecords.size(); i++) { + Record expectedRecord = expectedRecords.get(i); + Record actualRecord = actualRecords.get(i); + assertRecordEquals(expectedRecord, actualRecord); + } + } catch (AssertionFailedError e) { + assertionFailure() + .expected(expectedRecords) + .actual(actualRecords) + .buildAndThrow(); + } + } + + private void assertRecordEquals( + Record expected, + Record actual + ) { + try { + assertApiMessageAndVersionEquals(expected.key(), actual.key()); + assertApiMessageAndVersionEquals(expected.value(), actual.value()); + } catch (AssertionFailedError e) { + assertionFailure() + .expected(expected) + .actual(actual) + .buildAndThrow(); + } + } + + private void assertApiMessageAndVersionEquals( + ApiMessageAndVersion expected, + ApiMessageAndVersion actual + ) { + if (expected == actual) return; + + assertEquals(expected.version(), actual.version()); + + if (actual.message() instanceof ConsumerGroupCurrentMemberAssignmentValue) { + // The order of the topics stored in ConsumerGroupCurrentMemberAssignmentValue is not + // always guaranteed. Therefore, we need a special comparator. + ConsumerGroupCurrentMemberAssignmentValue expectedValue = + (ConsumerGroupCurrentMemberAssignmentValue) expected.message(); + ConsumerGroupCurrentMemberAssignmentValue actualValue = + (ConsumerGroupCurrentMemberAssignmentValue) actual.message(); + + assertEquals(expectedValue.memberEpoch(), actualValue.memberEpoch()); + assertEquals(expectedValue.previousMemberEpoch(), actualValue.previousMemberEpoch()); + assertEquals(expectedValue.targetMemberEpoch(), actualValue.targetMemberEpoch()); + assertEquals(expectedValue.error(), actualValue.error()); + assertEquals(expectedValue.metadataVersion(), actualValue.metadataVersion()); + assertEquals(expectedValue.metadataBytes(), actualValue.metadataBytes()); + + // We transform those to Maps before comparing them. + assertEquals(fromTopicPartitions(expectedValue.assignedPartitions()), + fromTopicPartitions(actualValue.assignedPartitions())); + assertEquals(fromTopicPartitions(expectedValue.partitionsPendingRevocation()), + fromTopicPartitions(actualValue.partitionsPendingRevocation())); + assertEquals(fromTopicPartitions(expectedValue.partitionsPendingAssignment()), + fromTopicPartitions(actualValue.partitionsPendingAssignment())); + } else { + assertEquals(expected.message(), actual.message()); + } + } + + private Map> fromTopicPartitions( + List assignment + ) { + Map> assignmentMap = new HashMap<>(); + assignment.forEach(topicPartitions -> { + assignmentMap.put(topicPartitions.topicId(), new HashSet<>(topicPartitions.partitions())); + }); + return assignmentMap; + } + + private List verifyGenericGroupJoinResponses( + List> responseFutures, + int expectedSuccessCount, + Errors expectedFailure + ) { + int successCount = 0; + List memberIds = new ArrayList<>(); + for (CompletableFuture responseFuture : responseFutures) { + if (!responseFuture.isDone()) { + fail("All responseFutures should be completed."); + } + try { + JoinGroupResponseData joinResponse = responseFuture.get(); + if (joinResponse.errorCode() == Errors.NONE.code()) { + successCount++; + } else { + assertEquals(expectedFailure.code(), joinResponse.errorCode()); + } + memberIds.add(joinResponse.memberId()); + } catch (Exception e) { + fail("Unexpected exception: " + e.getMessage()); + } + } + + assertEquals(expectedSuccessCount, successCount); + return memberIds; + } + + private void assertNoOrEmptyResult(List> timeouts) { + assertTrue(timeouts.size() <= 1); + timeouts.forEach(timeout -> assertEquals(EMPTY_RESULT, timeout.result)); + } + + private static class JoinGroupRequestBuilder { + String groupId = null; + String groupInstanceId = null; + String memberId = null; + String protocolType = "consumer"; + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(0); + int sessionTimeoutMs = 500; + int rebalanceTimeoutMs = 500; + String reason = null; + + JoinGroupRequestBuilder withGroupId(String groupId) { + this.groupId = groupId; + return this; + } + + JoinGroupRequestBuilder withGroupInstanceId(String groupInstanceId) { + this.groupInstanceId = groupInstanceId; + return this; + } + + JoinGroupRequestBuilder withMemberId(String memberId) { + this.memberId = memberId; + return this; + } + + JoinGroupRequestBuilder withDefaultProtocolTypeAndProtocols() { + this.protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(ConsumerProtocol.serializeSubscription(new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("foo"))).array()) + ); + return this; + } + + JoinGroupRequestBuilder withProtocolType(String protocolType) { + this.protocolType = protocolType; + return this; + } + + JoinGroupRequestBuilder withProtocols(JoinGroupRequestProtocolCollection protocols) { + this.protocols = protocols; + return this; + } + + JoinGroupRequestBuilder withRebalanceTimeoutMs(int rebalanceTimeoutMs) { + this.rebalanceTimeoutMs = rebalanceTimeoutMs; + return this; + } + + JoinGroupRequestBuilder withSessionTimeoutMs(int sessionTimeoutMs) { + this.sessionTimeoutMs = sessionTimeoutMs; + return this; + } + + JoinGroupRequestBuilder withReason(String reason) { + this.reason = reason; + return this; + } + + JoinGroupRequestData build() { + return new JoinGroupRequestData() + .setGroupId(groupId) + .setGroupInstanceId(groupInstanceId) + .setMemberId(memberId) + .setProtocolType(protocolType) + .setProtocols(protocols) + .setRebalanceTimeoutMs(rebalanceTimeoutMs) + .setSessionTimeoutMs(sessionTimeoutMs) + .setReason(reason); + } + } + + private static Record newGroupMetadataRecord( + String groupId, + GroupMetadataValue value, + MetadataVersion metadataVersion + ) { + return new Record( + new ApiMessageAndVersion( + new GroupMetadataKey() + .setGroup(groupId), + (short) 2 + ), + new ApiMessageAndVersion( + value, + metadataVersion.groupMetadataValueVersion() + ) + ); + } +} + diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/MockCoordinatorTimer.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/MockCoordinatorTimer.java index 01d692b012eb6..a8078c79a0287 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/MockCoordinatorTimer.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/MockCoordinatorTimer.java @@ -17,6 +17,7 @@ package org.apache.kafka.coordinator.group; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.coordinator.group.runtime.CoordinatorResult; import org.apache.kafka.coordinator.group.runtime.CoordinatorTimer; import java.util.ArrayList; @@ -33,19 +34,19 @@ * expire timeouts. They are only expired when {@link MockCoordinatorTimer#poll()} * is called. */ -public class MockCoordinatorTimer implements CoordinatorTimer { +public class MockCoordinatorTimer implements CoordinatorTimer { /** * Represents a scheduled timeout. */ - public static class ScheduledTimeout { + public static class ScheduledTimeout { public final String key; public final long deadlineMs; - public final TimeoutOperation operation; + public final TimeoutOperation operation; ScheduledTimeout( String key, long deadlineMs, - TimeoutOperation operation + TimeoutOperation operation ) { this.key = key; this.deadlineMs = deadlineMs; @@ -56,16 +57,16 @@ public static class ScheduledTimeout { /** * Represents an expired timeout. */ - public static class ExpiredTimeout { + public static class ExpiredTimeout { public final String key; - public final List records; + public final CoordinatorResult result; ExpiredTimeout( String key, - List records + CoordinatorResult result ) { this.key = key; - this.records = records; + this.result = result; } @Override @@ -73,24 +74,24 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - ExpiredTimeout that = (ExpiredTimeout) o; + ExpiredTimeout that = (ExpiredTimeout) o; if (!Objects.equals(key, that.key)) return false; - return Objects.equals(records, that.records); + return Objects.equals(result, that.result); } @Override public int hashCode() { - int result = key != null ? key.hashCode() : 0; - result = 31 * result + (records != null ? records.hashCode() : 0); - return result; + int result1 = key != null ? key.hashCode() : 0; + result1 = 31 * result1 + (result != null ? result.hashCode() : 0); + return result1; } } private final Time time; - private final Map> timeoutMap = new HashMap<>(); - private final PriorityQueue> timeoutQueue = new PriorityQueue<>( + private final Map> timeoutMap = new HashMap<>(); + private final PriorityQueue> timeoutQueue = new PriorityQueue<>( Comparator.comparingLong(entry -> entry.deadlineMs) ); @@ -107,12 +108,12 @@ public void schedule( long delay, TimeUnit unit, boolean retry, - TimeoutOperation operation + TimeoutOperation operation ) { cancel(key); long deadlineMs = time.milliseconds() + unit.toMillis(delay); - ScheduledTimeout timeout = new ScheduledTimeout<>(key, deadlineMs, operation); + ScheduledTimeout timeout = new ScheduledTimeout<>(key, deadlineMs, operation); timeoutQueue.add(timeout); timeoutMap.put(key, timeout); } @@ -122,7 +123,7 @@ public void schedule( */ @Override public void cancel(String key) { - ScheduledTimeout timeout = timeoutMap.remove(key); + ScheduledTimeout timeout = timeoutMap.remove(key); if (timeout != null) { timeoutQueue.remove(timeout); } @@ -138,7 +139,7 @@ public boolean contains(String key) { /** * @return The scheduled timeout for the key; null otherwise. */ - public ScheduledTimeout timeout(String key) { + public ScheduledTimeout timeout(String key) { return timeoutMap.get(key); } @@ -152,10 +153,10 @@ public int size() { /** * @return A list of expired timeouts based on the current time. */ - public List> poll() { - List> results = new ArrayList<>(); + public List> poll() { + List> results = new ArrayList<>(); - ScheduledTimeout timeout = timeoutQueue.peek(); + ScheduledTimeout timeout = timeoutQueue.peek(); while (timeout != null && timeout.deadlineMs <= time.milliseconds()) { timeoutQueue.poll(); timeoutMap.remove(timeout.key, timeout); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/RecordHelpersTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/RecordHelpersTest.java index cfa2d600f7c33..2b436672b0ed5 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/RecordHelpersTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/RecordHelpersTest.java @@ -17,6 +17,8 @@ package org.apache.kafka.coordinator.group; import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocol; +import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Time; @@ -41,7 +43,6 @@ import org.apache.kafka.coordinator.group.generic.GenericGroup; import org.apache.kafka.coordinator.group.generic.GenericGroupMember; import org.apache.kafka.coordinator.group.generic.GenericGroupState; -import org.apache.kafka.coordinator.group.generic.Protocol; import org.apache.kafka.server.common.ApiMessageAndVersion; import org.apache.kafka.server.common.MetadataVersion; import org.junit.jupiter.api.Test; @@ -467,6 +468,11 @@ public void testNewGroupMetadataRecord( ); expectedMembers.forEach(member -> { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(member.subscription())); + group.add(new GenericGroupMember( member.memberId(), Optional.of(member.groupInstanceId()), @@ -475,10 +481,7 @@ public void testNewGroupMetadataRecord( member.rebalanceTimeout(), member.sessionTimeout(), "consumer", - Collections.singletonList(new Protocol( - "range", - member.subscription() - )), + protocols, member.assignment() )); }); @@ -530,6 +533,11 @@ public void testNewGroupMetadataRecordThrowsWhenNullSubscription() { ); expectedMembers.forEach(member -> { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(null)); + group.add(new GenericGroupMember( member.memberId(), Optional.of(member.groupInstanceId()), @@ -538,10 +546,7 @@ public void testNewGroupMetadataRecordThrowsWhenNullSubscription() { member.rebalanceTimeout(), member.sessionTimeout(), "consumer", - Collections.singletonList(new Protocol( - "range", - null - )), + protocols, member.assignment() )); }); @@ -577,7 +582,12 @@ public void testNewGroupMetadataRecordThrowsWhenEmptyAssignment() { time ); - expectedMembers.forEach(member -> + expectedMembers.forEach(member -> { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(member.subscription())); + group.add(new GenericGroupMember( member.memberId(), Optional.of(member.groupInstanceId()), @@ -586,13 +596,10 @@ public void testNewGroupMetadataRecordThrowsWhenEmptyAssignment() { member.rebalanceTimeout(), member.sessionTimeout(), "consumer", - Collections.singletonList(new Protocol( - "range", - member.subscription() - )), + protocols, member.assignment() - )) - ); + )); + }); assertThrows(IllegalStateException.class, () -> RecordHelpers.newGroupMetadataRecord( @@ -600,4 +607,45 @@ public void testNewGroupMetadataRecordThrowsWhenEmptyAssignment() { MetadataVersion.IBP_3_5_IV2 )); } + + @ParameterizedTest + @MethodSource("metadataToExpectedGroupMetadataValue") + public void testEmptyGroupMetadataRecord( + MetadataVersion metadataVersion, + short expectedGroupMetadataValueVersion + ) { + Time time = new MockTime(); + + List expectedMembers = Collections.emptyList(); + + Record expectedRecord = new Record( + new ApiMessageAndVersion( + new GroupMetadataKey() + .setGroup("group-id"), + (short) 2), + new ApiMessageAndVersion( + new GroupMetadataValue() + .setProtocol(null) + .setProtocolType("") + .setLeader(null) + .setGeneration(0) + .setCurrentStateTimestamp(time.milliseconds()) + .setMembers(expectedMembers), + expectedGroupMetadataValueVersion)); + + GenericGroup group = new GenericGroup( + new LogContext(), + "group-id", + GenericGroupState.PREPARING_REBALANCE, + time + ); + + group.initNextGeneration(); + Record groupMetadataRecord = RecordHelpers.newEmptyGroupMetadataRecord( + group, + metadataVersion + ); + + assertEquals(expectedRecord, groupMetadataRecord); + } } diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/ReplicatedGroupCoordinatorTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/ReplicatedGroupCoordinatorTest.java index 7cc26dd7a3619..280262c02a4bd 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/ReplicatedGroupCoordinatorTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/ReplicatedGroupCoordinatorTest.java @@ -32,6 +32,8 @@ import org.apache.kafka.coordinator.group.generated.ConsumerGroupTargetAssignmentMemberValue; import org.apache.kafka.coordinator.group.generated.ConsumerGroupTargetAssignmentMetadataKey; import org.apache.kafka.coordinator.group.generated.ConsumerGroupTargetAssignmentMetadataValue; +import org.apache.kafka.coordinator.group.generated.GroupMetadataKey; +import org.apache.kafka.coordinator.group.generated.GroupMetadataValue; import org.apache.kafka.coordinator.group.runtime.CoordinatorResult; import org.apache.kafka.image.MetadataImage; import org.apache.kafka.server.common.ApiMessageAndVersion; @@ -326,4 +328,39 @@ public void testOnLoaded() { verify(groupMetadataManager, times(1)).onLoaded(); } + + @Test + public void testReplayGroupMetadata() { + GroupMetadataManager groupMetadataManager = mock(GroupMetadataManager.class); + ReplicatedGroupCoordinator coordinator = new ReplicatedGroupCoordinator( + groupMetadataManager + ); + + GroupMetadataKey key = new GroupMetadataKey(); + GroupMetadataValue value = new GroupMetadataValue(); + + coordinator.replay(new Record( + new ApiMessageAndVersion(key, (short) 2), + new ApiMessageAndVersion(value, (short) 4) + )); + + verify(groupMetadataManager, times(1)).replay(key, value); + } + + @Test + public void testReplayGroupMetadataWithNullValue() { + GroupMetadataManager groupMetadataManager = mock(GroupMetadataManager.class); + ReplicatedGroupCoordinator coordinator = new ReplicatedGroupCoordinator( + groupMetadataManager + ); + + GroupMetadataKey key = new GroupMetadataKey(); + + coordinator.replay(new Record( + new ApiMessageAndVersion(key, (short) 2), + null + )); + + verify(groupMetadataManager, times(1)).replay(key, null); + } } diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/generic/GenericGroupMemberTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/generic/GenericGroupMemberTest.java index 4a561b4ff0c7a..7263f5a07d9ad 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/generic/GenericGroupMemberTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/generic/GenericGroupMemberTest.java @@ -17,18 +17,17 @@ package org.apache.kafka.coordinator.group.generic; +import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocol; +import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection; import org.junit.jupiter.api.Test; -import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; -import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import static org.apache.kafka.coordinator.group.generic.GenericGroupMember.EMPTY_ASSIGNMENT; -import static org.apache.kafka.coordinator.group.generic.Protocol.EMPTY_METADATA; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -39,6 +38,11 @@ public class GenericGroupMemberTest { @Test public void testMatchesSupportedProtocols() { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(new byte[]{0})); + GenericGroupMember member = new GenericGroupMember( "member", Optional.of("group-instance-id"), @@ -47,33 +51,54 @@ public void testMatchesSupportedProtocols() { 10, 4500, "generic", - Collections.singletonList( - new Protocol("range", new byte[] {0}) - ), - EMPTY_METADATA + protocols, + new byte[0] ); - assertTrue(member.matches(Collections.singletonList( - new Protocol("range", new byte[] {0}) - ))); - assertFalse(member.matches(Collections.singletonList( - new Protocol("range", new byte[] {1}) - ))); - assertFalse(member.matches(Collections.singletonList( - new Protocol("roundrobin", EMPTY_METADATA) - ))); - - List protocols = new ArrayList<>(); - protocols.add(new Protocol("range", EMPTY_METADATA)); - protocols.add(new Protocol("roundrobin", EMPTY_METADATA)); - assertFalse(member.matches(protocols)); + JoinGroupRequestProtocolCollection collection = new JoinGroupRequestProtocolCollection(); + collection.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(new byte[] {0})); + + assertTrue(member.matches(collection)); + + collection = new JoinGroupRequestProtocolCollection(); + collection.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(new byte[] {1})); + + assertFalse(member.matches(collection)); + + collection = new JoinGroupRequestProtocolCollection(); + collection.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(new byte[0])); + + assertFalse(member.matches(collection)); + + collection = new JoinGroupRequestProtocolCollection(); + collection.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(new byte[0])); + + collection.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + + assertFalse(member.matches(collection)); + assertTrue(member.matches(protocols)); } @Test public void testVoteForPreferredProtocol() { - List protocols = new ArrayList<>(); - protocols.add(new Protocol("range", EMPTY_METADATA)); - protocols.add(new Protocol("roundrobin", EMPTY_METADATA)); + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(new byte[0])); + + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); GenericGroupMember member = new GenericGroupMember( "member", @@ -101,9 +126,13 @@ public void testVoteForPreferredProtocol() { @Test public void testMetadata() { - List protocols = new ArrayList<>(); - protocols.add(new Protocol("range", new byte[]{0})); - protocols.add(new Protocol("roundrobin", new byte[]{1})); + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(new byte[]{0})); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[]{1})); GenericGroupMember member = new GenericGroupMember( "member", @@ -123,10 +152,6 @@ public void testMetadata() { @Test public void testMetadataRaisesOnUnsupportedProtocol() { - List protocols = new ArrayList<>(); - protocols.add(new Protocol("range", new byte[]{0})); - protocols.add(new Protocol("roundrobin", new byte[]{1})); - GenericGroupMember member = new GenericGroupMember( "member", Optional.of("group-instance-id"), @@ -135,7 +160,7 @@ public void testMetadataRaisesOnUnsupportedProtocol() { 10, 4500, "generic", - protocols, + new JoinGroupRequestProtocolCollection(), EMPTY_ASSIGNMENT ); @@ -146,9 +171,13 @@ public void testMetadataRaisesOnUnsupportedProtocol() { @Test public void testVoteRaisesOnNoSupportedProtocols() { - List protocols = new ArrayList<>(); - protocols.add(new Protocol("range", new byte[]{0})); - protocols.add(new Protocol("roundrobin", new byte[]{1})); + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(new byte[]{0})); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[]{1})); GenericGroupMember member = new GenericGroupMember( "member", @@ -169,10 +198,6 @@ public void testVoteRaisesOnNoSupportedProtocols() { @Test public void testHasValidGroupInstanceId() { - List protocols = new ArrayList<>(); - protocols.add(new Protocol("range", new byte[]{0})); - protocols.add(new Protocol("roundrobin", new byte[]{1})); - GenericGroupMember member = new GenericGroupMember( "member", Optional.of("group-instance-id"), @@ -181,7 +206,7 @@ public void testHasValidGroupInstanceId() { 10, 4500, "generic", - protocols, + new JoinGroupRequestProtocolCollection(), EMPTY_ASSIGNMENT ); @@ -191,15 +216,19 @@ public void testHasValidGroupInstanceId() { @Test public void testPlainProtocolSet() { - List protocols = new ArrayList<>(); - protocols.add(new Protocol("range", new byte[]{0})); - protocols.add(new Protocol("roundrobin", new byte[]{1})); + JoinGroupRequestProtocolCollection protocolCollection = + new JoinGroupRequestProtocolCollection(); + + protocolCollection.add(new JoinGroupRequestProtocol() + .setName("range").setMetadata(new byte[]{0})); + protocolCollection.add(new JoinGroupRequestProtocol() + .setName("roundrobin").setMetadata(new byte[]{1})); Set expectedProtocolNames = new HashSet<>(); expectedProtocolNames.add("range"); expectedProtocolNames.add("roundrobin"); - - assertEquals(expectedProtocolNames, GenericGroupMember.plainProtocolSet(protocols)); + + assertEquals(expectedProtocolNames, GenericGroupMember.plainProtocolSet(protocolCollection)); } @Test @@ -212,19 +241,13 @@ public void testHasHeartbeatSatisfied() { 10, 4500, "generic", - Collections.singletonList( - new Protocol("range", EMPTY_METADATA) - ), + new JoinGroupRequestProtocolCollection(), EMPTY_ASSIGNMENT ); assertFalse(member.hasSatisfiedHeartbeat()); - member.setHeartBeatSatisfied(true); - assertTrue(member.hasSatisfiedHeartbeat()); - member.setIsNew(true); - member.setHeartBeatSatisfied(false); assertFalse(member.hasSatisfiedHeartbeat()); member.setIsNew(false); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/generic/GenericGroupTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/generic/GenericGroupTest.java index ad46c783abb50..ddc5870b8def7 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/generic/GenericGroupTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/generic/GenericGroupTest.java @@ -18,6 +18,8 @@ import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor; import org.apache.kafka.clients.consumer.internals.ConsumerProtocol; +import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocol; +import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection; import org.apache.kafka.common.message.JoinGroupResponseData; import org.apache.kafka.common.message.SyncGroupResponseData; import org.apache.kafka.common.protocol.Errors; @@ -26,10 +28,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -206,10 +206,14 @@ public void testDeadToAwaitingRebalanceIllegalTransition() { @Test public void testSelectProtocol() { - List member1Protocols = Arrays.asList( - new Protocol("range", new byte[0]), - new Protocol("roundrobin", new byte[0]) - ); + JoinGroupRequestProtocolCollection member1Protocols = new JoinGroupRequestProtocolCollection(); + member1Protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(new byte[0])); + + member1Protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); GenericGroupMember member1 = new GenericGroupMember( memberId, @@ -223,10 +227,14 @@ public void testSelectProtocol() { ); group.add(member1); - List member2Protocols = Arrays.asList( - new Protocol("roundrobin", new byte[0]), - new Protocol("range", new byte[0]) - ); + JoinGroupRequestProtocolCollection member2Protocols = new JoinGroupRequestProtocolCollection(); + member2Protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + + member2Protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(new byte[0])); GenericGroupMember member2 = new GenericGroupMember( "member2", @@ -267,10 +275,14 @@ public void testSelectProtocolRaisesIfNoMembers() { @Test public void testSelectProtocolChoosesCompatibleProtocol() { - List member1Protocols = Arrays.asList( - new Protocol("range", new byte[0]), - new Protocol("roundrobin", new byte[0]) - ); + JoinGroupRequestProtocolCollection member1Protocols = new JoinGroupRequestProtocolCollection(); + member1Protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(new byte[0])); + + member1Protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); GenericGroupMember member1 = new GenericGroupMember( memberId, @@ -284,10 +296,15 @@ public void testSelectProtocolChoosesCompatibleProtocol() { ); group.add(member1); - List member2Protocols = Arrays.asList( - new Protocol("roundrobin", new byte[0]), - new Protocol("blah", new byte[0]) - ); + JoinGroupRequestProtocolCollection member2Protocols = new JoinGroupRequestProtocolCollection(); + member2Protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + + member2Protocols.add(new JoinGroupRequestProtocol() + .setName("foo") + .setMetadata(new byte[0])); + GenericGroupMember member2 = new GenericGroupMember( "member2", @@ -306,10 +323,14 @@ public void testSelectProtocolChoosesCompatibleProtocol() { @Test public void testSupportsProtocols() { - List member1Protocols = Arrays.asList( - new Protocol("range", new byte[0]), - new Protocol("roundrobin", new byte[0]) - ); + JoinGroupRequestProtocolCollection member1Protocols = new JoinGroupRequestProtocolCollection(); + member1Protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(new byte[0])); + + member1Protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); GenericGroupMember member1 = new GenericGroupMember( memberId, @@ -338,6 +359,14 @@ public void testSubscribedTopics() { // not able to compute it for a newly created group assertEquals(Optional.empty(), group.subscribedTopics()); + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(ConsumerProtocol.serializeSubscription( + new ConsumerPartitionAssignor.Subscription( + Collections.singletonList("foo") + )).array())); + GenericGroupMember member = new GenericGroupMember( memberId, Optional.empty(), @@ -346,16 +375,7 @@ public void testSubscribedTopics() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "range", - ConsumerProtocol.serializeSubscription( - new ConsumerPartitionAssignor.Subscription( - Collections.singletonList("foo") - ) - ).array() - ) - ) + protocols ); group.transitionTo(PREPARING_REBALANCE); @@ -373,6 +393,11 @@ public void testSubscribedTopics() { assertEquals(Optional.of(Collections.emptySet()), group.subscribedTopics()); + protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(new byte[0])); + GenericGroupMember memberWithFaultyProtocol = new GenericGroupMember( memberId, Optional.empty(), @@ -381,12 +406,7 @@ public void testSubscribedTopics() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "range", - new byte[0] - ) - ) + protocols ); group.transitionTo(PREPARING_REBALANCE); @@ -402,6 +422,11 @@ public void testSubscribedTopicsNonConsumerGroup() { // not able to compute it for a newly created group assertEquals(Optional.empty(), group.subscribedTopics()); + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(new byte[0])); + GenericGroupMember memberWithNonConsumerProtocol = new GenericGroupMember( memberId, Optional.empty(), @@ -410,12 +435,7 @@ public void testSubscribedTopicsNonConsumerGroup() { rebalanceTimeoutMs, sessionTimeoutMs, "My Protocol", - Collections.singletonList( - new Protocol( - "range", - new byte[0] - ) - ) + protocols ); group.transitionTo(PREPARING_REBALANCE); @@ -428,6 +448,11 @@ public void testSubscribedTopicsNonConsumerGroup() { @Test public void testInitNextGeneration() { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + GenericGroupMember member = new GenericGroupMember( memberId, Optional.empty(), @@ -436,12 +461,7 @@ public void testInitNextGeneration() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); group.transitionTo(PREPARING_REBALANCE); @@ -471,6 +491,11 @@ public void testInitNextGenerationEmptyGroup() { @Test public void testUpdateMember() { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + GenericGroupMember member = new GenericGroupMember( memberId, Optional.empty(), @@ -479,26 +504,20 @@ public void testUpdateMember() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); group.add(member); - List newProtocols = Arrays.asList( - new Protocol( - "range", - new byte[0] - ), - new Protocol( - "roundrobin", - new byte[0] - ) - ); + JoinGroupRequestProtocolCollection newProtocols = new JoinGroupRequestProtocolCollection(); + newProtocols.add(new JoinGroupRequestProtocol() + .setName("range") + .setMetadata(new byte[0])); + + newProtocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + int newRebalanceTimeoutMs = 120000; int newSessionTimeoutMs = 20000; group.updateMember(member, newProtocols, newRebalanceTimeoutMs, newSessionTimeoutMs, null); @@ -517,6 +536,11 @@ public void testReplaceGroupInstanceWithNonExistingMember() { @Test public void testReplaceGroupInstance() throws Exception { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + GenericGroupMember member = new GenericGroupMember( memberId, Optional.of(groupInstanceId), @@ -525,12 +549,7 @@ public void testReplaceGroupInstance() throws Exception { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); CompletableFuture joinGroupFuture = new CompletableFuture<>(); @@ -555,6 +574,11 @@ public void testReplaceGroupInstance() throws Exception { @Test public void testCompleteJoinFuture() throws Exception { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + GenericGroupMember member = new GenericGroupMember( memberId, Optional.empty(), @@ -563,12 +587,7 @@ public void testCompleteJoinFuture() throws Exception { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); CompletableFuture joinGroupFuture = new CompletableFuture<>(); @@ -589,6 +608,11 @@ public void testCompleteJoinFuture() throws Exception { @Test public void testNotCompleteJoinFuture() { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + GenericGroupMember member = new GenericGroupMember( memberId, Optional.empty(), @@ -597,12 +621,7 @@ public void testNotCompleteJoinFuture() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); group.add(member); @@ -619,6 +638,11 @@ public void testNotCompleteJoinFuture() { @Test public void testCompleteSyncFuture() throws Exception { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + GenericGroupMember member = new GenericGroupMember( memberId, Optional.empty(), @@ -627,12 +651,7 @@ public void testCompleteSyncFuture() throws Exception { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); group.add(member); @@ -650,6 +669,11 @@ public void testCompleteSyncFuture() throws Exception { @Test public void testNotCompleteSyncFuture() { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + GenericGroupMember member = new GenericGroupMember( memberId, Optional.empty(), @@ -658,12 +682,7 @@ public void testNotCompleteSyncFuture() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); group.add(member); @@ -677,6 +696,11 @@ public void testNotCompleteSyncFuture() { @Test public void testCannotAddPendingMemberIfStable() { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + GenericGroupMember member = new GenericGroupMember( memberId, Optional.empty(), @@ -685,12 +709,7 @@ public void testCannotAddPendingMemberIfStable() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); group.add(member); @@ -699,6 +718,11 @@ public void testCannotAddPendingMemberIfStable() { @Test public void testRemovalFromPendingAfterMemberIsStable() { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + group.addPendingMember(memberId); assertFalse(group.hasMemberId(memberId)); assertTrue(group.isPendingMember(memberId)); @@ -711,12 +735,7 @@ public void testRemovalFromPendingAfterMemberIsStable() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); group.add(member); @@ -737,6 +756,11 @@ public void testRemovalFromPendingWhenMemberIsRemoved() { @Test public void testCannotAddStaticMemberIfAlreadyPresent() { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + GenericGroupMember member = new GenericGroupMember( memberId, Optional.of(groupInstanceId), @@ -745,12 +769,7 @@ public void testCannotAddStaticMemberIfAlreadyPresent() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); group.add(member); @@ -775,6 +794,11 @@ public void testCannotRemovePendingSyncOfUnknownMember() { @Test public void testCanAddAndRemovePendingSyncMember() { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + GenericGroupMember member = new GenericGroupMember( memberId, Optional.empty(), @@ -783,12 +807,7 @@ public void testCanAddAndRemovePendingSyncMember() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); group.add(member); @@ -800,6 +819,11 @@ public void testCanAddAndRemovePendingSyncMember() { @Test public void testRemovalFromPendingSyncWhenMemberIsRemoved() { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + GenericGroupMember member = new GenericGroupMember( memberId, Optional.of(groupInstanceId), @@ -808,12 +832,7 @@ public void testRemovalFromPendingSyncWhenMemberIsRemoved() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); group.add(member); @@ -825,6 +844,11 @@ public void testRemovalFromPendingSyncWhenMemberIsRemoved() { @Test public void testNewGenerationClearsPendingSyncMembers() { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + GenericGroupMember member = new GenericGroupMember( memberId, Optional.empty(), @@ -833,12 +857,7 @@ public void testNewGenerationClearsPendingSyncMembers() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); group.add(member); @@ -851,6 +870,11 @@ public void testNewGenerationClearsPendingSyncMembers() { @Test public void testElectNewJoinedLeader() { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + GenericGroupMember leader = new GenericGroupMember( memberId, Optional.empty(), @@ -859,12 +883,7 @@ public void testElectNewJoinedLeader() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); group.add(leader); @@ -879,12 +898,7 @@ public void testElectNewJoinedLeader() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); group.add(newLeader, new CompletableFuture<>()); @@ -896,12 +910,7 @@ public void testElectNewJoinedLeader() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); group.add(newMember); @@ -911,6 +920,11 @@ public void testElectNewJoinedLeader() { @Test public void testMaybeElectNewJoinedLeaderChooseExisting() { + JoinGroupRequestProtocolCollection protocols = new JoinGroupRequestProtocolCollection(); + protocols.add(new JoinGroupRequestProtocol() + .setName("roundrobin") + .setMetadata(new byte[0])); + GenericGroupMember leader = new GenericGroupMember( memberId, Optional.empty(), @@ -919,12 +933,7 @@ public void testMaybeElectNewJoinedLeaderChooseExisting() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); group.add(leader, new CompletableFuture<>()); @@ -939,12 +948,7 @@ public void testMaybeElectNewJoinedLeaderChooseExisting() { rebalanceTimeoutMs, sessionTimeoutMs, protocolType, - Collections.singletonList( - new Protocol( - "roundrobin", - new byte[0] - ) - ) + protocols ); group.add(newMember); diff --git a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/runtime/CoordinatorRuntimeTest.java b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/runtime/CoordinatorRuntimeTest.java index fa8529154a5bd..8a1b1511c861d 100644 --- a/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/runtime/CoordinatorRuntimeTest.java +++ b/group-coordinator/src/test/java/org/apache/kafka/coordinator/group/runtime/CoordinatorRuntimeTest.java @@ -168,11 +168,11 @@ public long append(TopicPartition tp, List records) throws KafkaExceptio */ private static class MockCoordinator implements Coordinator { private final TimelineHashSet records; - private final CoordinatorTimer timer; + private final CoordinatorTimer timer; MockCoordinator( SnapshotRegistry snapshotRegistry, - CoordinatorTimer timer + CoordinatorTimer timer ) { this.records = new TimelineHashSet<>(snapshotRegistry, 0); this.timer = timer; @@ -187,7 +187,7 @@ Set records() { return Collections.unmodifiableSet(new HashSet<>(records)); } - CoordinatorTimer timer() { + CoordinatorTimer timer() { return timer; } } @@ -197,7 +197,7 @@ CoordinatorTimer timer() { */ private static class MockCoordinatorBuilder implements CoordinatorBuilder { private SnapshotRegistry snapshotRegistry; - private CoordinatorTimer timer; + private CoordinatorTimer timer; @Override public CoordinatorBuilder withSnapshotRegistry( @@ -223,12 +223,18 @@ public CoordinatorBuilder withTime( @Override public CoordinatorBuilder withTimer( - CoordinatorTimer timer + CoordinatorTimer timer ) { this.timer = timer; return this; } + public CoordinatorBuilder withTopicPartition( + TopicPartition topicPartition + ) { + return this; + } + @Override public MockCoordinator build() { return new MockCoordinator( @@ -271,6 +277,7 @@ public void testScheduleLoading() { when(builder.withLogContext(any())).thenReturn(builder); when(builder.withTime(any())).thenReturn(builder); when(builder.withTimer(any())).thenReturn(builder); + when(builder.withTopicPartition(any())).thenReturn(builder); when(builder.build()).thenReturn(coordinator); when(supplier.get()).thenReturn(builder); CompletableFuture future = new CompletableFuture<>(); @@ -334,6 +341,7 @@ public void testScheduleLoadingWithFailure() { when(builder.withLogContext(any())).thenReturn(builder); when(builder.withTime(any())).thenReturn(builder); when(builder.withTimer(any())).thenReturn(builder); + when(builder.withTopicPartition(any())).thenReturn(builder); when(builder.build()).thenReturn(coordinator); when(supplier.get()).thenReturn(builder); CompletableFuture future = new CompletableFuture<>(); @@ -384,6 +392,7 @@ public void testScheduleLoadingWithStalePartitionEpoch() { when(builder.withLogContext(any())).thenReturn(builder); when(builder.withTime(any())).thenReturn(builder); when(builder.withTimer(any())).thenReturn(builder); + when(builder.withTopicPartition(any())).thenReturn(builder); when(builder.build()).thenReturn(coordinator); when(supplier.get()).thenReturn(builder); CompletableFuture future = new CompletableFuture<>(); @@ -432,6 +441,7 @@ public void testScheduleLoadingAfterLoadingFailure() { when(builder.withLogContext(any())).thenReturn(builder); when(builder.withTime(any())).thenReturn(builder); when(builder.withTimer(any())).thenReturn(builder); + when(builder.withTopicPartition(any())).thenReturn(builder); when(builder.build()).thenReturn(coordinator); when(supplier.get()).thenReturn(builder); CompletableFuture future = new CompletableFuture<>(); @@ -497,6 +507,7 @@ public void testScheduleUnloading() { when(builder.withLogContext(any())).thenReturn(builder); when(builder.withTime(any())).thenReturn(builder); when(builder.withTimer(any())).thenReturn(builder); + when(builder.withTopicPartition(any())).thenReturn(builder); when(builder.build()).thenReturn(coordinator); when(supplier.get()).thenReturn(builder); @@ -546,6 +557,7 @@ public void testScheduleUnloadingWithStalePartitionEpoch() { when(builder.withTimer(any())).thenReturn(builder); when(builder.withTime(any())).thenReturn(builder); when(builder.withTimer(any())).thenReturn(builder); + when(builder.withTopicPartition(any())).thenReturn(builder); when(builder.build()).thenReturn(coordinator); when(supplier.get()).thenReturn(builder); @@ -962,7 +974,7 @@ public void testClose() throws Exception { // Timer #1. This is never executed. ctx.coordinator.timer.schedule("timer-1", 10, TimeUnit.SECONDS, true, - () -> Arrays.asList("record5", "record6")); + () -> new CoordinatorResult<>(Arrays.asList("record5", "record6"), null)); // The coordinator timer should have one pending task. assertEquals(1, ctx.timer.size()); @@ -1010,6 +1022,8 @@ public void testOnNewMetadataImage() { when(builder.withLogContext(any())).thenReturn(builder); when(builder.withTime(any())).thenReturn(builder); when(builder.withTimer(any())).thenReturn(builder); + when(builder.withTopicPartition(any())).thenReturn(builder); + when(builder.withTime(any())).thenReturn(builder); when(builder.build()) .thenReturn(coordinator0) .thenReturn(coordinator1); @@ -1068,11 +1082,11 @@ public void testScheduleTimer() throws InterruptedException { // Timer #1. ctx.coordinator.timer.schedule("timer-1", 10, TimeUnit.MILLISECONDS, true, - () -> Arrays.asList("record1", "record2")); + () -> new CoordinatorResult<>(Arrays.asList("record1", "record2"), null)); // Timer #2. ctx.coordinator.timer.schedule("timer-2", 20, TimeUnit.MILLISECONDS, true, - () -> Arrays.asList("record3", "record4")); + () -> new CoordinatorResult<>(Arrays.asList("record3", "record4"), null)); // The coordinator timer should have two pending tasks. assertEquals(2, ctx.timer.size()); @@ -1122,7 +1136,7 @@ public void testRescheduleTimer() throws InterruptedException { // Timer #1. ctx.coordinator.timer.schedule("timer-1", 10, TimeUnit.MILLISECONDS, true, - () -> Collections.singletonList("record1")); + () -> new CoordinatorResult<>(Collections.singletonList("record1"), null)); // The coordinator timer should have one pending task. assertEquals(1, ctx.timer.size()); @@ -1135,14 +1149,14 @@ public void testRescheduleTimer() throws InterruptedException { // Schedule a second timer with the same key. ctx.coordinator.timer.schedule("timer-1", 10, TimeUnit.MILLISECONDS, true, - () -> Collections.singletonList("record2")); + () -> new CoordinatorResult<>(Collections.singletonList("record2"), null)); // The coordinator timer should still have one pending task. assertEquals(1, ctx.timer.size()); // Schedule a third timer with the same key. ctx.coordinator.timer.schedule("timer-1", 10, TimeUnit.MILLISECONDS, true, - () -> Collections.singletonList("record3")); + () -> new CoordinatorResult<>(Collections.singletonList("record3"), null)); // The coordinator timer should still have one pending task. assertEquals(1, ctx.timer.size()); @@ -1193,7 +1207,7 @@ public void testCancelTimer() throws InterruptedException { // Timer #1. ctx.coordinator.timer.schedule("timer-1", 10, TimeUnit.MILLISECONDS, true, - () -> Collections.singletonList("record1")); + () -> new CoordinatorResult<>(Collections.singletonList("record1"), null)); // The coordinator timer should have one pending task. assertEquals(1, ctx.timer.size()); @@ -1206,7 +1220,7 @@ public void testCancelTimer() throws InterruptedException { // Schedule a second timer with the same key. ctx.coordinator.timer.schedule("timer-1", 10, TimeUnit.MILLISECONDS, true, - () -> Collections.singletonList("record2")); + () -> new CoordinatorResult<>(Collections.singletonList("record2"), null)); // The coordinator timer should still have one pending task. assertEquals(1, ctx.timer.size());