From 4cf89e63bd39d701b72311e77b55bbf525671f63 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 1 Apr 2019 16:20:35 -0700 Subject: [PATCH 01/59] bump up consumer protocol to v2 --- .../consumer/internals/ConsumerProtocol.java | 238 ++++++++++++++++-- .../consumer/internals/PartitionAssignor.java | 31 ++- .../org/apache/kafka/common/utils/Bytes.java | 2 +- .../internals/ConsumerProtocolTest.java | 128 ++++++++-- 4 files changed, 345 insertions(+), 54 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java index 8a4aef8b33487..96d8fa21665bc 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java @@ -30,6 +30,8 @@ import java.util.List; import java.util.Map; +import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE; + /** * ConsumerProtocol contains the schemas for consumer subscriptions and assignments for use with * Kafka's generalized group management protocol. Below is the version 0 format: @@ -39,12 +41,20 @@ * Version => Int16 * Topics => [String] * UserData => Bytes + * OwnedPartitions => [Topic Partitions] + * Topic => String + * Partitions => [int32] * * Assignment => Version TopicPartitions - * Version => int16 - * TopicPartitions => [Topic Partitions] - * Topic => String - * Partitions => [int32] + * Version => int16 + * AssignedPartitions => [Topic Partitions] + * Topic => String + * Partitions => [int32] + * UserData => Bytes + * RevokedPartitions => [Topic Partitions] + * Topic => String + * Partitions => [int32] + * ErrorCode => [int16] * * * The current implementation assumes that future versions will not break compatibility. When @@ -59,29 +69,75 @@ public class ConsumerProtocol { public static final String TOPICS_KEY_NAME = "topics"; public static final String TOPIC_KEY_NAME = "topic"; public static final String PARTITIONS_KEY_NAME = "partitions"; + public static final String OWNED_PARTITIONS_KEY_NAME = "owned_partitions"; + public static final String REVOKED_PARTITIONS_KEY_NAME = "revoked_partitions"; public static final String TOPIC_PARTITIONS_KEY_NAME = "topic_partitions"; public static final String USER_DATA_KEY_NAME = "user_data"; public static final short CONSUMER_PROTOCOL_V0 = 0; + public static final short CONSUMER_PROTOCOL_V1 = 1; + public static final Schema CONSUMER_PROTOCOL_HEADER_SCHEMA = new Schema( new Field(VERSION_KEY_NAME, Type.INT16)); private static final Struct CONSUMER_PROTOCOL_HEADER_V0 = new Struct(CONSUMER_PROTOCOL_HEADER_SCHEMA) .set(VERSION_KEY_NAME, CONSUMER_PROTOCOL_V0); + private static final Struct CONSUMER_PROTOCOL_HEADER_V1 = new Struct(CONSUMER_PROTOCOL_HEADER_SCHEMA) + .set(VERSION_KEY_NAME, CONSUMER_PROTOCOL_V1); + + public static final Schema TOPIC_ASSIGNMENT_V0 = new Schema( + new Field(TOPIC_KEY_NAME, Type.STRING), + new Field(PARTITIONS_KEY_NAME, new ArrayOf(Type.INT32))); public static final Schema SUBSCRIPTION_V0 = new Schema( new Field(TOPICS_KEY_NAME, new ArrayOf(Type.STRING)), new Field(USER_DATA_KEY_NAME, Type.NULLABLE_BYTES)); - public static final Schema TOPIC_ASSIGNMENT_V0 = new Schema( - new Field(TOPIC_KEY_NAME, Type.STRING), - new Field(PARTITIONS_KEY_NAME, new ArrayOf(Type.INT32))); + + public static final Schema SUBSCRIPTION_V1 = new Schema( + new Field(TOPICS_KEY_NAME, new ArrayOf(Type.STRING)), + new Field(USER_DATA_KEY_NAME, Type.NULLABLE_BYTES), + new Field(OWNED_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0))); + public static final Schema ASSIGNMENT_V0 = new Schema( new Field(TOPIC_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)), new Field(USER_DATA_KEY_NAME, Type.NULLABLE_BYTES)); - public static ByteBuffer serializeSubscription(PartitionAssignor.Subscription subscription) { + public static final Schema ASSIGNMENT_V1 = new Schema( + new Field(TOPIC_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)), + new Field(USER_DATA_KEY_NAME, Type.NULLABLE_BYTES), + new Field(REVOKED_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)), + ERROR_CODE); + + public enum Errors { + NONE(0), + NEED_REJOIN(1); + + private final short code; + + Errors(final int code) { + this.code = (short) code; + } + + public short code() { + return code; + } + + public static Errors fromCode(final short code) { + switch (code) { + case 0: + return NONE; + case 1: + return NEED_REJOIN; + default: + throw new IllegalArgumentException("Unknown error code: " + code); + } + } + } + + public static ByteBuffer serializeSubscriptionV0(PartitionAssignor.Subscription subscription) { Struct struct = new Struct(SUBSCRIPTION_V0); struct.set(USER_DATA_KEY_NAME, subscription.userData()); struct.set(TOPICS_KEY_NAME, subscription.topics().toArray()); + ByteBuffer buffer = ByteBuffer.allocate(CONSUMER_PROTOCOL_HEADER_V0.sizeOf() + SUBSCRIPTION_V0.sizeOf(struct)); CONSUMER_PROTOCOL_HEADER_V0.writeTo(buffer); SUBSCRIPTION_V0.write(buffer, struct); @@ -89,37 +145,80 @@ public static ByteBuffer serializeSubscription(PartitionAssignor.Subscription su return buffer; } - public static PartitionAssignor.Subscription deserializeSubscription(ByteBuffer buffer) { - Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer); - Short version = header.getShort(VERSION_KEY_NAME); - checkVersionCompatibility(version); + public static ByteBuffer serializeSubscriptionV1(PartitionAssignor.Subscription subscription) { + Struct struct = new Struct(SUBSCRIPTION_V1); + struct.set(USER_DATA_KEY_NAME, subscription.userData()); + struct.set(TOPICS_KEY_NAME, subscription.topics().toArray()); + List topicAssignments = new ArrayList<>(); + Map> partitionsByTopic = CollectionUtils.groupPartitionsByTopic(subscription.ownedPartitions()); + for (Map.Entry> topicEntry : partitionsByTopic.entrySet()) { + Struct topicAssignment = new Struct(TOPIC_ASSIGNMENT_V0); + topicAssignment.set(TOPIC_KEY_NAME, topicEntry.getKey()); + topicAssignment.set(PARTITIONS_KEY_NAME, topicEntry.getValue().toArray()); + topicAssignments.add(topicAssignment); + } + struct.set(OWNED_PARTITIONS_KEY_NAME, topicAssignments.toArray()); + + ByteBuffer buffer = ByteBuffer.allocate(CONSUMER_PROTOCOL_HEADER_V1.sizeOf() + SUBSCRIPTION_V1.sizeOf(struct)); + CONSUMER_PROTOCOL_HEADER_V1.writeTo(buffer); + SUBSCRIPTION_V1.write(buffer, struct); + buffer.flip(); + return buffer; + + } + + public static ByteBuffer serializeSubscription(PartitionAssignor.Subscription subscription) { + return serializeSubscriptionV1(subscription); + } + + public static PartitionAssignor.Subscription deserializeSubscriptionV0(ByteBuffer buffer) { Struct struct = SUBSCRIPTION_V0.read(buffer); ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME); List topics = new ArrayList<>(); for (Object topicObj : struct.getArray(TOPICS_KEY_NAME)) topics.add((String) topicObj); + return new PartitionAssignor.Subscription(topics, userData); } - public static PartitionAssignor.Assignment deserializeAssignment(ByteBuffer buffer) { - Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer); - Short version = header.getShort(VERSION_KEY_NAME); - checkVersionCompatibility(version); - Struct struct = ASSIGNMENT_V0.read(buffer); + public static PartitionAssignor.Subscription deserializeSubscriptionV1(ByteBuffer buffer) { + Struct struct = SUBSCRIPTION_V1.read(buffer); ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME); - List partitions = new ArrayList<>(); - for (Object structObj : struct.getArray(TOPIC_PARTITIONS_KEY_NAME)) { + List topics = new ArrayList<>(); + for (Object topicObj : struct.getArray(TOPICS_KEY_NAME)) + topics.add((String) topicObj); + + List ownedPartitions = new ArrayList<>(); + for (Object structObj : struct.getArray(OWNED_PARTITIONS_KEY_NAME)) { Struct assignment = (Struct) structObj; String topic = assignment.getString(TOPIC_KEY_NAME); for (Object partitionObj : assignment.getArray(PARTITIONS_KEY_NAME)) { Integer partition = (Integer) partitionObj; - partitions.add(new TopicPartition(topic, partition)); + ownedPartitions.add(new TopicPartition(topic, partition)); } } - return new PartitionAssignor.Assignment(partitions, userData); + + return new PartitionAssignor.Subscription(topics, userData, ownedPartitions); } - public static ByteBuffer serializeAssignment(PartitionAssignor.Assignment assignment) { + public static PartitionAssignor.Subscription deserializeSubscription(ByteBuffer buffer) { + Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer); + Short version = header.getShort(VERSION_KEY_NAME); + + if (version < CONSUMER_PROTOCOL_V0) + throw new SchemaException("Unsupported subscription version: " + version); + + switch (version) { + case CONSUMER_PROTOCOL_V0: + return deserializeSubscriptionV0(buffer); + + // assume all higher versions can be parsed as V1 + default: + return deserializeSubscriptionV1(buffer); + } + } + + public static ByteBuffer serializeAssignmentV0(PartitionAssignor.Assignment assignment) { Struct struct = new Struct(ASSIGNMENT_V0); struct.set(USER_DATA_KEY_NAME, assignment.userData()); List topicAssignments = new ArrayList<>(); @@ -131,6 +230,7 @@ public static ByteBuffer serializeAssignment(PartitionAssignor.Assignment assign topicAssignments.add(topicAssignment); } struct.set(TOPIC_PARTITIONS_KEY_NAME, topicAssignments.toArray()); + ByteBuffer buffer = ByteBuffer.allocate(CONSUMER_PROTOCOL_HEADER_V0.sizeOf() + ASSIGNMENT_V0.sizeOf(struct)); CONSUMER_PROTOCOL_HEADER_V0.writeTo(buffer); ASSIGNMENT_V0.write(buffer, struct); @@ -138,12 +238,96 @@ public static ByteBuffer serializeAssignment(PartitionAssignor.Assignment assign return buffer; } - private static void checkVersionCompatibility(short version) { - // check for invalid versions - if (version < CONSUMER_PROTOCOL_V0) - throw new SchemaException("Unsupported subscription version: " + version); + public static ByteBuffer serializeAssignmentV1(PartitionAssignor.Assignment assignment) { + Struct struct = new Struct(ASSIGNMENT_V1); + struct.set(USER_DATA_KEY_NAME, assignment.userData()); + List topicAssignments = new ArrayList<>(); + Map> partitionsByTopic = CollectionUtils.groupPartitionsByTopic(assignment.partitions()); + for (Map.Entry> topicEntry : partitionsByTopic.entrySet()) { + Struct topicAssignment = new Struct(TOPIC_ASSIGNMENT_V0); + topicAssignment.set(TOPIC_KEY_NAME, topicEntry.getKey()); + topicAssignment.set(PARTITIONS_KEY_NAME, topicEntry.getValue().toArray()); + topicAssignments.add(topicAssignment); + } + struct.set(TOPIC_PARTITIONS_KEY_NAME, topicAssignments.toArray()); + List revokedAssignments = new ArrayList<>(); + partitionsByTopic = CollectionUtils.groupPartitionsByTopic(assignment.revokedPartitions()); + for (Map.Entry> topicEntry : partitionsByTopic.entrySet()) { + Struct topicAssignment = new Struct(TOPIC_ASSIGNMENT_V0); + topicAssignment.set(TOPIC_KEY_NAME, topicEntry.getKey()); + topicAssignment.set(PARTITIONS_KEY_NAME, topicEntry.getValue().toArray()); + revokedAssignments.add(topicAssignment); + } + struct.set(REVOKED_PARTITIONS_KEY_NAME, revokedAssignments.toArray()); + struct.set(ERROR_CODE.name, assignment.error().code); + + ByteBuffer buffer = ByteBuffer.allocate(CONSUMER_PROTOCOL_HEADER_V1.sizeOf() + ASSIGNMENT_V1.sizeOf(struct)); + CONSUMER_PROTOCOL_HEADER_V1.writeTo(buffer); + ASSIGNMENT_V1.write(buffer, struct); + buffer.flip(); + return buffer; + } + + public static ByteBuffer serializeAssignment(PartitionAssignor.Assignment assignment) { + return serializeAssignmentV1(assignment); + } + + public static PartitionAssignor.Assignment deserializeAssignmentV0(ByteBuffer buffer) { + Struct struct = ASSIGNMENT_V0.read(buffer); + ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME); + List partitions = new ArrayList<>(); + for (Object structObj : struct.getArray(TOPIC_PARTITIONS_KEY_NAME)) { + Struct assignment = (Struct) structObj; + String topic = assignment.getString(TOPIC_KEY_NAME); + for (Object partitionObj : assignment.getArray(PARTITIONS_KEY_NAME)) { + Integer partition = (Integer) partitionObj; + partitions.add(new TopicPartition(topic, partition)); + } + } + return new PartitionAssignor.Assignment(partitions, userData); + } + + public static PartitionAssignor.Assignment deserializeAssignmentV1(ByteBuffer buffer) { + Struct struct = ASSIGNMENT_V1.read(buffer); + ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME); + List partitions = new ArrayList<>(); + for (Object structObj : struct.getArray(TOPIC_PARTITIONS_KEY_NAME)) { + Struct assignment = (Struct) structObj; + String topic = assignment.getString(TOPIC_KEY_NAME); + for (Object partitionObj : assignment.getArray(PARTITIONS_KEY_NAME)) { + Integer partition = (Integer) partitionObj; + partitions.add(new TopicPartition(topic, partition)); + } + } - // otherwise, assume versions can be parsed as V0 + List revokedPartitions = new ArrayList<>(); + for (Object structObj : struct.getArray(REVOKED_PARTITIONS_KEY_NAME)) { + Struct assignment = (Struct) structObj; + String topic = assignment.getString(TOPIC_KEY_NAME); + for (Object partitionObj : assignment.getArray(PARTITIONS_KEY_NAME)) { + Integer partition = (Integer) partitionObj; + revokedPartitions.add(new TopicPartition(topic, partition)); + } + } + Errors error = Errors.fromCode(struct.get(ERROR_CODE)); + + return new PartitionAssignor.Assignment(partitions, userData, revokedPartitions, error); } + public static PartitionAssignor.Assignment deserializeAssignment(ByteBuffer buffer) { + Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer); + Short version = header.getShort(VERSION_KEY_NAME); + + if (version < CONSUMER_PROTOCOL_V0) + throw new SchemaException("Unsupported assignment version: " + version); + + switch (version) { + case CONSUMER_PROTOCOL_V0: + return deserializeAssignmentV0(buffer); + + // assume all higher versions can be parsed as V1 + default: + return deserializeAssignmentV1(buffer); + } + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java index 4a7c7a8bbd341..ec4e2c3123aca 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.TopicPartition; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -75,10 +76,16 @@ public interface PartitionAssignor { class Subscription { private final List topics; private final ByteBuffer userData; + private final List ownedPartitions; - public Subscription(List topics, ByteBuffer userData) { + public Subscription(List topics, ByteBuffer userData, List ownedPartitions) { this.topics = topics; this.userData = userData; + this.ownedPartitions = ownedPartitions; + } + + public Subscription(List topics, ByteBuffer userData) { + this(topics, userData, Collections.emptyList()); } public Subscription(List topics) { @@ -89,6 +96,10 @@ public List topics() { return topics; } + public List ownedPartitions() { + return ownedPartitions; + } + public ByteBuffer userData() { return userData; } @@ -104,10 +115,18 @@ public String toString() { class Assignment { private final List partitions; private final ByteBuffer userData; + private final List revokedPartitions; + private final ConsumerProtocol.Errors error; - public Assignment(List partitions, ByteBuffer userData) { + public Assignment(List partitions, ByteBuffer userData, List revokedPartitions, ConsumerProtocol.Errors error) { this.partitions = partitions; this.userData = userData; + this.revokedPartitions = revokedPartitions; + this.error = error; + } + + public Assignment(List partitions, ByteBuffer userData) { + this(partitions, userData, Collections.emptyList(), ConsumerProtocol.Errors.NONE); } public Assignment(List partitions) { @@ -118,6 +137,14 @@ public List partitions() { return partitions; } + public List revokedPartitions() { + return revokedPartitions; + } + + public ConsumerProtocol.Errors error() { + return error; + } + public ByteBuffer userData() { return userData; } diff --git a/clients/src/main/java/org/apache/kafka/common/utils/Bytes.java b/clients/src/main/java/org/apache/kafka/common/utils/Bytes.java index 19cd711928fc8..d90a44131a6dc 100644 --- a/clients/src/main/java/org/apache/kafka/common/utils/Bytes.java +++ b/clients/src/main/java/org/apache/kafka/common/utils/Bytes.java @@ -141,7 +141,7 @@ private static String toString(final byte[] b, int off, int len) { } /** - * A byte array comparator based on lexicograpic ordering. + * A byte array comparator based on lexicographic ordering. */ public final static ByteArrayComparator BYTES_LEXICO_COMPARATOR = new LexicographicByteArrayComparator(); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java index 37d105cf1cc3a..d48a1986d5d9b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java @@ -16,6 +16,8 @@ */ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.clients.consumer.internals.ConsumerProtocol.Errors; +import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Assignment; import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.types.ArrayOf; @@ -28,21 +30,36 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; +import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.CONSUMER_PROTOCOL_HEADER_SCHEMA; +import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.OWNED_PARTITIONS_KEY_NAME; +import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.REVOKED_PARTITIONS_KEY_NAME; +import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.TOPICS_KEY_NAME; +import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.TOPIC_ASSIGNMENT_V0; +import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.TOPIC_PARTITIONS_KEY_NAME; +import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.USER_DATA_KEY_NAME; +import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.VERSION_KEY_NAME; +import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; public class ConsumerProtocolTest { + private final TopicPartition tp1 = new TopicPartition("foo", 1); + private final TopicPartition tp2 = new TopicPartition("bar", 2); + @Test public void serializeDeserializeMetadata() { Subscription subscription = new Subscription(Arrays.asList("foo", "bar")); ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription); Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer); assertEquals(subscription.topics(), parsedSubscription.topics()); + assertEquals(0, parsedSubscription.userData().limit()); } @Test @@ -51,26 +68,53 @@ public void serializeDeserializeNullSubscriptionUserData() { ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription); Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer); assertEquals(subscription.topics(), parsedSubscription.topics()); - assertNull(subscription.userData()); + assertNull(parsedSubscription.userData()); + } + + @Test + public void deserializeOldSubscriptionVersion() { + Subscription subscription = new Subscription(Arrays.asList("foo", "bar")); + ByteBuffer buffer = ConsumerProtocol.serializeSubscriptionV0(subscription); + Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer); + assertEquals(parsedSubscription.topics(), parsedSubscription.topics()); + assertEquals(0, parsedSubscription.userData().limit()); + assertTrue(parsedSubscription.ownedPartitions().isEmpty()); } @Test public void deserializeNewSubscriptionVersion() { + Subscription subscription = new Subscription(Arrays.asList("foo", "bar"), null, Collections.singletonList(tp2)); + ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription); + // ignore the version assuming it is the old byte code, as it will blindly deserialize as V0 + Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer); + header.getShort(VERSION_KEY_NAME); + Subscription parsedSubscription = ConsumerProtocol.deserializeSubscriptionV0(buffer); + assertEquals(subscription.topics(), parsedSubscription.topics()); + assertNull(parsedSubscription.userData()); + assertTrue(parsedSubscription.ownedPartitions().isEmpty()); + } + + @Test + public void deserializeFutureSubscriptionVersion() { // verify that a new version which adds a field is still parseable short version = 100; Schema subscriptionSchemaV100 = new Schema( - new Field(ConsumerProtocol.TOPICS_KEY_NAME, new ArrayOf(Type.STRING)), - new Field(ConsumerProtocol.USER_DATA_KEY_NAME, Type.BYTES), + new Field(TOPICS_KEY_NAME, new ArrayOf(Type.STRING)), + new Field(USER_DATA_KEY_NAME, Type.BYTES), + new Field(OWNED_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)), new Field("foo", Type.STRING)); Struct subscriptionV100 = new Struct(subscriptionSchemaV100); - subscriptionV100.set(ConsumerProtocol.TOPICS_KEY_NAME, new Object[]{"topic"}); - subscriptionV100.set(ConsumerProtocol.USER_DATA_KEY_NAME, ByteBuffer.wrap(new byte[0])); + subscriptionV100.set(TOPICS_KEY_NAME, new Object[]{"topic"}); + subscriptionV100.set(USER_DATA_KEY_NAME, ByteBuffer.wrap(new byte[0])); + subscriptionV100.set(OWNED_PARTITIONS_KEY_NAME, new Object[]{new Struct(TOPIC_ASSIGNMENT_V0) + .set(ConsumerProtocol.TOPIC_KEY_NAME, tp2.topic()) + .set(ConsumerProtocol.PARTITIONS_KEY_NAME, new Object[]{tp2.partition()})}); subscriptionV100.set("foo", "bar"); - Struct headerV100 = new Struct(ConsumerProtocol.CONSUMER_PROTOCOL_HEADER_SCHEMA); - headerV100.set(ConsumerProtocol.VERSION_KEY_NAME, version); + Struct headerV100 = new Struct(CONSUMER_PROTOCOL_HEADER_SCHEMA); + headerV100.set(VERSION_KEY_NAME, version); ByteBuffer buffer = ByteBuffer.allocate(subscriptionV100.sizeOf() + headerV100.sizeOf()); headerV100.writeTo(buffer); @@ -79,46 +123,80 @@ public void deserializeNewSubscriptionVersion() { buffer.flip(); Subscription subscription = ConsumerProtocol.deserializeSubscription(buffer); - assertEquals(Arrays.asList("topic"), subscription.topics()); + assertEquals(Collections.singletonList("topic"), subscription.topics()); + assertEquals(Collections.singletonList(tp2), subscription.ownedPartitions()); } @Test public void serializeDeserializeAssignment() { - List partitions = Arrays.asList(new TopicPartition("foo", 0), new TopicPartition("bar", 2)); - ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new PartitionAssignor.Assignment(partitions)); - PartitionAssignor.Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer); + List partitions = Arrays.asList(tp1, tp2); + ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment(partitions)); + Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer); assertEquals(toSet(partitions), toSet(parsedAssignment.partitions())); + assertEquals(0, parsedAssignment.userData().limit()); } @Test public void deserializeNullAssignmentUserData() { - List partitions = Arrays.asList(new TopicPartition("foo", 0), new TopicPartition("bar", 2)); - ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new PartitionAssignor.Assignment(partitions, null)); - PartitionAssignor.Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer); + List partitions = Arrays.asList(tp1, tp2); + ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment(partitions, null)); + Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer); assertEquals(toSet(partitions), toSet(parsedAssignment.partitions())); assertNull(parsedAssignment.userData()); } + @Test + public void deserializeOldAssignmentVersion() { + List partitions = Arrays.asList(tp1, tp2); + ByteBuffer buffer = ConsumerProtocol.serializeAssignmentV0(new Assignment(partitions)); + Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer); + assertEquals(toSet(partitions), toSet(parsedAssignment.partitions())); + assertEquals(0, parsedAssignment.userData().limit()); + assertTrue(parsedAssignment.revokedPartitions().isEmpty()); + assertEquals(Errors.NONE, parsedAssignment.error()); + } + @Test public void deserializeNewAssignmentVersion() { + List partitions = Collections.singletonList(tp1); + ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment(partitions, null, Collections.singletonList(tp2), Errors.NEED_REJOIN)); + // ignore the version assuming it is the old byte code, as it will blindly deserialize as 0 + Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer); + header.getShort(VERSION_KEY_NAME); + Assignment parsedAssignment = ConsumerProtocol.deserializeAssignmentV0(buffer); + assertEquals(toSet(partitions), toSet(parsedAssignment.partitions())); + assertNull(parsedAssignment.userData()); + assertTrue(parsedAssignment.revokedPartitions().isEmpty()); + assertEquals(Errors.NONE, parsedAssignment.error()); + } + + @Test + public void deserializeFutureAssignmentVersion() { // verify that a new version which adds a field is still parseable short version = 100; Schema assignmentSchemaV100 = new Schema( - new Field(ConsumerProtocol.TOPIC_PARTITIONS_KEY_NAME, new ArrayOf(ConsumerProtocol.TOPIC_ASSIGNMENT_V0)), - new Field(ConsumerProtocol.USER_DATA_KEY_NAME, Type.BYTES), + new Field(TOPIC_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)), + new Field(USER_DATA_KEY_NAME, Type.BYTES), + new Field(REVOKED_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)), + ERROR_CODE, new Field("foo", Type.STRING)); Struct assignmentV100 = new Struct(assignmentSchemaV100); - assignmentV100.set(ConsumerProtocol.TOPIC_PARTITIONS_KEY_NAME, - new Object[]{new Struct(ConsumerProtocol.TOPIC_ASSIGNMENT_V0) - .set(ConsumerProtocol.TOPIC_KEY_NAME, "foo") - .set(ConsumerProtocol.PARTITIONS_KEY_NAME, new Object[]{1})}); - assignmentV100.set(ConsumerProtocol.USER_DATA_KEY_NAME, ByteBuffer.wrap(new byte[0])); + assignmentV100.set(TOPIC_PARTITIONS_KEY_NAME, + new Object[]{new Struct(TOPIC_ASSIGNMENT_V0) + .set(ConsumerProtocol.TOPIC_KEY_NAME, tp1.topic()) + .set(ConsumerProtocol.PARTITIONS_KEY_NAME, new Object[]{tp1.partition()})}); + assignmentV100.set(USER_DATA_KEY_NAME, ByteBuffer.wrap(new byte[0])); + assignmentV100.set(REVOKED_PARTITIONS_KEY_NAME, + new Object[]{new Struct(TOPIC_ASSIGNMENT_V0) + .set(ConsumerProtocol.TOPIC_KEY_NAME, tp2.topic()) + .set(ConsumerProtocol.PARTITIONS_KEY_NAME, new Object[]{tp2.partition()})}); + assignmentV100.set(ERROR_CODE.name, Errors.NEED_REJOIN.code()); assignmentV100.set("foo", "bar"); - Struct headerV100 = new Struct(ConsumerProtocol.CONSUMER_PROTOCOL_HEADER_SCHEMA); - headerV100.set(ConsumerProtocol.VERSION_KEY_NAME, version); + Struct headerV100 = new Struct(CONSUMER_PROTOCOL_HEADER_SCHEMA); + headerV100.set(VERSION_KEY_NAME, version); ByteBuffer buffer = ByteBuffer.allocate(assignmentV100.sizeOf() + headerV100.sizeOf()); headerV100.writeTo(buffer); @@ -127,7 +205,9 @@ public void deserializeNewAssignmentVersion() { buffer.flip(); PartitionAssignor.Assignment assignment = ConsumerProtocol.deserializeAssignment(buffer); - assertEquals(toSet(Arrays.asList(new TopicPartition("foo", 1))), toSet(assignment.partitions())); + assertEquals(toSet(Collections.singletonList(tp1)), toSet(assignment.partitions())); + assertEquals(toSet(Collections.singletonList(tp2)), toSet(assignment.revokedPartitions())); + assertEquals(Errors.NEED_REJOIN, assignment.error()); } private static Set toSet(Collection collection) { From 094ab4343ace78622c9eac635c01e6fc2ddfb3f7 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 1 Apr 2019 22:04:23 -0700 Subject: [PATCH 02/59] update consumer coordinator based on rebalance protocol --- .../clients/consumer/ConsumerConfig.java | 16 +++ .../kafka/clients/consumer/KafkaConsumer.java | 5 +- .../internals/ConsumerCoordinator.java | 131 +++++++++++++++--- .../clients/consumer/KafkaConsumerTest.java | 3 +- .../internals/ConsumerCoordinatorTest.java | 3 +- .../internals/SubscriptionStateTest.java | 7 +- 6 files changed, 139 insertions(+), 26 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java index b92cbf916c213..2bad40afa8809 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java @@ -18,6 +18,7 @@ import org.apache.kafka.clients.ClientDnsLookup; import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; import org.apache.kafka.common.config.AbstractConfig; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigDef.Importance; @@ -262,6 +263,15 @@ public class ConsumerConfig extends AbstractConfig { public static final String DEFAULT_ISOLATION_LEVEL = IsolationLevel.READ_UNCOMMITTED.toString().toLowerCase(Locale.ROOT); + /** rebalance.protocol */ + public static final String REBALANCE_PROTOCOL_CONFIG = "rebalance.protocol"; + public static final String REBALANCE_PROTOCOL_DOC = "

Controls how consumer clients will participate in a rebalance. If set to eager, consumer will always revoke all its assigned partitions" + + " before sending the join group request. If set to cooperative, consumer will only revoke partitions as required in the previously received assignment information and then join group with the rest of the owned partitions" + + " encoded in the subscription information

When upgrading consumer from versions older than 2.3 to versions equal or newer than 2.3, users should keep this config as eager during the first rolling bounce to upgrade" + + " to the newer version, and only consider changing this config to cooperative after that with a second rolling bounce

"; + + public static final String DEFAULT_REBALANCE_PROTOCOL = ConsumerCoordinator.RebalanceProtocol.EAGER.toString().toLowerCase(Locale.ROOT); + static { CONFIG = new ConfigDef().define(BOOTSTRAP_SERVERS_CONFIG, Type.LIST, @@ -464,6 +474,12 @@ public class ConsumerConfig extends AbstractConfig { in(IsolationLevel.READ_COMMITTED.toString().toLowerCase(Locale.ROOT), IsolationLevel.READ_UNCOMMITTED.toString().toLowerCase(Locale.ROOT)), Importance.MEDIUM, ISOLATION_LEVEL_DOC) + .define(REBALANCE_PROTOCOL_CONFIG, + Type.STRING, + DEFAULT_REBALANCE_PROTOCOL, + in(ConsumerCoordinator.RebalanceProtocol.EAGER.toString().toLowerCase(Locale.ROOT), ConsumerCoordinator.RebalanceProtocol.COOPERATIVE.toString().toLowerCase(Locale.ROOT)), + Importance.MEDIUM, + REBALANCE_PROTOCOL_DOC) // security support .define(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, Type.STRING, diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index 4cee56a23959e..de9f32a27b4db 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -758,6 +758,8 @@ else if (enableAutoCommit) int maxPollIntervalMs = config.getInt(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG); int sessionTimeoutMs = config.getInt(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG); // no coordinator will be constructed for the default (null) group id + ConsumerCoordinator.RebalanceProtocol rebalanceProtocol = ConsumerCoordinator.RebalanceProtocol.valueOf( + config.getString(ConsumerConfig.REBALANCE_PROTOCOL_CONFIG).toUpperCase(Locale.ROOT)); this.coordinator = groupId == null ? null : new ConsumerCoordinator(logContext, this.client, @@ -775,7 +777,8 @@ else if (enableAutoCommit) enableAutoCommit, config.getInt(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG), this.interceptors, - config.getBoolean(ConsumerConfig.LEAVE_GROUP_ON_CLOSE_CONFIG)); + config.getBoolean(ConsumerConfig.LEAVE_GROUP_ON_CLOSE_CONFIG), + rebalanceProtocol); this.fetcher = new Fetcher<>( logContext, this.client, diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 2b949a37deb21..f5487fb51be91 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -111,6 +111,33 @@ private boolean sameRequest(final Set currentRequest, final Gene } } + public enum RebalanceProtocol { + EAGER((byte) 0), COOPERATIVE((byte) 1); + + private final byte id; + + RebalanceProtocol(byte id) { + this.id = id; + } + + public byte id() { + return id; + } + + public static RebalanceProtocol forId(byte id) { + switch (id) { + case 0: + return EAGER; + case 1: + return COOPERATIVE; + default: + throw new IllegalArgumentException("Unknown isolation level " + id); + } + } + } + + private final RebalanceProtocol protocol; + /** * Initialize the coordination manager. */ @@ -130,7 +157,8 @@ public ConsumerCoordinator(LogContext logContext, boolean autoCommitEnabled, int autoCommitIntervalMs, ConsumerInterceptors interceptors, - final boolean leaveGroupOnClose) { + final boolean leaveGroupOnClose, + final RebalanceProtocol protocol) { super(logContext, client, groupId, @@ -144,6 +172,7 @@ public ConsumerCoordinator(LogContext logContext, leaveGroupOnClose); this.log = logContext.logger(ConsumerCoordinator.class); this.metadata = metadata; + this.protocol = protocol; this.metadataSnapshot = new MetadataSnapshot(subscriptions, metadata.fetch(), metadata.updateVersion()); this.subscriptions = subscriptions; this.defaultOffsetCommitCallback = new DefaultOffsetCommitCallback(); @@ -249,6 +278,8 @@ protected void onJoinComplete(int generation, if (assignor == null) throw new IllegalStateException("Coordinator selected invalid assignment protocol: " + assignmentStrategy); + Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions()); + Assignment assignment = ConsumerProtocol.deserializeAssignment(assignmentBuffer); if (!subscriptions.assignFromSubscribed(assignment.partitions())) { handleAssignmentMismatch(assignment); @@ -270,14 +301,65 @@ protected void onJoinComplete(int generation, // execute the user's callback after rebalance ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); - log.info("Setting newly assigned partitions: {}", Utils.join(assignedPartitions, ", ")); - try { - listener.onPartitionsAssigned(assignedPartitions); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); + + switch (protocol) { + case EAGER: + if (!ownedPartitions.isEmpty()) { + throw new IllegalStateException("Some partitions are not revoked with EAGER rebalance protocol, this should never happen."); + } + + log.info("Setting newly assigned partitions: {}", Utils.join(assignedPartitions, ", ")); + try { + listener.onPartitionsAssigned(assignedPartitions); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); + } + break; + + case COOPERATIVE: + Set addedPartitions = new HashSet<>(assignedPartitions); + addedPartitions.removeAll(ownedPartitions); + + log.info("Updating with newly assigned partitions: {}, compare with already owned partitions: {}, " + + "Newly added partitions: {}, required revoking partitions: {}", + Utils.join(assignedPartitions, ", "), + Utils.join(ownedPartitions, ", "), + Utils.join(addedPartitions, ", "), + Utils.join(assignment.revokedPartitions(), ", ")); + + try { + listener.onPartitionsAssigned(addedPartitions); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); + } + + Set revokedPartitions = new HashSet<>(ownedPartitions); + revokedPartitions.removeAll(assignedPartitions); + + if (revokedPartitions.equals(new HashSet<>(assignment.revokedPartitions()))) { + throw new IllegalStateException("Some partitions are no longer assigned, but not included in the revoked partitions list;" + + "this should never happen"); + } + + if (!revokedPartitions.isEmpty()) { + try { + listener.onPartitionsRevoked(revokedPartitions); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); + } + } + + if (assignment.error() == ConsumerProtocol.Errors.NEED_REJOIN) { + requestRejoin(); + } } + } void maybeUpdateSubscriptionMetadata() { @@ -462,15 +544,30 @@ protected void onJoinPrepare(int generation, String memberId) { // execute the user's callback before rebalance ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); - // copy since about to be handed to user code - Set revoked = new HashSet<>(subscriptions.assignedPartitions()); - log.info("Revoking previously assigned partitions {}", revoked); - try { - listener.onPartitionsRevoked(revoked); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e); + + // with EAGER protocol always revoke all the assigned partitions; + // with COOPERATIVE protocol we will not revoke any partitions. + switch (protocol) { + case EAGER: + Set revoked = new HashSet<>(subscriptions.assignedPartitions()); + log.info("Revoking previously assigned partitions {}", revoked); + try { + listener.onPartitionsRevoked(revoked); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e); + } + + // also clear the assigned partitions since all have been revoked + if (!subscriptions.assignFromSubscribed(Collections.emptySet())){ + throw new IllegalStateException("Revoking all assigned partitions does not match with subscriptions, this should never happen."); + } + + break; + + case COOPERATIVE: + break; } isLeader = false; diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index e17a6efcd9f7a..9f6991d02bbd1 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -1870,7 +1870,8 @@ private KafkaConsumer newConsumer(Time time, autoCommitEnabled, autoCommitIntervalMs, interceptors, - true); + true, + ConsumerCoordinator.RebalanceProtocol.EAGER); Fetcher fetcher = new Fetcher<>( loggerFactory, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index 49bbcec206b0a..775c7fc1b6cf4 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -2142,7 +2142,8 @@ private ConsumerCoordinator buildCoordinator(final Metrics metrics, autoCommitEnabled, autoCommitIntervalMs, null, - leaveGroup); + leaveGroup, + ConsumerCoordinator.RebalanceProtocol.EAGER); } private FindCoordinatorResponse groupCoordinatorResponse(Node node, Errors error) { diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java index 8f8e96068dc65..7bcb7a1fc19e6 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java @@ -148,12 +148,7 @@ public void partitionAssignmentChangeOnPatternSubscription() { @Test public void verifyAssignmentListener() { final AtomicReference> assignmentRef = new AtomicReference<>(); - state.addListener(new SubscriptionState.Listener() { - @Override - public void onAssignment(Set assignment) { - assignmentRef.set(assignment); - } - }); + state.addListener(assignmentRef::set); Set userAssignment = Utils.mkSet(tp0, tp1); state.assignFromUser(userAssignment); assertEquals(userAssignment, assignmentRef.get()); From cb9fb493afefc95a345aa2f5577db002c343b8ff Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 2 Apr 2019 15:43:50 -0700 Subject: [PATCH 03/59] minor fix --- .../internals/ConsumerCoordinator.java | 134 ++++++++++++------ 1 file changed, 88 insertions(+), 46 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index f5487fb51be91..9204a0560f5dd 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -301,7 +301,7 @@ protected void onJoinComplete(int generation, // execute the user's callback after rebalance ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); - + switch (protocol) { case EAGER: if (!ownedPartitions.isEmpty()) { @@ -312,54 +312,69 @@ protected void onJoinComplete(int generation, try { listener.onPartitionsAssigned(assignedPartitions); } catch (WakeupException | InterruptException e) { + // also clear the assigned partitions + subscriptions.assignFromSubscribed(Collections.emptySet()); throw e; } catch (Exception e) { + subscriptions.assignFromSubscribed(Collections.emptySet()); log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); } break; case COOPERATIVE: - Set addedPartitions = new HashSet<>(assignedPartitions); - addedPartitions.removeAll(ownedPartitions); - - log.info("Updating with newly assigned partitions: {}, compare with already owned partitions: {}, " + - "Newly added partitions: {}, required revoking partitions: {}", - Utils.join(assignedPartitions, ", "), - Utils.join(ownedPartitions, ", "), - Utils.join(addedPartitions, ", "), - Utils.join(assignment.revokedPartitions(), ", ")); + assignAndRevoke(listener, assignedPartitions, ownedPartitions); - try { - listener.onPartitionsAssigned(addedPartitions); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); + if (assignment.error() == ConsumerProtocol.Errors.NEED_REJOIN) { + requestRejoin(); } - Set revokedPartitions = new HashSet<>(ownedPartitions); - revokedPartitions.removeAll(assignedPartitions); + break; + } + + } - if (revokedPartitions.equals(new HashSet<>(assignment.revokedPartitions()))) { - throw new IllegalStateException("Some partitions are no longer assigned, but not included in the revoked partitions list;" + - "this should never happen"); - } + private void assignAndRevoke(final ConsumerRebalanceListener listener, + final Set assignedPartitions, + final Set ownedPartitions) { + Set addedPartitions = new HashSet<>(assignedPartitions); + Set revokedPartitions = new HashSet<>(ownedPartitions); + addedPartitions.removeAll(ownedPartitions); + revokedPartitions.removeAll(assignedPartitions); - if (!revokedPartitions.isEmpty()) { - try { - listener.onPartitionsRevoked(revokedPartitions); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); - } - } + // TODO: ignored assignment.revokedPartitions - if (assignment.error() == ConsumerProtocol.Errors.NEED_REJOIN) { - requestRejoin(); - } + log.info("Updating with newly assigned partitions: {}, compare with already owned partitions: {}, " + + "newly added partitions: {}, revoking partitions: {}", + Utils.join(assignedPartitions, ", "), + Utils.join(ownedPartitions, ", "), + Utils.join(addedPartitions, ", "), + Utils.join(revokedPartitions, ", ")); + + try { + listener.onPartitionsAssigned(addedPartitions); + } catch (WakeupException | InterruptException e) { + // reset the assigned to owned partitions + subscriptions.assignFromSubscribed(ownedPartitions); + throw e; + } catch (Exception e) { + // reset the assigned to owned partitions + subscriptions.assignFromSubscribed(ownedPartitions); + log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); } + if (!revokedPartitions.isEmpty()) { + try { + listener.onPartitionsRevoked(revokedPartitions); + } catch (WakeupException | InterruptException e) { + // reset the assigned to owned partitions + subscriptions.assignFromSubscribed(ownedPartitions); + throw e; + } catch (Exception e) { + // reset the assigned to owned partitions + subscriptions.assignFromSubscribed(ownedPartitions); + log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e); + } + } } void maybeUpdateSubscriptionMetadata() { @@ -479,10 +494,13 @@ protected Map performAssignment(String leaderId, Set allSubscribedTopics = new HashSet<>(); Map subscriptions = new HashMap<>(); - for (JoinGroupResponseData.JoinGroupResponseMember memberSubScription : allSubscriptions) { - Subscription subscription = ConsumerProtocol.deserializeSubscription(ByteBuffer.wrap(memberSubScription.metadata())); - subscriptions.put(memberSubScription.memberId(), subscription); + // collect all the owned partitions + Map ownedPartitions = new HashMap<>(); + for (JoinGroupResponseData.JoinGroupResponseMember memberSubscription : allSubscriptions) { + Subscription subscription = ConsumerProtocol.deserializeSubscription(ByteBuffer.wrap(memberSubscription.metadata())); + subscriptions.put(memberSubscription.memberId(), subscription); allSubscribedTopics.addAll(subscription.topics()); + ownedPartitions.putAll(subscription.ownedPartitions().stream().collect(Collectors.toMap(item -> item, item -> memberSubscription.memberId()))); } // the leader will begin watching for changes to any of the topics the group is interested in, @@ -493,7 +511,16 @@ protected Map performAssignment(String leaderId, log.debug("Performing assignment using strategy {} with subscriptions {}", assignor.name(), subscriptions); - Map assignment = assignor.assign(metadata.fetch(), subscriptions); + Map assignments = assignor.assign(metadata.fetch(), subscriptions); + + switch (protocol) { + case EAGER: + break; + + case COOPERATIVE: + adjustAssignment(ownedPartitions, assignments); + break; + } // user-customized assignor may have created some topics that are not in the subscription list // and assign their partitions to the members; in this case we would like to update the leader's @@ -503,7 +530,7 @@ protected Map performAssignment(String leaderId, // TODO: this is a hack and not something we want to support long-term unless we push regex into the protocol // we may need to modify the PartitionAssignor API to better support this case. Set assignedTopics = new HashSet<>(); - for (Assignment assigned : assignment.values()) { + for (Assignment assigned : assignments.values()) { for (TopicPartition tp : assigned.partitions()) assignedTopics.add(tp.topic()); } @@ -526,10 +553,10 @@ protected Map performAssignment(String leaderId, assignmentSnapshot = metadataSnapshot; - log.debug("Finished assignment for group: {}", assignment); + log.debug("Finished assignment for group: {}", assignments); Map groupAssignment = new HashMap<>(); - for (Map.Entry assignmentEntry : assignment.entrySet()) { + for (Map.Entry assignmentEntry : assignments.entrySet()) { ByteBuffer buffer = ConsumerProtocol.serializeAssignment(assignmentEntry.getValue()); groupAssignment.put(assignmentEntry.getKey(), buffer); } @@ -537,6 +564,25 @@ protected Map performAssignment(String leaderId, return groupAssignment; } + private void adjustAssignment(final Map ownedPartitions, + final Map assignments) { + // update the assignment if the partition is owned by another different owner + Set assignedPartitions = new HashSet<>(); + for (final Map.Entry entry : assignments.entrySet()) { + final Assignment assignment = entry.getValue(); + assignedPartitions.addAll(assignment.partitions()); + assignment.partitions().removeIf(tp -> ownedPartitions.containsKey(tp) && !entry.getKey().equals(ownedPartitions.get(tp))); + } + + // for all owned but not assigned partitions, blindly add them to assignment + for (final Map.Entry entry : ownedPartitions.entrySet()) { + final TopicPartition tp = entry.getKey(); + if (!assignedPartitions.contains(tp)) { + assignments.get(entry.getValue()).partitions().add(tp); + } + } + } + @Override protected void onJoinPrepare(int generation, String memberId) { // commit offsets prior to rebalance if auto-commit enabled @@ -545,8 +591,6 @@ protected void onJoinPrepare(int generation, String memberId) { // execute the user's callback before rebalance ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); - // with EAGER protocol always revoke all the assigned partitions; - // with COOPERATIVE protocol we will not revoke any partitions. switch (protocol) { case EAGER: Set revoked = new HashSet<>(subscriptions.assignedPartitions()); @@ -560,9 +604,7 @@ protected void onJoinPrepare(int generation, String memberId) { } // also clear the assigned partitions since all have been revoked - if (!subscriptions.assignFromSubscribed(Collections.emptySet())){ - throw new IllegalStateException("Revoking all assigned partitions does not match with subscriptions, this should never happen."); - } + subscriptions.assignFromSubscribed(Collections.emptySet()); break; From f7ab5e272fa98fef6b25e0f5c779a6be769be37f Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 2 Apr 2019 15:52:00 -0700 Subject: [PATCH 04/59] set re-join error code --- .../consumer/internals/ConsumerCoordinator.java | 15 +++++++++++++-- .../consumer/internals/PartitionAssignor.java | 6 +++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 9204a0560f5dd..4d5c543cd9850 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -301,7 +301,7 @@ protected void onJoinComplete(int generation, // execute the user's callback after rebalance ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); - + switch (protocol) { case EAGER: if (!ownedPartitions.isEmpty()) { @@ -566,12 +566,16 @@ protected Map performAssignment(String leaderId, private void adjustAssignment(final Map ownedPartitions, final Map assignments) { + boolean revocationsNeeded = false; // update the assignment if the partition is owned by another different owner Set assignedPartitions = new HashSet<>(); for (final Map.Entry entry : assignments.entrySet()) { final Assignment assignment = entry.getValue(); assignedPartitions.addAll(assignment.partitions()); - assignment.partitions().removeIf(tp -> ownedPartitions.containsKey(tp) && !entry.getKey().equals(ownedPartitions.get(tp))); + + if (assignment.partitions().removeIf(tp -> ownedPartitions.containsKey(tp) && !entry.getKey().equals(ownedPartitions.get(tp)))) { + revocationsNeeded = true; + } } // for all owned but not assigned partitions, blindly add them to assignment @@ -581,6 +585,13 @@ private void adjustAssignment(final Map ownedPartitions, assignments.get(entry.getValue()).partitions().add(tp); } } + + // if revocations are triggered, tell everyone to re-join immediately. + if (revocationsNeeded) { + for (final Assignment assignment : assignments.values()) { + assignment.setError(ConsumerProtocol.Errors.NEED_REJOIN); + } + } } @Override diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java index ec4e2c3123aca..6d9899c102a4d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java @@ -116,7 +116,7 @@ class Assignment { private final List partitions; private final ByteBuffer userData; private final List revokedPartitions; - private final ConsumerProtocol.Errors error; + private ConsumerProtocol.Errors error; public Assignment(List partitions, ByteBuffer userData, List revokedPartitions, ConsumerProtocol.Errors error) { this.partitions = partitions; @@ -145,6 +145,10 @@ public ConsumerProtocol.Errors error() { return error; } + public void setError(ConsumerProtocol.Errors error) { + this.error = error; + } + public ByteBuffer userData() { return userData; } From 347ca5f46480811c8af6c8095c71dc3afaa8bc74 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 2 Apr 2019 17:09:24 -0700 Subject: [PATCH 05/59] add emigrate callback --- .../consumer/ConsumerRebalanceListener.java | 25 ++++++++- .../internals/ConsumerCoordinator.java | 52 +++++++++++++------ 2 files changed, 60 insertions(+), 17 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java index 74e8b060c73fa..7148f20cd4547 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java @@ -95,7 +95,7 @@ public interface ConsumerRebalanceListener { * invocation of {@link KafkaConsumer#poll(java.time.Duration)} in which this callback is being executed. This means it is not * necessary to catch these exceptions and re-attempt to wakeup or interrupt the consumer thread. * - * @param partitions The list of partitions that were assigned to the consumer on the last rebalance + * @param partitions The list of partitions that were assigned to the consumer and now need to be revoked * @throws org.apache.kafka.common.errors.WakeupException If raised from a nested call to {@link KafkaConsumer} * @throws org.apache.kafka.common.errors.InterruptException If raised from a nested call to {@link KafkaConsumer} */ @@ -122,4 +122,27 @@ public interface ConsumerRebalanceListener { * @throws org.apache.kafka.common.errors.InterruptException If raised from a nested call to {@link KafkaConsumer} */ void onPartitionsAssigned(Collection partitions); + + /** + * A callback method the user can implement to provide handling of cleaning up resources for partitions that have already + * been re-assigned to other consumers. This method will be called during normal execution as the owned partitions would + * first be revoked by calling the {@link ConsumerRebalanceListener#onPartitionsRevoked} first, before being re-assigned + * to other consumers. However, when the consumer is being kicked out of the group and hence were not aware when the new + * group is formed without itself, this function will then be called when the consumer finally be notified about the + * partitions that have already been emigrated to other consumers. + * + * It is possible + * for a {@link org.apache.kafka.common.errors.WakeupException} or {@link org.apache.kafka.common.errors.InterruptException} + * to be raised from one these nested invocations. In this case, the exception will be propagated to the current + * invocation of {@link KafkaConsumer#poll(java.time.Duration)} in which this callback is being executed. This means it is not + * necessary to catch these exceptions and re-attempt to wakeup or interrupt the consumer thread. + * + * @param partitions The list of partitions that were assigned to the consumer and now have been re-assigned + * to other consumers + * @throws org.apache.kafka.common.errors.WakeupException If raised from a nested call to {@link KafkaConsumer} + * @throws org.apache.kafka.common.errors.InterruptException If raised from a nested call to {@link KafkaConsumer} + */ + default void onPartitionsEmigrated(Collection partitions) { + onPartitionsAssigned(partitions); + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 4d5c543cd9850..c89662e8fb84d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -42,6 +42,7 @@ import org.apache.kafka.common.metrics.stats.Avg; import org.apache.kafka.common.metrics.stats.Max; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.JoinGroupResponse; import org.apache.kafka.common.requests.OffsetCommitRequest; import org.apache.kafka.common.requests.OffsetCommitResponse; import org.apache.kafka.common.requests.OffsetFetchRequest; @@ -602,25 +603,44 @@ protected void onJoinPrepare(int generation, String memberId) { // execute the user's callback before rebalance ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); - switch (protocol) { - case EAGER: - Set revoked = new HashSet<>(subscriptions.assignedPartitions()); - log.info("Revoking previously assigned partitions {}", revoked); - try { - listener.onPartitionsRevoked(revoked); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e); - } + // if member.id is unknown, it means a reset generation, + // hence we should always call onPartitionsEmigrated to all + // previously assigned partitions regardless of the rebalance protocol + if (memberId.equals(JoinGroupResponse.UNKNOWN_MEMBER_ID)) { + Set emigrated = new HashSet<>(subscriptions.assignedPartitions()); + log.info("Emigrating previously assigned partitions {}", emigrated); - // also clear the assigned partitions since all have been revoked - subscriptions.assignFromSubscribed(Collections.emptySet()); + try { + listener.onPartitionsEmigrated(emigrated); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition emigration", listener.getClass().getName(), e); + } - break; + // also clear the assigned partitions since all have been emigrated + subscriptions.assignFromSubscribed(Collections.emptySet()); + } else { + switch (protocol) { + case EAGER: + Set revoked = new HashSet<>(subscriptions.assignedPartitions()); + log.info("Revoking previously assigned partitions {}", revoked); + try { + listener.onPartitionsEmigrated(revoked); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e); + } - case COOPERATIVE: - break; + // also clear the assigned partitions since all have been revoked + subscriptions.assignFromSubscribed(Collections.emptySet()); + + break; + + case COOPERATIVE: + break; + } } isLeader = false; From b290283e3c7bdd492964ab98d7349169383b8767 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 2 Apr 2019 17:12:32 -0700 Subject: [PATCH 06/59] KAFKA-7190: KIP-443; Remove streams overrides on repartition topics (#6511) As described in KIP-443 (https://cwiki.apache.org/confluence/display/KAFKA/KIP-443%3A+Return+to+default+segment.ms+and+segment.index.bytes+in+Streams+repartition+topics). We want to remove the aggressive overrides of segment.ms and segment.index.bytes for repartition topics. The remaining segment.bytes should still be effective in bounding its footprint. Reviewers: Bill Bejeck --- .../org/apache/kafka/streams/StreamsConfig.java | 15 +++++---------- .../internals/RepartitionTopicConfig.java | 6 ++---- .../integration/InternalTopicIntegrationTest.java | 4 ++-- .../internals/InternalTopologyBuilderTest.java | 2 +- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java index 2ba73128528f5..f607d1d19ced7 100644 --- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java @@ -1003,17 +1003,12 @@ public Map getMainConsumerConfigs(final String groupId, final St // verify that producer batch config is no larger than segment size, then add topic configs required for creating topics final Map topicProps = originalsWithPrefix(TOPIC_PREFIX, false); + final Map producerProps = getClientPropsWithPrefix(PRODUCER_PREFIX, ProducerConfig.configNames()); - if (topicProps.containsKey(topicPrefix(TopicConfig.SEGMENT_INDEX_BYTES_CONFIG))) { - final int segmentSize = Integer.parseInt(topicProps.get(topicPrefix(TopicConfig.SEGMENT_INDEX_BYTES_CONFIG)).toString()); - final Map producerProps = getClientPropsWithPrefix(PRODUCER_PREFIX, ProducerConfig.configNames()); - final int batchSize; - if (producerProps.containsKey(ProducerConfig.BATCH_SIZE_CONFIG)) { - batchSize = Integer.parseInt(producerProps.get(ProducerConfig.BATCH_SIZE_CONFIG).toString()); - } else { - final ProducerConfig producerDefaultConfig = new ProducerConfig(new Properties()); - batchSize = producerDefaultConfig.getInt(ProducerConfig.BATCH_SIZE_CONFIG); - } + if (topicProps.containsKey(topicPrefix(TopicConfig.SEGMENT_BYTES_CONFIG)) && + producerProps.containsKey(ProducerConfig.BATCH_SIZE_CONFIG)) { + final int segmentSize = Integer.parseInt(topicProps.get(topicPrefix(TopicConfig.SEGMENT_BYTES_CONFIG)).toString()); + final int batchSize = Integer.parseInt(producerProps.get(ProducerConfig.BATCH_SIZE_CONFIG).toString()); if (segmentSize < batchSize) { throw new IllegalArgumentException(String.format("Specified topic segment size %d is is smaller than the configured producer batch size %d, this will cause produced batch not able to be appended to the topic", diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java index 466520e393635..7161a3fc88030 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/RepartitionTopicConfig.java @@ -33,10 +33,8 @@ public class RepartitionTopicConfig extends InternalTopicConfig { static { final Map tempTopicDefaultOverrides = new HashMap<>(); tempTopicDefaultOverrides.put(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE); - tempTopicDefaultOverrides.put(TopicConfig.SEGMENT_INDEX_BYTES_CONFIG, "52428800"); // 50 MB - tempTopicDefaultOverrides.put(TopicConfig.SEGMENT_BYTES_CONFIG, "52428800"); // 50 MB - tempTopicDefaultOverrides.put(TopicConfig.SEGMENT_MS_CONFIG, "600000"); // 10 min - tempTopicDefaultOverrides.put(TopicConfig.RETENTION_MS_CONFIG, String.valueOf(Long.MAX_VALUE)); // Infinity + tempTopicDefaultOverrides.put(TopicConfig.SEGMENT_BYTES_CONFIG, "52428800"); // 50 MB + tempTopicDefaultOverrides.put(TopicConfig.RETENTION_MS_CONFIG, String.valueOf(-1)); // Infinity REPARTITION_TOPIC_DEFAULT_OVERRIDES = Collections.unmodifiableMap(tempTopicDefaultOverrides); } diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java index 8bcaf5d97ec6c..345b581f22870 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/InternalTopicIntegrationTest.java @@ -171,7 +171,7 @@ public void shouldCompactTopicsForKeyValueStoreChangelogs() throws Exception { final Properties repartitionProps = getTopicProperties(appID + "-Counts-repartition"); assertEquals(LogConfig.Delete(), repartitionProps.getProperty(LogConfig.CleanupPolicyProp())); - assertEquals(5, repartitionProps.size()); + assertEquals(3, repartitionProps.size()); } @Test @@ -216,6 +216,6 @@ public void shouldCompactAndDeleteTopicsForWindowStoreChangelogs() throws Except final Properties repartitionProps = getTopicProperties(appID + "-CountWindows-repartition"); assertEquals(LogConfig.Delete(), repartitionProps.getProperty(LogConfig.CleanupPolicyProp())); - assertEquals(5, repartitionProps.size()); + assertEquals(3, repartitionProps.size()); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java index 1a46af8631480..fbeae18d25211 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/InternalTopologyBuilderTest.java @@ -643,7 +643,7 @@ public void shouldAddInternalTopicConfigForRepartitionTopics() { final InternalTopologyBuilder.TopicsInfo topicsInfo = builder.topicGroups().values().iterator().next(); final InternalTopicConfig topicConfig = topicsInfo.repartitionSourceTopics.get("appId-foo"); final Map properties = topicConfig.getProperties(Collections.emptyMap(), 10000); - assertEquals(5, properties.size()); + assertEquals(3, properties.size()); assertEquals(String.valueOf(Long.MAX_VALUE), properties.get(TopicConfig.RETENTION_MS_CONFIG)); assertEquals(TopicConfig.CLEANUP_POLICY_DELETE, properties.get(TopicConfig.CLEANUP_POLICY_CONFIG)); assertEquals("appId-foo", topicConfig.name()); From d3af860dc70208ebd7f2e5c33c379845d6f7e5b1 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 2 Apr 2019 17:26:26 -0700 Subject: [PATCH 07/59] typo fix --- .../kafka/clients/consumer/ConsumerRebalanceListener.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java index 7148f20cd4547..4f764dde5cb7d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java @@ -131,6 +131,9 @@ public interface ConsumerRebalanceListener { * group is formed without itself, this function will then be called when the consumer finally be notified about the * partitions that have already been emigrated to other consumers. * + * By default it will just trigger {@link ConsumerRebalanceListener#onPartitionsRevoked}; for advanced users who want to distinguish + * the handling logic of revoked partitions v.s. emigrated partitions, they can override the default implementation. + * * It is possible * for a {@link org.apache.kafka.common.errors.WakeupException} or {@link org.apache.kafka.common.errors.InterruptException} * to be raised from one these nested invocations. In this case, the exception will be propagated to the current @@ -143,6 +146,6 @@ public interface ConsumerRebalanceListener { * @throws org.apache.kafka.common.errors.InterruptException If raised from a nested call to {@link KafkaConsumer} */ default void onPartitionsEmigrated(Collection partitions) { - onPartitionsAssigned(partitions); + onPartitionsRevoked(partitions); } } From cb25f6c0df3b75953e79876869da1e677e0d97c2 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Thu, 4 Apr 2019 11:31:21 -0700 Subject: [PATCH 08/59] github comments --- .../consumer/internals/ConsumerProtocol.java | 30 +++---------------- .../consumer/internals/PartitionAssignor.java | 10 ++----- .../internals/ConsumerProtocolTest.java | 11 +------ 3 files changed, 7 insertions(+), 44 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java index 96d8fa21665bc..ed2df86edb62e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java @@ -34,7 +34,7 @@ /** * ConsumerProtocol contains the schemas for consumer subscriptions and assignments for use with - * Kafka's generalized group management protocol. Below is the version 0 format: + * Kafka's generalized group management protocol. Below is the version 1 format: * * 
  * Subscription => Version Topics
@@ -51,12 +51,11 @@
  *     Topic            => String
  *     Partitions       => [int32]
  *     UserData   => Bytes
- *   RevokedPartitions  => [Topic Partitions]
- *     Topic            => String
- *     Partitions       => [int32]
  *   ErrorCode          => [int16]
  *
* + * Older versioned formats can be inferred by reading the code below. + * * The current implementation assumes that future versions will not break compatibility. When * it encounters a newer version, it parses it using the current format. This basically means * that new versions cannot remove or reorder any of the existing fields. @@ -70,7 +69,6 @@ public class ConsumerProtocol { public static final String TOPIC_KEY_NAME = "topic"; public static final String PARTITIONS_KEY_NAME = "partitions"; public static final String OWNED_PARTITIONS_KEY_NAME = "owned_partitions"; - public static final String REVOKED_PARTITIONS_KEY_NAME = "revoked_partitions"; public static final String TOPIC_PARTITIONS_KEY_NAME = "topic_partitions"; public static final String USER_DATA_KEY_NAME = "user_data"; @@ -104,7 +102,6 @@ public class ConsumerProtocol { public static final Schema ASSIGNMENT_V1 = new Schema( new Field(TOPIC_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)), new Field(USER_DATA_KEY_NAME, Type.NULLABLE_BYTES), - new Field(REVOKED_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)), ERROR_CODE); public enum Errors { @@ -164,7 +161,6 @@ public static ByteBuffer serializeSubscriptionV1(PartitionAssignor.Subscription SUBSCRIPTION_V1.write(buffer, struct); buffer.flip(); return buffer; - } public static ByteBuffer serializeSubscription(PartitionAssignor.Subscription subscription) { @@ -250,15 +246,6 @@ public static ByteBuffer serializeAssignmentV1(PartitionAssignor.Assignment assi topicAssignments.add(topicAssignment); } struct.set(TOPIC_PARTITIONS_KEY_NAME, topicAssignments.toArray()); - List revokedAssignments = new ArrayList<>(); - partitionsByTopic = CollectionUtils.groupPartitionsByTopic(assignment.revokedPartitions()); - for (Map.Entry> topicEntry : partitionsByTopic.entrySet()) { - Struct topicAssignment = new Struct(TOPIC_ASSIGNMENT_V0); - topicAssignment.set(TOPIC_KEY_NAME, topicEntry.getKey()); - topicAssignment.set(PARTITIONS_KEY_NAME, topicEntry.getValue().toArray()); - revokedAssignments.add(topicAssignment); - } - struct.set(REVOKED_PARTITIONS_KEY_NAME, revokedAssignments.toArray()); struct.set(ERROR_CODE.name, assignment.error().code); ByteBuffer buffer = ByteBuffer.allocate(CONSUMER_PROTOCOL_HEADER_V1.sizeOf() + ASSIGNMENT_V1.sizeOf(struct)); @@ -300,18 +287,9 @@ public static PartitionAssignor.Assignment deserializeAssignmentV1(ByteBuffer bu } } - List revokedPartitions = new ArrayList<>(); - for (Object structObj : struct.getArray(REVOKED_PARTITIONS_KEY_NAME)) { - Struct assignment = (Struct) structObj; - String topic = assignment.getString(TOPIC_KEY_NAME); - for (Object partitionObj : assignment.getArray(PARTITIONS_KEY_NAME)) { - Integer partition = (Integer) partitionObj; - revokedPartitions.add(new TopicPartition(topic, partition)); - } - } Errors error = Errors.fromCode(struct.get(ERROR_CODE)); - return new PartitionAssignor.Assignment(partitions, userData, revokedPartitions, error); + return new PartitionAssignor.Assignment(partitions, userData, error); } public static PartitionAssignor.Assignment deserializeAssignment(ByteBuffer buffer) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java index ec4e2c3123aca..c1d35395d6db7 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java @@ -115,18 +115,16 @@ public String toString() { class Assignment { private final List partitions; private final ByteBuffer userData; - private final List revokedPartitions; private final ConsumerProtocol.Errors error; - public Assignment(List partitions, ByteBuffer userData, List revokedPartitions, ConsumerProtocol.Errors error) { + public Assignment(List partitions, ByteBuffer userData, ConsumerProtocol.Errors error) { this.partitions = partitions; this.userData = userData; - this.revokedPartitions = revokedPartitions; this.error = error; } public Assignment(List partitions, ByteBuffer userData) { - this(partitions, userData, Collections.emptyList(), ConsumerProtocol.Errors.NONE); + this(partitions, userData, ConsumerProtocol.Errors.NONE); } public Assignment(List partitions) { @@ -137,10 +135,6 @@ public List partitions() { return partitions; } - public List revokedPartitions() { - return revokedPartitions; - } - public ConsumerProtocol.Errors error() { return error; } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java index d48a1986d5d9b..cb6a36cc4420c 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java @@ -37,7 +37,6 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.CONSUMER_PROTOCOL_HEADER_SCHEMA; import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.OWNED_PARTITIONS_KEY_NAME; -import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.REVOKED_PARTITIONS_KEY_NAME; import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.TOPICS_KEY_NAME; import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.TOPIC_ASSIGNMENT_V0; import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.TOPIC_PARTITIONS_KEY_NAME; @@ -152,21 +151,19 @@ public void deserializeOldAssignmentVersion() { Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer); assertEquals(toSet(partitions), toSet(parsedAssignment.partitions())); assertEquals(0, parsedAssignment.userData().limit()); - assertTrue(parsedAssignment.revokedPartitions().isEmpty()); assertEquals(Errors.NONE, parsedAssignment.error()); } @Test public void deserializeNewAssignmentVersion() { List partitions = Collections.singletonList(tp1); - ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment(partitions, null, Collections.singletonList(tp2), Errors.NEED_REJOIN)); + ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment(partitions, null, Errors.NEED_REJOIN)); // ignore the version assuming it is the old byte code, as it will blindly deserialize as 0 Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer); header.getShort(VERSION_KEY_NAME); Assignment parsedAssignment = ConsumerProtocol.deserializeAssignmentV0(buffer); assertEquals(toSet(partitions), toSet(parsedAssignment.partitions())); assertNull(parsedAssignment.userData()); - assertTrue(parsedAssignment.revokedPartitions().isEmpty()); assertEquals(Errors.NONE, parsedAssignment.error()); } @@ -178,7 +175,6 @@ public void deserializeFutureAssignmentVersion() { Schema assignmentSchemaV100 = new Schema( new Field(TOPIC_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)), new Field(USER_DATA_KEY_NAME, Type.BYTES), - new Field(REVOKED_PARTITIONS_KEY_NAME, new ArrayOf(TOPIC_ASSIGNMENT_V0)), ERROR_CODE, new Field("foo", Type.STRING)); @@ -188,10 +184,6 @@ public void deserializeFutureAssignmentVersion() { .set(ConsumerProtocol.TOPIC_KEY_NAME, tp1.topic()) .set(ConsumerProtocol.PARTITIONS_KEY_NAME, new Object[]{tp1.partition()})}); assignmentV100.set(USER_DATA_KEY_NAME, ByteBuffer.wrap(new byte[0])); - assignmentV100.set(REVOKED_PARTITIONS_KEY_NAME, - new Object[]{new Struct(TOPIC_ASSIGNMENT_V0) - .set(ConsumerProtocol.TOPIC_KEY_NAME, tp2.topic()) - .set(ConsumerProtocol.PARTITIONS_KEY_NAME, new Object[]{tp2.partition()})}); assignmentV100.set(ERROR_CODE.name, Errors.NEED_REJOIN.code()); assignmentV100.set("foo", "bar"); @@ -206,7 +198,6 @@ public void deserializeFutureAssignmentVersion() { PartitionAssignor.Assignment assignment = ConsumerProtocol.deserializeAssignment(buffer); assertEquals(toSet(Collections.singletonList(tp1)), toSet(assignment.partitions())); - assertEquals(toSet(Collections.singletonList(tp2)), toSet(assignment.revokedPartitions())); assertEquals(Errors.NEED_REJOIN, assignment.error()); } From 0a4aa1df4846f6ff7443b0e10da704364ce4cca1 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Thu, 4 Apr 2019 13:03:09 -0700 Subject: [PATCH 09/59] remove logic for handling rebalance callback errors --- .../consumer/internals/ConsumerCoordinator.java | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 4d5c543cd9850..efb4060e9f1f5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -312,11 +312,8 @@ protected void onJoinComplete(int generation, try { listener.onPartitionsAssigned(assignedPartitions); } catch (WakeupException | InterruptException e) { - // also clear the assigned partitions - subscriptions.assignFromSubscribed(Collections.emptySet()); throw e; } catch (Exception e) { - subscriptions.assignFromSubscribed(Collections.emptySet()); log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); } break; @@ -341,8 +338,6 @@ private void assignAndRevoke(final ConsumerRebalanceListener listener, addedPartitions.removeAll(ownedPartitions); revokedPartitions.removeAll(assignedPartitions); - // TODO: ignored assignment.revokedPartitions - log.info("Updating with newly assigned partitions: {}, compare with already owned partitions: {}, " + "newly added partitions: {}, revoking partitions: {}", Utils.join(assignedPartitions, ", "), @@ -353,12 +348,8 @@ private void assignAndRevoke(final ConsumerRebalanceListener listener, try { listener.onPartitionsAssigned(addedPartitions); } catch (WakeupException | InterruptException e) { - // reset the assigned to owned partitions - subscriptions.assignFromSubscribed(ownedPartitions); throw e; } catch (Exception e) { - // reset the assigned to owned partitions - subscriptions.assignFromSubscribed(ownedPartitions); log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); } @@ -366,12 +357,8 @@ private void assignAndRevoke(final ConsumerRebalanceListener listener, try { listener.onPartitionsRevoked(revokedPartitions); } catch (WakeupException | InterruptException e) { - // reset the assigned to owned partitions - subscriptions.assignFromSubscribed(ownedPartitions); throw e; } catch (Exception e) { - // reset the assigned to owned partitions - subscriptions.assignFromSubscribed(ownedPartitions); log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e); } } @@ -567,12 +554,12 @@ protected Map performAssignment(String leaderId, private void adjustAssignment(final Map ownedPartitions, final Map assignments) { boolean revocationsNeeded = false; - // update the assignment if the partition is owned by another different owner Set assignedPartitions = new HashSet<>(); for (final Map.Entry entry : assignments.entrySet()) { final Assignment assignment = entry.getValue(); assignedPartitions.addAll(assignment.partitions()); + // update the assignment if the partition is owned by another different owner if (assignment.partitions().removeIf(tp -> ownedPartitions.containsKey(tp) && !entry.getKey().equals(ownedPartitions.get(tp)))) { revocationsNeeded = true; } From 3f2fe2eb0ad141cb117e6da00a6dd50f780322af Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Fri, 5 Apr 2019 09:38:19 -0700 Subject: [PATCH 10/59] fixes on unit tests, add more unit tests --- .../internals/ConsumerCoordinator.java | 3 +- .../internals/ConsumerCoordinatorTest.java | 236 ++++++++++++------ .../internals/ConsumerProtocolTest.java | 8 +- .../java/org/apache/kafka/test/TestUtils.java | 5 + 4 files changed, 167 insertions(+), 85 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index efb4060e9f1f5..289b0b6ec7a53 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -305,7 +305,8 @@ protected void onJoinComplete(int generation, switch (protocol) { case EAGER: if (!ownedPartitions.isEmpty()) { - throw new IllegalStateException("Some partitions are not revoked with EAGER rebalance protocol, this should never happen."); + log.warn("Some partitions are not revoked with EAGER rebalance protocol, " + + "it is likely that the previous rebalance did not complete due to some errors"); } log.info("Setting newly assigned partitions: {}", Utils.join(assignedPartitions, ", ")); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index 775c7fc1b6cf4..269f8022205d5 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -57,11 +57,14 @@ import org.apache.kafka.common.requests.SyncGroupResponse; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.Utils; import org.apache.kafka.test.TestCondition; import org.apache.kafka.test.TestUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -88,6 +91,7 @@ import static java.util.Collections.singleton; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; +import static org.apache.kafka.test.TestUtils.toSet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -96,6 +100,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +@RunWith(value = Parameterized.class) public class ConsumerCoordinatorTest { private final String topic1 = "test1"; private final String topic2 = "test2"; @@ -111,6 +116,7 @@ public class ConsumerCoordinatorTest { private final MockTime time = new MockTime(); private final Heartbeat heartbeat = new Heartbeat(time, sessionTimeoutMs, heartbeatIntervalMs, rebalanceTimeoutMs, retryBackoffMs); + private ConsumerCoordinator.RebalanceProtocol protocol; private MockPartitionAssignor partitionAssignor = new MockPartitionAssignor(); private List assignors = Collections.singletonList(partitionAssignor); @@ -130,6 +136,19 @@ public class ConsumerCoordinatorTest { private MockCommitCallback mockOffsetCommitCallback; private ConsumerCoordinator coordinator; + public ConsumerCoordinatorTest(final ConsumerCoordinator.RebalanceProtocol protocol) { + this.protocol = protocol; + } + + @Parameterized.Parameters(name = "rebalance protocol = {0}") + public static Collection data() { + final List values = new ArrayList<>(); + for (final ConsumerCoordinator.RebalanceProtocol protocol: ConsumerCoordinator.RebalanceProtocol.values()) { + values.add(new Object[]{protocol}); + } + return values; + } + @Before public void setup() { LogContext logContext = new LogContext(); @@ -145,7 +164,7 @@ public void setup() { this.mockOffsetCommitCallback = new MockCommitCallback(); this.partitionAssignor.clear(); - this.coordinator = buildCoordinator(metrics, assignors, false, true); + this.coordinator = buildCoordinator(metrics, assignors, false, true, protocol); } @After @@ -380,6 +399,9 @@ public void testJoinGroupInvalidGroupId() { @Test public void testNormalJoinGroupLeader() { final String consumerId = "leader"; + final Set subscription = singleton(topic1); + final List owned = Collections.emptyList(); + final List assigned = Arrays.asList(t1p); subscriptions.subscribe(singleton(topic1), rebalanceListener); @@ -391,7 +413,7 @@ public void testNormalJoinGroupLeader() { // normal join group Map> memberSubscriptions = singletonMap(consumerId, singletonList(topic1)); - partitionAssignor.prepare(singletonMap(consumerId, singletonList(t1p))); + partitionAssignor.prepare(singletonMap(consumerId, assigned)); client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE)); client.prepareResponse(new MockClient.RequestMatcher() { @@ -402,16 +424,18 @@ public boolean matches(AbstractRequest body) { sync.generationId() == 1 && sync.groupAssignment().containsKey(consumerId); } - }, syncGroupResponse(singletonList(t1p), Errors.NONE)); + }, syncGroupResponse(assigned, Errors.NONE)); coordinator.poll(time.timer(Long.MAX_VALUE)); + final int revokeCount = getRevoked(owned, assigned) == null ? 0 : 1; + final int addCount = getAdded(owned, assigned) == null ? 0 : 1; assertFalse(coordinator.rejoinNeededOrPending()); - assertEquals(singleton(t1p), subscriptions.assignedPartitions()); - assertEquals(singleton(topic1), subscriptions.groupSubscription()); - assertEquals(1, rebalanceListener.revokedCount); - assertEquals(Collections.emptySet(), rebalanceListener.revoked); - assertEquals(1, rebalanceListener.assignedCount); - assertEquals(singleton(t1p), rebalanceListener.assigned); + assertEquals(toSet(assigned), subscriptions.assignedPartitions()); + assertEquals(subscription, subscriptions.groupSubscription()); + assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); + assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @Test @@ -424,7 +448,7 @@ public void testOutdatedCoordinatorAssignment() { coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); // Test coordinator returning unsubscribed partitions - partitionAssignor.prepare(singletonMap(consumerId, singletonList(t1p))); + partitionAssignor.prepare(singletonMap(consumerId, Arrays.asList(t1p))); // First incorrect assignment for subscription client.prepareResponse( @@ -483,7 +507,7 @@ public void testInvalidCoordinatorAssignment() { // normal join group Map> memberSubscriptions = singletonMap(consumerId, singletonList(topic2)); - partitionAssignor.prepare(singletonMap(consumerId, singletonList(t2p))); + partitionAssignor.prepare(singletonMap(consumerId, Arrays.asList(t2p))); client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE)); client.prepareResponse(new MockClient.RequestMatcher() { @@ -494,13 +518,15 @@ public boolean matches(AbstractRequest body) { sync.generationId() == 1 && sync.groupAssignment().containsKey(consumerId); } - }, syncGroupResponse(singletonList(t2p), Errors.NONE)); + }, syncGroupResponse(Collections.singletonList(t2p), Errors.NONE)); assertThrows(IllegalStateException.class, () -> coordinator.poll(time.timer(Long.MAX_VALUE))); } @Test public void testPatternJoinGroupLeader() { final String consumerId = "leader"; + final List assigned = Arrays.asList(t1p, t2p); + final List owned = Collections.emptyList(); subscriptions.subscribe(Pattern.compile("test.*"), rebalanceListener); @@ -513,7 +539,7 @@ public void testPatternJoinGroupLeader() { // normal join group Map> memberSubscriptions = singletonMap(consumerId, singletonList(topic1)); - partitionAssignor.prepare(singletonMap(consumerId, Arrays.asList(t1p, t2p))); + partitionAssignor.prepare(singletonMap(consumerId, assigned)); client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE)); client.prepareResponse(new MockClient.RequestMatcher() { @@ -524,20 +550,22 @@ public boolean matches(AbstractRequest body) { sync.generationId() == 1 && sync.groupAssignment().containsKey(consumerId); } - }, syncGroupResponse(Arrays.asList(t1p, t2p), Errors.NONE)); + }, syncGroupResponse(assigned, Errors.NONE)); // expect client to force updating the metadata, if yes gives it both topics client.prepareMetadataUpdate(metadataResponse); coordinator.poll(time.timer(Long.MAX_VALUE)); + final int revokeCount = getRevoked(owned, assigned) == null ? 0 : 1; + final int addCount = getAdded(owned, assigned) == null ? 0 : 1; assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(2, subscriptions.numAssignedPartitions()); assertEquals(2, subscriptions.groupSubscription().size()); assertEquals(2, subscriptions.subscription().size()); - assertEquals(1, rebalanceListener.revokedCount); - assertEquals(Collections.emptySet(), rebalanceListener.revoked); - assertEquals(1, rebalanceListener.assignedCount); - assertEquals(2, rebalanceListener.assigned.size()); + assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); + assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @Test @@ -554,7 +582,7 @@ public void testMetadataRefreshDuringRebalance() { coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); Map> initialSubscription = singletonMap(consumerId, singletonList(topic1)); - partitionAssignor.prepare(singletonMap(consumerId, singletonList(t1p))); + partitionAssignor.prepare(singletonMap(consumerId, Arrays.asList(t1p))); // the metadata will be updated in flight with a new topic added final List updatedSubscription = Arrays.asList(topic1, topic2); @@ -655,6 +683,8 @@ public boolean matches(AbstractRequest body) { @Test public void testWakeupDuringJoin() { final String consumerId = "leader"; + final List owned = Collections.emptyList(); + final List assigned = Arrays.asList(t1p); subscriptions.subscribe(singleton(topic1), rebalanceListener); @@ -665,7 +695,7 @@ public void testWakeupDuringJoin() { coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); Map> memberSubscriptions = singletonMap(consumerId, singletonList(topic1)); - partitionAssignor.prepare(singletonMap(consumerId, singletonList(t1p))); + partitionAssignor.prepare(singletonMap(consumerId, assigned)); // prepare only the first half of the join and then trigger the wakeup client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE)); @@ -678,22 +708,27 @@ public void testWakeupDuringJoin() { } // now complete the second half - client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE)); + client.prepareResponse(syncGroupResponse(assigned, Errors.NONE)); coordinator.poll(time.timer(Long.MAX_VALUE)); + final int revokeCount = getRevoked(owned, assigned) == null ? 0 : 1; + final int addCount = getAdded(owned, assigned) == null ? 0 : 1; assertFalse(coordinator.rejoinNeededOrPending()); - assertEquals(singleton(t1p), subscriptions.assignedPartitions()); - assertEquals(1, rebalanceListener.revokedCount); - assertEquals(Collections.emptySet(), rebalanceListener.revoked); - assertEquals(1, rebalanceListener.assignedCount); - assertEquals(singleton(t1p), rebalanceListener.assigned); + assertEquals(toSet(assigned), subscriptions.assignedPartitions()); + assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); + assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @Test public void testNormalJoinGroupFollower() { final String consumerId = "consumer"; + final Set subscription = singleton(topic1); + final List owned = Collections.emptyList(); + final List assigned = Arrays.asList(t1p); - subscriptions.subscribe(singleton(topic1), rebalanceListener); + subscriptions.subscribe(subscription, rebalanceListener); client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); @@ -708,17 +743,19 @@ public boolean matches(AbstractRequest body) { sync.generationId() == 1 && sync.groupAssignment().isEmpty(); } - }, syncGroupResponse(singletonList(t1p), Errors.NONE)); + }, syncGroupResponse(assigned, Errors.NONE)); coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE)); + final int revokeCount = getRevoked(owned, assigned) == null ? 0 : 1; + final int addCount = getAdded(owned, assigned) == null ? 0 : 1; assertFalse(coordinator.rejoinNeededOrPending()); - assertEquals(singleton(t1p), subscriptions.assignedPartitions()); - assertEquals(singleton(topic1), subscriptions.groupSubscription()); - assertEquals(1, rebalanceListener.revokedCount); - assertEquals(Collections.emptySet(), rebalanceListener.revoked); - assertEquals(1, rebalanceListener.assignedCount); - assertEquals(singleton(t1p), rebalanceListener.assigned); + assertEquals(toSet(assigned), subscriptions.assignedPartitions()); + assertEquals(subscription, subscriptions.groupSubscription()); + assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); + assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @Test @@ -753,6 +790,9 @@ public void testUpdateLastHeartbeatPollWhenCoordinatorUnknown() throws Exception @Test public void testPatternJoinGroupFollower() { final String consumerId = "consumer"; + final Set subscription = Utils.mkSet(topic1, topic2); + final List owned = Collections.emptyList(); + final List assigned = Arrays.asList(t1p, t2p); subscriptions.subscribe(Pattern.compile("test.*"), rebalanceListener); @@ -773,18 +813,21 @@ public boolean matches(AbstractRequest body) { sync.generationId() == 1 && sync.groupAssignment().isEmpty(); } - }, syncGroupResponse(Arrays.asList(t1p, t2p), Errors.NONE)); + }, syncGroupResponse(assigned, Errors.NONE)); // expect client to force updating the metadata, if yes gives it both topics client.prepareMetadataUpdate(metadataResponse); coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE)); + final int revokeCount = getRevoked(owned, assigned) == null ? 0 : 1; + final int addCount = getAdded(owned, assigned) == null ? 0 : 1; assertFalse(coordinator.rejoinNeededOrPending()); - assertEquals(2, subscriptions.numAssignedPartitions()); - assertEquals(2, subscriptions.subscription().size()); - assertEquals(1, rebalanceListener.revokedCount); - assertEquals(1, rebalanceListener.assignedCount); - assertEquals(2, rebalanceListener.assigned.size()); + assertEquals(assigned.size(), subscriptions.numAssignedPartitions()); + assertEquals(subscription, subscriptions.subscription()); + assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); + assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @Test @@ -975,7 +1018,7 @@ public void testMetadataChangeTriggersRebalance() { coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); Map> memberSubscriptions = singletonMap(consumerId, singletonList(topic1)); - partitionAssignor.prepare(singletonMap(consumerId, singletonList(t1p))); + partitionAssignor.prepare(singletonMap(consumerId, Arrays.asList(t1p))); // the leader is responsible for picking up metadata changes and forcing a group rebalance client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE)); @@ -1013,7 +1056,7 @@ public void testUpdateMetadataDuringRebalance() { // prepare initial rebalance Map> memberSubscriptions = singletonMap(consumerId, topics); - partitionAssignor.prepare(singletonMap(consumerId, Collections.singletonList(tp1))); + partitionAssignor.prepare(singletonMap(consumerId, Arrays.asList(tp1))); client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE)); client.prepareResponse(new MockClient.RequestMatcher() { @@ -1047,13 +1090,11 @@ public boolean matches(AbstractRequest body) { @Test public void testWakeupFromAssignmentCallback() { - ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - false, true); + final List owned = Collections.emptyList(); + final List assigned = Arrays.asList(t1p); - final String topic = "topic1"; - TopicPartition partition = new TopicPartition(topic, 0); final String consumerId = "follower"; - Set topics = Collections.singleton(topic); + Set topics = Collections.singleton(topic1); MockRebalanceListener rebalanceListener = new MockRebalanceListener() { @Override public void onPartitionsAssigned(Collection partitions) { @@ -1074,11 +1115,10 @@ public void onPartitionsAssigned(Collection partitions) { coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); // prepare initial rebalance - partitionAssignor.prepare(singletonMap(consumerId, Collections.singletonList(partition))); + partitionAssignor.prepare(singletonMap(consumerId, Collections.singletonList(t1p))); client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE)); - client.prepareResponse(syncGroupResponse(Collections.singletonList(partition), Errors.NONE)); - + client.prepareResponse(syncGroupResponse(Collections.singletonList(t1p), Errors.NONE)); // The first call to poll should raise the exception from the rebalance listener try { @@ -1090,9 +1130,11 @@ public void onPartitionsAssigned(Collection partitions) { // The second call should retry the assignment callback and succeed coordinator.poll(time.timer(Long.MAX_VALUE)); + final int revokeCount = getRevoked(owned, assigned) == null ? 0 : 1; + final int addCount = getAdded(owned, assigned) == null ? 0 : 1; assertFalse(coordinator.rejoinNeededOrPending()); - assertEquals(1, rebalanceListener.revokedCount); - assertEquals(2, rebalanceListener.assignedCount); + assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(addCount * 2, rebalanceListener.assignedCount); } @Test @@ -1165,7 +1207,7 @@ private void testInternalTopicInclusion(boolean includeInternalTopics) { metadata = new ConsumerMetadata(0, Long.MAX_VALUE, includeInternalTopics, subscriptions, new LogContext(), new ClusterResourceListeners()); client = new MockClient(time, metadata); - coordinator = buildCoordinator(new Metrics(), assignors, false, true); + coordinator = buildCoordinator(new Metrics(), assignors, false, true, protocol); subscriptions.subscribe(Pattern.compile(".*"), rebalanceListener); @@ -1186,16 +1228,20 @@ private void testInternalTopicInclusion(boolean includeInternalTopics) { @Test public void testRejoinGroup() { String otherTopic = "otherTopic"; + final List owned = Collections.emptyList(); + final List assigned = Arrays.asList(t1p); subscriptions.subscribe(singleton(topic1), rebalanceListener); // join the group once - joinAsFollowerAndReceiveAssignment("consumer", coordinator, singletonList(t1p)); + joinAsFollowerAndReceiveAssignment("consumer", coordinator, assigned); - assertEquals(1, rebalanceListener.revokedCount); - assertTrue(rebalanceListener.revoked.isEmpty()); - assertEquals(1, rebalanceListener.assignedCount); - assertEquals(singleton(t1p), rebalanceListener.assigned); + int revokeCount = getRevoked(owned, assigned) == null ? 0 : 1; + int addCount = getAdded(owned, assigned) == null ? 0 : 1; + assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); + assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); // and join the group again subscriptions.subscribe(new HashSet<>(Arrays.asList(topic1, otherTopic)), rebalanceListener); @@ -1203,15 +1249,19 @@ public void testRejoinGroup() { client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE)); coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE)); - assertEquals(2, rebalanceListener.revokedCount); - assertEquals(singleton(t1p), rebalanceListener.revoked); - assertEquals(2, rebalanceListener.assignedCount); - assertEquals(singleton(t1p), rebalanceListener.assigned); + revokeCount += getRevoked(assigned, assigned) == null ? 0 : 1; + addCount += getAdded(assigned, assigned) == null ? 0 : 1; + assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(getRevoked(assigned, assigned), rebalanceListener.revoked); + assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(getAdded(assigned, assigned), rebalanceListener.assigned); } @Test public void testDisconnectInJoin() { subscriptions.subscribe(singleton(topic1), rebalanceListener); + final List owned = Collections.emptyList(); + final List assigned = Arrays.asList(t1p); client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); @@ -1220,14 +1270,17 @@ public void testDisconnectInJoin() { client.prepareResponse(joinGroupFollowerResponse(1, "consumer", "leader", Errors.NONE), true); client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); client.prepareResponse(joinGroupFollowerResponse(1, "consumer", "leader", Errors.NONE)); - client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE)); + client.prepareResponse(syncGroupResponse(assigned, Errors.NONE)); coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE)); + final int revokeCount = getRevoked(owned, assigned) == null ? 0 : 1; + final int addCount = getAdded(owned, assigned) == null ? 0 : 1; assertFalse(coordinator.rejoinNeededOrPending()); - assertEquals(singleton(t1p), subscriptions.assignedPartitions()); - assertEquals(1, rebalanceListener.revokedCount); - assertEquals(1, rebalanceListener.assignedCount); - assertEquals(singleton(t1p), rebalanceListener.assigned); + assertEquals(toSet(assigned), subscriptions.assignedPartitions()); + assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); + assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @Test(expected = ApiException.class) @@ -1294,7 +1347,7 @@ public void testAutoCommitDynamicAssignment() { final String consumerId = "consumer"; ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, true); + true, true, protocol); subscriptions.subscribe(singleton(topic1), rebalanceListener); joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p)); @@ -1310,7 +1363,7 @@ public void testAutoCommitDynamicAssignment() { public void testAutoCommitRetryBackoff() { final String consumerId = "consumer"; ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, true); + true, true, protocol); subscriptions.subscribe(singleton(topic1), rebalanceListener); joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p)); @@ -1344,7 +1397,7 @@ public void testAutoCommitRetryBackoff() { public void testAutoCommitAwaitsInterval() { final String consumerId = "consumer"; ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, true); + true, true, protocol); subscriptions.subscribe(singleton(topic1), rebalanceListener); joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p)); @@ -1383,7 +1436,7 @@ public void testAutoCommitDynamicAssignmentRebalance() { final String consumerId = "consumer"; ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, true); + true, true, protocol); subscriptions.subscribe(singleton(topic1), rebalanceListener); @@ -1409,7 +1462,7 @@ public void testAutoCommitDynamicAssignmentRebalance() { @Test public void testAutoCommitManualAssignment() { ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, true); + true, true, protocol); subscriptions.assignFromUser(singleton(t1p)); subscriptions.seek(t1p, 100); @@ -1426,7 +1479,7 @@ public void testAutoCommitManualAssignment() { @Test public void testAutoCommitManualAssignmentCoordinatorUnknown() { ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, true); + true, true, protocol); subscriptions.assignFromUser(singleton(t1p)); subscriptions.seek(t1p, 100); @@ -2010,7 +2063,7 @@ public void testHeartbeatThreadClose() throws Exception { @Test public void testAutoCommitAfterCoordinatorBackToService() { ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, true); + true, true, protocol); subscriptions.assignFromUser(Collections.singleton(t1p)); subscriptions.seek(t1p, 100L); @@ -2031,7 +2084,7 @@ private ConsumerCoordinator prepareCoordinatorForCloseTest(final boolean useGrou final boolean leaveGroup) { final String consumerId = "consumer"; ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - autoCommit, leaveGroup); + autoCommit, leaveGroup, protocol); client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); if (useGroupManagement) { @@ -2124,7 +2177,8 @@ public boolean matches(AbstractRequest body) { private ConsumerCoordinator buildCoordinator(final Metrics metrics, final List assignors, final boolean autoCommitEnabled, - final boolean leaveGroup) { + final boolean leaveGroup, + final ConsumerCoordinator.RebalanceProtocol protocol) { return new ConsumerCoordinator( new LogContext(), consumerClient, @@ -2143,7 +2197,35 @@ private ConsumerCoordinator buildCoordinator(final Metrics metrics, autoCommitIntervalMs, null, leaveGroup, - ConsumerCoordinator.RebalanceProtocol.EAGER); + protocol); + } + + private Collection getRevoked(final List owned, + final List assigned) { + switch (protocol) { + case EAGER: + return toSet(owned); + case COOPERATIVE: + final List revoked = new ArrayList<>(owned); + revoked.removeAll(assigned); + return revoked.isEmpty() ? null : toSet(revoked); + default: + throw new IllegalStateException("This should not happen"); + } + } + + private Collection getAdded(final List owned, + final List assigned) { + switch (protocol) { + case EAGER: + return toSet(assigned); + case COOPERATIVE: + final List added = new ArrayList<>(assigned); + added.removeAll(owned); + return added.isEmpty() ? null : toSet(added); + default: + throw new IllegalStateException("This should not happen"); + } } private FindCoordinatorResponse groupCoordinatorResponse(Node node, Errors error) { diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java index cb6a36cc4420c..616d89f8cc8e8 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java @@ -29,11 +29,8 @@ import java.nio.ByteBuffer; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Set; import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.CONSUMER_PROTOCOL_HEADER_SCHEMA; import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.OWNED_PARTITIONS_KEY_NAME; @@ -43,6 +40,7 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.USER_DATA_KEY_NAME; import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.VERSION_KEY_NAME; import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE; +import static org.apache.kafka.test.TestUtils.toSet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -200,8 +198,4 @@ public void deserializeFutureAssignmentVersion() { assertEquals(toSet(Collections.singletonList(tp1)), toSet(assignment.partitions())); assertEquals(Errors.NEED_REJOIN, assignment.error()); } - - private static Set toSet(Collection collection) { - return new HashSet<>(collection); - } } diff --git a/clients/src/test/java/org/apache/kafka/test/TestUtils.java b/clients/src/test/java/org/apache/kafka/test/TestUtils.java index f7a37baf4f360..173382fd696b5 100644 --- a/clients/src/test/java/org/apache/kafka/test/TestUtils.java +++ b/clients/src/test/java/org/apache/kafka/test/TestUtils.java @@ -39,6 +39,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Base64; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -420,6 +421,10 @@ public static List toList(Iterable iterable) { return list; } + public static Set toSet(Collection collection) { + return new HashSet<>(collection); + } + public static ByteBuffer toBuffer(Struct struct) { ByteBuffer buffer = ByteBuffer.allocate(struct.sizeOf()); struct.writeTo(buffer); From d1064c404af9005bc703dc784b2f69a405d3d82c Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Fri, 5 Apr 2019 16:09:06 -0700 Subject: [PATCH 11/59] add version varible to subscription and assignment --- .../consumer/internals/ConsumerProtocol.java | 40 ++++++++++++++--- .../consumer/internals/PartitionAssignor.java | 44 +++++++++++++++++-- .../internals/ConsumerProtocolTest.java | 18 ++++---- 3 files changed, 82 insertions(+), 20 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java index ed2df86edb62e..97494a1942183 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java @@ -164,7 +164,17 @@ public static ByteBuffer serializeSubscriptionV1(PartitionAssignor.Subscription } public static ByteBuffer serializeSubscription(PartitionAssignor.Subscription subscription) { - return serializeSubscriptionV1(subscription); + switch (subscription.version()) { + case CONSUMER_PROTOCOL_V0: + return serializeSubscriptionV0(subscription); + + case CONSUMER_PROTOCOL_V1: + return serializeSubscriptionV1(subscription); + + default: + // for any versions higher than known, try to serialize it as V1 + return serializeSubscriptionV1(subscription); + } } public static PartitionAssignor.Subscription deserializeSubscriptionV0(ByteBuffer buffer) { @@ -174,7 +184,7 @@ public static PartitionAssignor.Subscription deserializeSubscriptionV0(ByteBuffe for (Object topicObj : struct.getArray(TOPICS_KEY_NAME)) topics.add((String) topicObj); - return new PartitionAssignor.Subscription(topics, userData); + return new PartitionAssignor.Subscription(CONSUMER_PROTOCOL_V0, topics, userData); } public static PartitionAssignor.Subscription deserializeSubscriptionV1(ByteBuffer buffer) { @@ -194,7 +204,7 @@ public static PartitionAssignor.Subscription deserializeSubscriptionV1(ByteBuffe } } - return new PartitionAssignor.Subscription(topics, userData, ownedPartitions); + return new PartitionAssignor.Subscription(CONSUMER_PROTOCOL_V1, topics, userData, ownedPartitions); } public static PartitionAssignor.Subscription deserializeSubscription(ByteBuffer buffer) { @@ -208,6 +218,9 @@ public static PartitionAssignor.Subscription deserializeSubscription(ByteBuffer case CONSUMER_PROTOCOL_V0: return deserializeSubscriptionV0(buffer); + case CONSUMER_PROTOCOL_V1: + return deserializeSubscriptionV1(buffer); + // assume all higher versions can be parsed as V1 default: return deserializeSubscriptionV1(buffer); @@ -256,7 +269,17 @@ public static ByteBuffer serializeAssignmentV1(PartitionAssignor.Assignment assi } public static ByteBuffer serializeAssignment(PartitionAssignor.Assignment assignment) { - return serializeAssignmentV1(assignment); + switch (assignment.version()) { + case CONSUMER_PROTOCOL_V0: + return serializeAssignmentV0(assignment); + + case CONSUMER_PROTOCOL_V1: + return serializeAssignmentV1(assignment); + + default: + // for any versions higher than known, try to serialize it as V1 + return serializeAssignmentV1(assignment); + } } public static PartitionAssignor.Assignment deserializeAssignmentV0(ByteBuffer buffer) { @@ -271,7 +294,7 @@ public static PartitionAssignor.Assignment deserializeAssignmentV0(ByteBuffer bu partitions.add(new TopicPartition(topic, partition)); } } - return new PartitionAssignor.Assignment(partitions, userData); + return new PartitionAssignor.Assignment(CONSUMER_PROTOCOL_V0, partitions, userData); } public static PartitionAssignor.Assignment deserializeAssignmentV1(ByteBuffer buffer) { @@ -289,7 +312,7 @@ public static PartitionAssignor.Assignment deserializeAssignmentV1(ByteBuffer bu Errors error = Errors.fromCode(struct.get(ERROR_CODE)); - return new PartitionAssignor.Assignment(partitions, userData, error); + return new PartitionAssignor.Assignment(CONSUMER_PROTOCOL_V1, partitions, userData, error); } public static PartitionAssignor.Assignment deserializeAssignment(ByteBuffer buffer) { @@ -303,8 +326,11 @@ public static PartitionAssignor.Assignment deserializeAssignment(ByteBuffer buff case CONSUMER_PROTOCOL_V0: return deserializeAssignmentV0(buffer); - // assume all higher versions can be parsed as V1 + case CONSUMER_PROTOCOL_V1: + return deserializeAssignmentV1(buffer); + default: + // assume all higher versions can be parsed as V1 return deserializeAssignmentV1(buffer); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java index c1d35395d6db7..a0c37a3736692 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.Cluster; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.protocol.types.SchemaException; import java.nio.ByteBuffer; import java.util.Collections; @@ -25,6 +26,9 @@ import java.util.Map; import java.util.Set; +import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.CONSUMER_PROTOCOL_V0; +import static org.apache.kafka.clients.consumer.internals.ConsumerProtocol.CONSUMER_PROTOCOL_V1; + /** * This interface is used to define custom partition assignment for use in * {@link org.apache.kafka.clients.consumer.KafkaConsumer}. Members of the consumer group subscribe @@ -74,24 +78,40 @@ public interface PartitionAssignor { String name(); class Subscription { + private final Short version; private final List topics; private final ByteBuffer userData; private final List ownedPartitions; - public Subscription(List topics, ByteBuffer userData, List ownedPartitions) { + public Subscription(Short version, List topics, ByteBuffer userData, List ownedPartitions) { + this.version = version; this.topics = topics; this.userData = userData; this.ownedPartitions = ownedPartitions; + + if (version < CONSUMER_PROTOCOL_V0) + throw new SchemaException("Unsupported subscription version: " + version); + + if (version < CONSUMER_PROTOCOL_V1 && !ownedPartitions.isEmpty()) + throw new IllegalArgumentException("Subscription version smaller than 1 should not have owned partitions"); + } + + public Subscription(Short version, List topics, ByteBuffer userData) { + this(version, topics, userData, Collections.emptyList()); } public Subscription(List topics, ByteBuffer userData) { - this(topics, userData, Collections.emptyList()); + this(CONSUMER_PROTOCOL_V1, topics, userData); } public Subscription(List topics) { this(topics, ByteBuffer.wrap(new byte[0])); } + public Short version() { + return version; + } + public List topics() { return topics; } @@ -113,24 +133,40 @@ public String toString() { } class Assignment { + private final Short version; private final List partitions; private final ByteBuffer userData; private final ConsumerProtocol.Errors error; - public Assignment(List partitions, ByteBuffer userData, ConsumerProtocol.Errors error) { + public Assignment(Short version, List partitions, ByteBuffer userData, ConsumerProtocol.Errors error) { + this.version = version; this.partitions = partitions; this.userData = userData; this.error = error; + + if (version < CONSUMER_PROTOCOL_V0) + throw new SchemaException("Unsupported subscription version: " + version); + + if (version < CONSUMER_PROTOCOL_V1 && error != ConsumerProtocol.Errors.NONE) + throw new IllegalArgumentException("Assignment version smaller than 1 should not have error code."); + } + + public Assignment(Short version, List partitions, ByteBuffer userData) { + this(version, partitions, userData, ConsumerProtocol.Errors.NONE); } public Assignment(List partitions, ByteBuffer userData) { - this(partitions, userData, ConsumerProtocol.Errors.NONE); + this(CONSUMER_PROTOCOL_V1, partitions, userData); } public Assignment(List partitions) { this(partitions, ByteBuffer.wrap(new byte[0])); } + public Short version() { + return version; + } + public List partitions() { return partitions; } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java index cb6a36cc4420c..8a8ba0a82e199 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java @@ -72,17 +72,17 @@ public void serializeDeserializeNullSubscriptionUserData() { @Test public void deserializeOldSubscriptionVersion() { - Subscription subscription = new Subscription(Arrays.asList("foo", "bar")); - ByteBuffer buffer = ConsumerProtocol.serializeSubscriptionV0(subscription); + Subscription subscription = new Subscription((short) 0, Arrays.asList("foo", "bar"), null); + ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription); Subscription parsedSubscription = ConsumerProtocol.deserializeSubscription(buffer); assertEquals(parsedSubscription.topics(), parsedSubscription.topics()); - assertEquals(0, parsedSubscription.userData().limit()); + assertNull(parsedSubscription.userData()); assertTrue(parsedSubscription.ownedPartitions().isEmpty()); } @Test - public void deserializeNewSubscriptionVersion() { - Subscription subscription = new Subscription(Arrays.asList("foo", "bar"), null, Collections.singletonList(tp2)); + public void deserializeNewSubscriptionWithOldVersion() { + Subscription subscription = new Subscription((short) 1, Arrays.asList("foo", "bar"), null, Collections.singletonList(tp2)); ByteBuffer buffer = ConsumerProtocol.serializeSubscription(subscription); // ignore the version assuming it is the old byte code, as it will blindly deserialize as V0 Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer); @@ -147,17 +147,17 @@ public void deserializeNullAssignmentUserData() { @Test public void deserializeOldAssignmentVersion() { List partitions = Arrays.asList(tp1, tp2); - ByteBuffer buffer = ConsumerProtocol.serializeAssignmentV0(new Assignment(partitions)); + ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment((short) 0, partitions, null)); Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer); assertEquals(toSet(partitions), toSet(parsedAssignment.partitions())); - assertEquals(0, parsedAssignment.userData().limit()); + assertNull(parsedAssignment.userData()); assertEquals(Errors.NONE, parsedAssignment.error()); } @Test - public void deserializeNewAssignmentVersion() { + public void deserializeNewAssignmentWithOldVersion() { List partitions = Collections.singletonList(tp1); - ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment(partitions, null, Errors.NEED_REJOIN)); + ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment((short) 1, partitions, null, Errors.NEED_REJOIN)); // ignore the version assuming it is the old byte code, as it will blindly deserialize as 0 Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer); header.getShort(VERSION_KEY_NAME); From 2b2d8b1c765be29c9e28c9f6cd06248ba30421ef Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Sat, 6 Apr 2019 21:09:02 -0700 Subject: [PATCH 12/59] more minor fixes --- .../internals/ConsumerCoordinatorTest.java | 44 +++++++++++++------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index 269f8022205d5..deb6b9fde14d8 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -571,6 +571,9 @@ public boolean matches(AbstractRequest body) { @Test public void testMetadataRefreshDuringRebalance() { final String consumerId = "leader"; + final List owned = Collections.emptyList(); + final List oldAssigned = Arrays.asList(t1p); + subscriptions.subscribe(Pattern.compile(".*"), rebalanceListener); client.updateMetadata(TestUtils.metadataUpdateWith(1, singletonMap(topic1, 1))); @@ -582,11 +585,10 @@ public void testMetadataRefreshDuringRebalance() { coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); Map> initialSubscription = singletonMap(consumerId, singletonList(topic1)); - partitionAssignor.prepare(singletonMap(consumerId, Arrays.asList(t1p))); + partitionAssignor.prepare(singletonMap(consumerId, oldAssigned)); // the metadata will be updated in flight with a new topic added final List updatedSubscription = Arrays.asList(topic1, topic2); - final Set updatedSubscriptionSet = new HashSet<>(updatedSubscription); client.prepareResponse(joinGroupLeaderResponse(1, consumerId, initialSubscription, Errors.NONE)); client.prepareResponse(new MockClient.RequestMatcher() { @@ -598,14 +600,25 @@ public boolean matches(AbstractRequest body) { client.updateMetadata(TestUtils.metadataUpdateWith(1, updatedPartitions)); return true; } - }, syncGroupResponse(singletonList(t1p), Errors.NONE)); + }, syncGroupResponse(oldAssigned, Errors.NONE)); coordinator.poll(time.timer(Long.MAX_VALUE)); - List newAssignment = Arrays.asList(t1p, t2p); - Set newAssignmentSet = new HashSet<>(newAssignment); + int revokeCount = getRevoked(owned, oldAssigned) == null ? 0 : 1; + int addCount = getAdded(owned, oldAssigned) == null ? 0 : 1; + + // rejoin will only be set in the next poll call + assertFalse(coordinator.rejoinNeededOrPending()); + assertEquals(singleton(topic1), subscriptions.subscription()); + assertEquals(toSet(oldAssigned), subscriptions.assignedPartitions()); + assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(getRevoked(owned, oldAssigned), rebalanceListener.revoked); + assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(getAdded(owned, oldAssigned), rebalanceListener.assigned); + + List newAssigned = Arrays.asList(t1p, t2p); Map> updatedSubscriptions = singletonMap(consumerId, Arrays.asList(topic1, topic2)); - partitionAssignor.prepare(singletonMap(consumerId, newAssignment)); + partitionAssignor.prepare(singletonMap(consumerId, newAssigned)); // we expect to see a second rebalance with the new-found topics client.prepareResponse(new MockClient.RequestMatcher() { @@ -620,20 +633,23 @@ public boolean matches(AbstractRequest body) { ByteBuffer metadata = ByteBuffer.wrap(protocolMetadata.metadata()); PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(metadata); metadata.rewind(); - return subscription.topics().containsAll(updatedSubscriptionSet); + return subscription.topics().containsAll(updatedSubscription); } }, joinGroupLeaderResponse(2, consumerId, updatedSubscriptions, Errors.NONE)); - client.prepareResponse(syncGroupResponse(newAssignment, Errors.NONE)); + client.prepareResponse(syncGroupResponse(newAssigned, Errors.NONE)); coordinator.poll(time.timer(Long.MAX_VALUE)); + revokeCount += getRevoked(oldAssigned, newAssigned) == null ? 0 : 1; + addCount += getAdded(oldAssigned, newAssigned) == null ? 0 : 1; + assertFalse(coordinator.rejoinNeededOrPending()); - assertEquals(updatedSubscriptionSet, subscriptions.subscription()); - assertEquals(newAssignmentSet, subscriptions.assignedPartitions()); - assertEquals(2, rebalanceListener.revokedCount); - assertEquals(singleton(t1p), rebalanceListener.revoked); - assertEquals(2, rebalanceListener.assignedCount); - assertEquals(newAssignmentSet, rebalanceListener.assigned); + assertEquals(toSet(updatedSubscription), subscriptions.subscription()); + assertEquals(toSet(newAssigned), subscriptions.assignedPartitions()); + assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(getRevoked(oldAssigned, newAssigned), rebalanceListener.revoked); + assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(getAdded(oldAssigned, newAssigned), rebalanceListener.assigned); } @Test From c22266be9d44d4374578f9aa90ebc6149ff40700 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Sun, 7 Apr 2019 13:32:45 -0700 Subject: [PATCH 13/59] update toString --- .../clients/consumer/internals/PartitionAssignor.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java index a0c37a3736692..fd0362f2af1f7 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java @@ -127,7 +127,9 @@ public ByteBuffer userData() { @Override public String toString() { return "Subscription(" + - "topics=" + topics + + "version=" + version + + ", topics=" + topics + + ", ownedPartitions=" + ownedPartitions + ')'; } } @@ -182,7 +184,9 @@ public ByteBuffer userData() { @Override public String toString() { return "Assignment(" + - "partitions=" + partitions + + "version=" + version + + ", partitions=" + partitions + + ", error=" + error + ')'; } } From 16f75abe65b86a32a55c947dddc0836bb2f60c47 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 8 Apr 2019 18:53:56 -0700 Subject: [PATCH 14/59] minor fix --- .../internals/ConsumerCoordinator.java | 14 +++--- .../org/apache/kafka/clients/MockClient.java | 2 +- .../internals/ConsumerCoordinatorTest.java | 46 +++++++++++++------ 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 289b0b6ec7a53..eb0465441ecf5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -354,14 +354,12 @@ private void assignAndRevoke(final ConsumerRebalanceListener listener, log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); } - if (!revokedPartitions.isEmpty()) { - try { - listener.onPartitionsRevoked(revokedPartitions); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e); - } + try { + listener.onPartitionsRevoked(revokedPartitions); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/MockClient.java b/clients/src/test/java/org/apache/kafka/clients/MockClient.java index c80582d461451..4f754f7a1a08b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/MockClient.java +++ b/clients/src/test/java/org/apache/kafka/clients/MockClient.java @@ -224,7 +224,7 @@ public void send(ClientRequest request, long now) { builder.latestAllowedVersion()); AbstractRequest abstractRequest = request.requestBuilder().build(version); if (!futureResp.requestMatcher.matches(abstractRequest)) - throw new IllegalStateException("Request matcher did not match next-in-line request " + abstractRequest); + throw new IllegalStateException("Request matcher did not match next-in-line request " + abstractRequest + " with prepared response " + futureResp.responseBody); UnsupportedVersionException unsupportedVersionException = null; if (futureResp.isUnsupportedRequest) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index deb6b9fde14d8..821bf2fc91524 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -441,19 +441,24 @@ public boolean matches(AbstractRequest body) { @Test public void testOutdatedCoordinatorAssignment() { final String consumerId = "outdated_assignment"; + final List owned = Collections.emptyList(); + final List oldSubscription = singletonList(topic2); + final List oldAssignment = Arrays.asList(t2p); + final List newSubscription = singletonList(topic1); + final List newAssignment = Arrays.asList(t1p); - subscriptions.subscribe(singleton(topic2), rebalanceListener); + subscriptions.subscribe(toSet(oldSubscription), rebalanceListener); client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); // Test coordinator returning unsubscribed partitions - partitionAssignor.prepare(singletonMap(consumerId, Arrays.asList(t1p))); + partitionAssignor.prepare(singletonMap(consumerId, newAssignment)); // First incorrect assignment for subscription client.prepareResponse( joinGroupLeaderResponse( - 1, consumerId, singletonMap(consumerId, singletonList(topic2)), Errors.NONE)); + 1, consumerId, singletonMap(consumerId, oldSubscription), Errors.NONE)); client.prepareResponse(new MockClient.RequestMatcher() { @Override public boolean matches(AbstractRequest body) { @@ -462,12 +467,12 @@ public boolean matches(AbstractRequest body) { sync.generationId() == 1 && sync.groupAssignment().containsKey(consumerId); } - }, syncGroupResponse(Arrays.asList(t2p), Errors.NONE)); + }, syncGroupResponse(oldAssignment, Errors.NONE)); // Second correct assignment for subscription client.prepareResponse( joinGroupLeaderResponse( - 1, consumerId, singletonMap(consumerId, singletonList(topic1)), Errors.NONE)); + 1, consumerId, singletonMap(consumerId, newSubscription), Errors.NONE)); client.prepareResponse(new MockClient.RequestMatcher() { @Override public boolean matches(AbstractRequest body) { @@ -476,24 +481,35 @@ public boolean matches(AbstractRequest body) { sync.generationId() == 1 && sync.groupAssignment().containsKey(consumerId); } - }, syncGroupResponse(singletonList(t1p), Errors.NONE)); + }, syncGroupResponse(newAssignment, Errors.NONE)); // Poll once so that the join group future gets created and complete coordinator.poll(time.timer(0)); // Before the sync group response gets completed change the subscription - subscriptions.subscribe(singleton(topic1), rebalanceListener); + subscriptions.subscribe(toSet(newSubscription), rebalanceListener); coordinator.poll(time.timer(0)); coordinator.poll(time.timer(Long.MAX_VALUE)); + final Collection revoked = getRevoked(owned, newAssignment); + final Collection assigned = getAdded(owned, newAssignment); + + int revokeCount = revoked == null ? 0 : 1; + final int addCount = assigned == null ? 0 : 1; + + // with eager protocol we will call revoke on the old assignment as well + if (protocol == ConsumerCoordinator.RebalanceProtocol.EAGER) { + revokeCount += getRevoked(owned, oldAssignment) == null ? 0 : 1; + } + assertFalse(coordinator.rejoinNeededOrPending()); - assertEquals(singleton(t1p), subscriptions.assignedPartitions()); - assertEquals(singleton(topic1), subscriptions.groupSubscription()); - assertEquals(2, rebalanceListener.revokedCount); - assertEquals(Collections.emptySet(), rebalanceListener.revoked); - assertEquals(1, rebalanceListener.assignedCount); - assertEquals(singleton(t1p), rebalanceListener.assigned); + assertEquals(toSet(newAssignment), subscriptions.assignedPartitions()); + assertEquals(toSet(newSubscription), subscriptions.groupSubscription()); + assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(revoked, rebalanceListener.revoked); + assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(assigned, rebalanceListener.assigned); } @Test @@ -1260,9 +1276,11 @@ public void testRejoinGroup() { assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); // and join the group again + rebalanceListener.revoked = null; + rebalanceListener.assigned = null; subscriptions.subscribe(new HashSet<>(Arrays.asList(topic1, otherTopic)), rebalanceListener); client.prepareResponse(joinGroupFollowerResponse(2, "consumer", "leader", Errors.NONE)); - client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE)); + client.prepareResponse(syncGroupResponse(assigned, Errors.NONE)); coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE)); revokeCount += getRevoked(assigned, assigned) == null ? 0 : 1; From e049e5f90a896300cfc5fe9d8b20e8957fbefa1c Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 8 Apr 2019 19:04:23 -0700 Subject: [PATCH 15/59] found another lurking bug on my code --- .../internals/ConsumerCoordinatorTest.java | 81 +++++++------------ 1 file changed, 27 insertions(+), 54 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index 821bf2fc91524..8653a626d5166 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -427,14 +427,12 @@ public boolean matches(AbstractRequest body) { }, syncGroupResponse(assigned, Errors.NONE)); coordinator.poll(time.timer(Long.MAX_VALUE)); - final int revokeCount = getRevoked(owned, assigned) == null ? 0 : 1; - final int addCount = getAdded(owned, assigned) == null ? 0 : 1; assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(toSet(assigned), subscriptions.assignedPartitions()); assertEquals(subscription, subscriptions.groupSubscription()); - assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(1, rebalanceListener.revokedCount); assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); - assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @@ -495,12 +493,12 @@ public boolean matches(AbstractRequest body) { final Collection revoked = getRevoked(owned, newAssignment); final Collection assigned = getAdded(owned, newAssignment); - int revokeCount = revoked == null ? 0 : 1; - final int addCount = assigned == null ? 0 : 1; + int revokeCount = 1; + final int addCount = 1; // with eager protocol we will call revoke on the old assignment as well if (protocol == ConsumerCoordinator.RebalanceProtocol.EAGER) { - revokeCount += getRevoked(owned, oldAssignment) == null ? 0 : 1; + revokeCount += 1; } assertFalse(coordinator.rejoinNeededOrPending()); @@ -572,15 +570,13 @@ public boolean matches(AbstractRequest body) { coordinator.poll(time.timer(Long.MAX_VALUE)); - final int revokeCount = getRevoked(owned, assigned) == null ? 0 : 1; - final int addCount = getAdded(owned, assigned) == null ? 0 : 1; assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(2, subscriptions.numAssignedPartitions()); assertEquals(2, subscriptions.groupSubscription().size()); assertEquals(2, subscriptions.subscription().size()); - assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(1, rebalanceListener.revokedCount); assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); - assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @@ -619,16 +615,13 @@ public boolean matches(AbstractRequest body) { }, syncGroupResponse(oldAssigned, Errors.NONE)); coordinator.poll(time.timer(Long.MAX_VALUE)); - int revokeCount = getRevoked(owned, oldAssigned) == null ? 0 : 1; - int addCount = getAdded(owned, oldAssigned) == null ? 0 : 1; - // rejoin will only be set in the next poll call assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(singleton(topic1), subscriptions.subscription()); assertEquals(toSet(oldAssigned), subscriptions.assignedPartitions()); - assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(1, rebalanceListener.revokedCount); assertEquals(getRevoked(owned, oldAssigned), rebalanceListener.revoked); - assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, oldAssigned), rebalanceListener.assigned); List newAssigned = Arrays.asList(t1p, t2p); @@ -656,15 +649,12 @@ public boolean matches(AbstractRequest body) { coordinator.poll(time.timer(Long.MAX_VALUE)); - revokeCount += getRevoked(oldAssigned, newAssigned) == null ? 0 : 1; - addCount += getAdded(oldAssigned, newAssigned) == null ? 0 : 1; - assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(toSet(updatedSubscription), subscriptions.subscription()); assertEquals(toSet(newAssigned), subscriptions.assignedPartitions()); - assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(2, rebalanceListener.revokedCount); assertEquals(getRevoked(oldAssigned, newAssigned), rebalanceListener.revoked); - assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(2, rebalanceListener.assignedCount); assertEquals(getAdded(oldAssigned, newAssigned), rebalanceListener.assigned); } @@ -743,13 +733,11 @@ public void testWakeupDuringJoin() { client.prepareResponse(syncGroupResponse(assigned, Errors.NONE)); coordinator.poll(time.timer(Long.MAX_VALUE)); - final int revokeCount = getRevoked(owned, assigned) == null ? 0 : 1; - final int addCount = getAdded(owned, assigned) == null ? 0 : 1; assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(toSet(assigned), subscriptions.assignedPartitions()); - assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(1, rebalanceListener.revokedCount); assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); - assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @@ -779,14 +767,12 @@ public boolean matches(AbstractRequest body) { coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE)); - final int revokeCount = getRevoked(owned, assigned) == null ? 0 : 1; - final int addCount = getAdded(owned, assigned) == null ? 0 : 1; assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(toSet(assigned), subscriptions.assignedPartitions()); assertEquals(subscription, subscriptions.groupSubscription()); - assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(1, rebalanceListener.revokedCount); assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); - assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @@ -851,14 +837,12 @@ public boolean matches(AbstractRequest body) { coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE)); - final int revokeCount = getRevoked(owned, assigned) == null ? 0 : 1; - final int addCount = getAdded(owned, assigned) == null ? 0 : 1; assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(assigned.size(), subscriptions.numAssignedPartitions()); assertEquals(subscription, subscriptions.subscription()); - assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(1, rebalanceListener.revokedCount); assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); - assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @@ -1122,9 +1106,6 @@ public boolean matches(AbstractRequest body) { @Test public void testWakeupFromAssignmentCallback() { - final List owned = Collections.emptyList(); - final List assigned = Arrays.asList(t1p); - final String consumerId = "follower"; Set topics = Collections.singleton(topic1); MockRebalanceListener rebalanceListener = new MockRebalanceListener() { @@ -1162,11 +1143,9 @@ public void onPartitionsAssigned(Collection partitions) { // The second call should retry the assignment callback and succeed coordinator.poll(time.timer(Long.MAX_VALUE)); - final int revokeCount = getRevoked(owned, assigned) == null ? 0 : 1; - final int addCount = getAdded(owned, assigned) == null ? 0 : 1; assertFalse(coordinator.rejoinNeededOrPending()); - assertEquals(revokeCount, rebalanceListener.revokedCount); - assertEquals(addCount * 2, rebalanceListener.assignedCount); + assertEquals(1, rebalanceListener.revokedCount); + assertEquals(2, rebalanceListener.assignedCount); } @Test @@ -1268,11 +1247,9 @@ public void testRejoinGroup() { // join the group once joinAsFollowerAndReceiveAssignment("consumer", coordinator, assigned); - int revokeCount = getRevoked(owned, assigned) == null ? 0 : 1; - int addCount = getAdded(owned, assigned) == null ? 0 : 1; - assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(1, rebalanceListener.revokedCount); assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); - assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); // and join the group again @@ -1283,11 +1260,9 @@ public void testRejoinGroup() { client.prepareResponse(syncGroupResponse(assigned, Errors.NONE)); coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE)); - revokeCount += getRevoked(assigned, assigned) == null ? 0 : 1; - addCount += getAdded(assigned, assigned) == null ? 0 : 1; - assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(2, rebalanceListener.revokedCount); assertEquals(getRevoked(assigned, assigned), rebalanceListener.revoked); - assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(2, rebalanceListener.assignedCount); assertEquals(getAdded(assigned, assigned), rebalanceListener.assigned); } @@ -1307,13 +1282,11 @@ public void testDisconnectInJoin() { client.prepareResponse(syncGroupResponse(assigned, Errors.NONE)); coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE)); - final int revokeCount = getRevoked(owned, assigned) == null ? 0 : 1; - final int addCount = getAdded(owned, assigned) == null ? 0 : 1; assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(toSet(assigned), subscriptions.assignedPartitions()); - assertEquals(revokeCount, rebalanceListener.revokedCount); + assertEquals(1, rebalanceListener.revokedCount); assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); - assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @@ -2242,7 +2215,7 @@ private Collection getRevoked(final List owned, case COOPERATIVE: final List revoked = new ArrayList<>(owned); revoked.removeAll(assigned); - return revoked.isEmpty() ? null : toSet(revoked); + return toSet(revoked); default: throw new IllegalStateException("This should not happen"); } @@ -2256,7 +2229,7 @@ private Collection getAdded(final List owned, case COOPERATIVE: final List added = new ArrayList<>(assigned); added.removeAll(owned); - return added.isEmpty() ? null : toSet(added); + return toSet(added); default: throw new IllegalStateException("This should not happen"); } From ff1b9a67bf0ef8a6bbca0492abd92badd11ddea7 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Wed, 24 Apr 2019 18:50:02 -0700 Subject: [PATCH 16/59] new API --- .../consumer/internals/PartitionAssignor.java | 52 ++++++++++++++++--- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java index 070a515a4b576..b6de935409aba 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java @@ -78,6 +78,13 @@ default void onAssignment(Assignment assignment, int generation) { onAssignment(assignment); } + /** + * Indicate which rebalance protocol this assignor can would work with; + * By default it should always work with {@link RebalanceProtocol#EAGER}. + */ + default List supportedProtocols() { + return Collections.singletonList(RebalanceProtocol.EAGER); + } /** * Unique name for this assignor (e.g. "range" or "roundrobin" or "sticky") @@ -85,13 +92,38 @@ default void onAssignment(Assignment assignment, int generation) { */ String name(); + enum RebalanceProtocol { + EAGER((byte) 0), COOPERATIVE((byte) 1); + + private final byte id; + + RebalanceProtocol(byte id) { + this.id = id; + } + + public byte id() { + return id; + } + + public static RebalanceProtocol forId(byte id) { + switch (id) { + case 0: + return EAGER; + case 1: + return COOPERATIVE; + default: + throw new IllegalArgumentException("Unknown rebalance protocol id: " + id); + } + } + } + class Subscription { private final Short version; private final List topics; private final ByteBuffer userData; private final List ownedPartitions; - public Subscription(Short version, List topics, ByteBuffer userData, List ownedPartitions) { + Subscription(Short version, List topics, ByteBuffer userData, List ownedPartitions) { this.version = version; this.topics = topics; this.userData = userData; @@ -104,10 +136,14 @@ public Subscription(Short version, List topics, ByteBuffer userData, Lis throw new IllegalArgumentException("Subscription version smaller than 1 should not have owned partitions"); } - public Subscription(Short version, List topics, ByteBuffer userData) { + Subscription(Short version, List topics, ByteBuffer userData) { this(version, topics, userData, Collections.emptyList()); } + public Subscription(List topics, ByteBuffer userData, List ownedPartitions) { + this(CONSUMER_PROTOCOL_V1, topics, userData, ownedPartitions); + } + public Subscription(List topics, ByteBuffer userData) { this(CONSUMER_PROTOCOL_V1, topics, userData); } @@ -116,7 +152,7 @@ public Subscription(List topics) { this(topics, ByteBuffer.wrap(new byte[0])); } - public Short version() { + Short version() { return version; } @@ -148,7 +184,7 @@ class Assignment { private final ByteBuffer userData; private final ConsumerProtocol.Errors error; - public Assignment(Short version, List partitions, ByteBuffer userData, ConsumerProtocol.Errors error) { + Assignment(Short version, List partitions, ByteBuffer userData, ConsumerProtocol.Errors error) { this.version = version; this.partitions = partitions; this.userData = userData; @@ -161,10 +197,14 @@ public Assignment(Short version, List partitions, ByteBuffer use throw new IllegalArgumentException("Assignment version smaller than 1 should not have error code."); } - public Assignment(Short version, List partitions, ByteBuffer userData) { + Assignment(Short version, List partitions, ByteBuffer userData) { this(version, partitions, userData, ConsumerProtocol.Errors.NONE); } + public Assignment(List partitions, ByteBuffer userData, ConsumerProtocol.Errors error) { + this(CONSUMER_PROTOCOL_V1, partitions, userData, error); + } + public Assignment(List partitions, ByteBuffer userData) { this(CONSUMER_PROTOCOL_V1, partitions, userData); } @@ -173,7 +213,7 @@ public Assignment(List partitions) { this(partitions, ByteBuffer.wrap(new byte[0])); } - public Short version() { + Short version() { return version; } From 908fda4dc6c630c67bbaa4ac45a924abebaa862d Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Thu, 25 Apr 2019 14:45:08 -0700 Subject: [PATCH 17/59] add version API --- .../clients/consumer/internals/PartitionAssignor.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java index b6de935409aba..5c76fd66dfef0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java @@ -86,6 +86,14 @@ default List supportedProtocols() { return Collections.singletonList(RebalanceProtocol.EAGER); } + /** + * Return the version of the assignor which indicate how the user metadata encodings + * and the assignment algorithm gets evolved. + */ + default short version() { + return (short) 0; + } + /** * Unique name for this assignor (e.g. "range" or "roundrobin" or "sticky") * @return non-null unique name From 2e9a2914261d3ebc61f32869e50cd59880c468fc Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Wed, 8 May 2019 18:46:02 -0700 Subject: [PATCH 18/59] github comments --- .../clients/consumer/internals/ConsumerProtocol.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java index 8c43dfde66cf7..b4ad4514eb60b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java @@ -199,8 +199,7 @@ public static PartitionAssignor.Subscription deserializeSubscriptionV1(ByteBuffe Struct assignment = (Struct) structObj; String topic = assignment.getString(TOPIC_KEY_NAME); for (Object partitionObj : assignment.getArray(PARTITIONS_KEY_NAME)) { - Integer partition = (Integer) partitionObj; - ownedPartitions.add(new TopicPartition(topic, partition)); + ownedPartitions.add(new TopicPartition(topic, (Integer) partitionObj)); } } @@ -290,8 +289,7 @@ public static PartitionAssignor.Assignment deserializeAssignmentV0(ByteBuffer bu Struct assignment = (Struct) structObj; String topic = assignment.getString(TOPIC_KEY_NAME); for (Object partitionObj : assignment.getArray(PARTITIONS_KEY_NAME)) { - Integer partition = (Integer) partitionObj; - partitions.add(new TopicPartition(topic, partition)); + partitions.add(new TopicPartition(topic, (Integer) partitionObj)); } } return new PartitionAssignor.Assignment(CONSUMER_PROTOCOL_V0, partitions, userData); @@ -305,8 +303,7 @@ public static PartitionAssignor.Assignment deserializeAssignmentV1(ByteBuffer bu Struct assignment = (Struct) structObj; String topic = assignment.getString(TOPIC_KEY_NAME); for (Object partitionObj : assignment.getArray(PARTITIONS_KEY_NAME)) { - Integer partition = (Integer) partitionObj; - partitions.add(new TopicPartition(topic, partition)); + partitions.add(new TopicPartition(topic, (Integer) partitionObj)); } } From 16c13ad77a46edbb0e2fde27ee35582a4474b356 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 20 May 2019 17:11:09 -0700 Subject: [PATCH 19/59] throw from user rebalance callback --- .../clients/consumer/internals/ConsumerCoordinator.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 55a157fbaa9e8..8f2708430deb3 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -303,7 +303,7 @@ protected void onJoinComplete(int generation, switch (protocol) { case EAGER: if (!ownedPartitions.isEmpty()) { - log.warn("Some partitions are not revoked with EAGER rebalance protocol, " + + log.warn("Some partitions are not revoked with " + protocol + " protocol, " + "it is likely that the previous rebalance did not complete due to some errors"); } @@ -314,6 +314,7 @@ protected void onJoinComplete(int generation, throw e; } catch (Exception e) { log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); + throw e; } break; @@ -596,6 +597,7 @@ protected void onJoinPrepare(int generation, String memberId) { throw e; } catch (Exception e) { log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e); + throw e; } // also clear the assigned partitions since all have been revoked @@ -604,6 +606,8 @@ protected void onJoinPrepare(int generation, String memberId) { break; case COOPERATIVE: + log.info("Not revoking any previously assigned partitions before joining group"); + break; } From ec80f42c37e5a04f2e6ce7ea81bc5663d88af788 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 20 May 2019 18:57:41 -0700 Subject: [PATCH 20/59] fix unit tests --- .../internals/ConsumerCoordinator.java | 8 +++----- .../internals/ConsumerCoordinatorTest.java | 18 ++++++++++++------ .../internals/MockPartitionAssignor.java | 12 ++++++++++++ 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 8f2708430deb3..af425907acd8c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -314,7 +314,6 @@ protected void onJoinComplete(int generation, throw e; } catch (Exception e) { log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); - throw e; } break; @@ -589,7 +588,10 @@ protected void onJoinPrepare(int generation, String memberId) { switch (protocol) { case EAGER: + // clear the assigned partitions since all have been revoked Set revoked = new HashSet<>(subscriptions.assignedPartitions()); + subscriptions.assignFromSubscribed(Collections.emptySet()); + log.info("Revoking previously assigned partitions {}", revoked); try { listener.onPartitionsRevoked(revoked); @@ -597,12 +599,8 @@ protected void onJoinPrepare(int generation, String memberId) { throw e; } catch (Exception e) { log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e); - throw e; } - // also clear the assigned partitions since all have been revoked - subscriptions.assignFromSubscribed(Collections.emptySet()); - break; case COOPERATIVE: diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index 4e7d9c2276cf1..2ea3652d7a8d3 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -124,8 +124,9 @@ public class ConsumerCoordinatorTest { private final Heartbeat heartbeat = new Heartbeat(time, sessionTimeoutMs, heartbeatIntervalMs, rebalanceTimeoutMs, retryBackoffMs); - private MockPartitionAssignor partitionAssignor = new MockPartitionAssignor(); - private List assignors = Collections.singletonList(partitionAssignor); + private final PartitionAssignor.RebalanceProtocol protocol; + private final MockPartitionAssignor partitionAssignor; + private final List assignors; private MockClient client; private MetadataResponse metadataResponse = TestUtils.metadataUpdateWith(1, new HashMap() { { @@ -142,14 +143,16 @@ public class ConsumerCoordinatorTest { private MockCommitCallback mockOffsetCommitCallback; private ConsumerCoordinator coordinator; - public ConsumerCoordinatorTest(final ConsumerCoordinator.RebalanceProtocol protocol) { + public ConsumerCoordinatorTest(final PartitionAssignor.RebalanceProtocol protocol) { this.protocol = protocol; + this.partitionAssignor = new MockPartitionAssignor(protocol); + this.assignors = Collections.singletonList(partitionAssignor); } @Parameterized.Parameters(name = "rebalance protocol = {0}") public static Collection data() { final List values = new ArrayList<>(); - for (final ConsumerCoordinator.RebalanceProtocol protocol: ConsumerCoordinator.RebalanceProtocol.values()) { + for (final PartitionAssignor.RebalanceProtocol protocol: PartitionAssignor.RebalanceProtocol.values()) { values.add(new Object[]{protocol}); } return values; @@ -517,7 +520,7 @@ public boolean matches(AbstractRequest body) { final int addCount = 1; // with eager protocol we will call revoke on the old assignment as well - if (protocol == ConsumerCoordinator.RebalanceProtocol.EAGER) { + if (protocol == PartitionAssignor.RebalanceProtocol.EAGER) { revokeCount += 1; } @@ -2105,8 +2108,11 @@ public void testHeartbeatThreadClose() throws Exception { closeVerifyTimeout(coordinator, Long.MAX_VALUE, requestTimeoutMs, requestTimeoutMs); Thread[] threads = new Thread[Thread.activeCount()]; int threadCount = Thread.enumerate(threads); - for (int i = 0; i < threadCount; i++) + for (int i = 0; i < threadCount; i++) { + System.out.println(threads[i].getName()); + assertFalse("Heartbeat thread active after close", threads[i].getName().contains(groupId)); + } } @Test diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MockPartitionAssignor.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MockPartitionAssignor.java index 609c773ff6ef7..fe29b9c2a2d32 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MockPartitionAssignor.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MockPartitionAssignor.java @@ -18,13 +18,20 @@ import org.apache.kafka.common.TopicPartition; +import java.util.Collections; import java.util.List; import java.util.Map; public class MockPartitionAssignor extends AbstractPartitionAssignor { + private final RebalanceProtocol supportedProtocol; + private Map> result = null; + public MockPartitionAssignor(final RebalanceProtocol supportedProtocol) { + this.supportedProtocol = supportedProtocol; + } + @Override public Map> assign(Map partitionsPerTopic, Map subscriptions) { @@ -38,6 +45,11 @@ public String name() { return "consumer-mock-assignor"; } + @Override + public List supportedProtocols() { + return Collections.singletonList(supportedProtocol); + } + public void clear() { this.result = null; } From 5325e5653e2b4a288cdb07ec30838fda3134cb74 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 21 May 2019 09:52:55 -0700 Subject: [PATCH 21/59] fix unit test ConsumerCoordinatorTest --- .../internals/ConsumerCoordinatorTest.java | 446 ++++++++++-------- 1 file changed, 238 insertions(+), 208 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index 2ea3652d7a8d3..86c6dbdee53d9 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -178,8 +178,22 @@ public void setup() { @After public void teardown() { + + Thread[] threads = new Thread[Thread.activeCount()]; + int threadCount = Thread.enumerate(threads); + for (int i = 0; i < threadCount; i++) { + System.out.println(threads[i].getName()); + } + this.metrics.close(); this.coordinator.close(time.timer(0)); + + System.out.println("-----------------------------------"); + threads = new Thread[Thread.activeCount()]; + threadCount = Thread.enumerate(threads); + for (int i = 0; i < threadCount; i++) { + System.out.println(threads[i].getName()); + } } @Test @@ -1129,51 +1143,52 @@ public boolean matches(AbstractRequest body) { @Test public void testWakeupFromAssignmentCallback() { - ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - false, Optional.empty()); + try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, + false, Optional.empty()) + ) { + final String topic = "topic1"; + TopicPartition partition = new TopicPartition(topic, 0); + final String consumerId = "follower"; + Set topics = Collections.singleton(topic1); + MockRebalanceListener rebalanceListener = new MockRebalanceListener() { + @Override + public void onPartitionsAssigned(Collection partitions) { + boolean raiseWakeup = this.assignedCount == 0; + super.onPartitionsAssigned(partitions); - final String topic = "topic1"; - TopicPartition partition = new TopicPartition(topic, 0); - final String consumerId = "follower"; - Set topics = Collections.singleton(topic1); - MockRebalanceListener rebalanceListener = new MockRebalanceListener() { - @Override - public void onPartitionsAssigned(Collection partitions) { - boolean raiseWakeup = this.assignedCount == 0; - super.onPartitionsAssigned(partitions); + if (raiseWakeup) + throw new WakeupException(); + } + }; - if (raiseWakeup) - throw new WakeupException(); - } - }; + subscriptions.subscribe(topics, rebalanceListener); - subscriptions.subscribe(topics, rebalanceListener); + // we only have metadata for one topic initially + client.updateMetadata(TestUtils.metadataUpdateWith(1, singletonMap(topic1, 1))); - // we only have metadata for one topic initially - client.updateMetadata(TestUtils.metadataUpdateWith(1, singletonMap(topic1, 1))); + client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); + coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); - client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); - coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); + // prepare initial rebalance + partitionAssignor.prepare(singletonMap(consumerId, Collections.singletonList(t1p))); - // prepare initial rebalance - partitionAssignor.prepare(singletonMap(consumerId, Collections.singletonList(t1p))); + client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE)); + client.prepareResponse(syncGroupResponse(Collections.singletonList(t1p), Errors.NONE)); - client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE)); - client.prepareResponse(syncGroupResponse(Collections.singletonList(t1p), Errors.NONE)); + // The first call to poll should raise the exception from the rebalance listener + try { + coordinator.poll(time.timer(Long.MAX_VALUE)); + fail("Expected exception thrown from assignment callback"); + } catch (WakeupException e) { + } - // The first call to poll should raise the exception from the rebalance listener - try { + // The second call should retry the assignment callback and succeed coordinator.poll(time.timer(Long.MAX_VALUE)); - fail("Expected exception thrown from assignment callback"); - } catch (WakeupException e) { - } - // The second call should retry the assignment callback and succeed - coordinator.poll(time.timer(Long.MAX_VALUE)); - - assertFalse(coordinator.rejoinNeededOrPending()); - assertEquals(1, rebalanceListener.revokedCount); - assertEquals(2, rebalanceListener.assignedCount); + assertFalse(coordinator.rejoinNeededOrPending()); + assertEquals(1, rebalanceListener.revokedCount); + assertEquals(2, rebalanceListener.assignedCount); + } } @Test @@ -1246,22 +1261,22 @@ private void testInternalTopicInclusion(boolean includeInternalTopics) { metadata = new ConsumerMetadata(0, Long.MAX_VALUE, includeInternalTopics, false, subscriptions, new LogContext(), new ClusterResourceListeners()); client = new MockClient(time, metadata); - coordinator = buildCoordinator(new Metrics(), assignors, false, Optional.empty()); + try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, false, Optional.empty())) { + subscriptions.subscribe(Pattern.compile(".*"), rebalanceListener); - subscriptions.subscribe(Pattern.compile(".*"), rebalanceListener); - - Node node = new Node(0, "localhost", 9999); - MetadataResponse.PartitionMetadata partitionMetadata = + Node node = new Node(0, "localhost", 9999); + MetadataResponse.PartitionMetadata partitionMetadata = new MetadataResponse.PartitionMetadata(Errors.NONE, 0, node, Optional.empty(), - singletonList(node), singletonList(node), singletonList(node)); - MetadataResponse.TopicMetadata topicMetadata = new MetadataResponse.TopicMetadata(Errors.NONE, + singletonList(node), singletonList(node), singletonList(node)); + MetadataResponse.TopicMetadata topicMetadata = new MetadataResponse.TopicMetadata(Errors.NONE, Topic.GROUP_METADATA_TOPIC_NAME, true, singletonList(partitionMetadata)); - client.updateMetadata(MetadataResponse.prepareResponse(singletonList(node), "clusterId", node.id(), + client.updateMetadata(MetadataResponse.prepareResponse(singletonList(node), "clusterId", node.id(), singletonList(topicMetadata))); - coordinator.maybeUpdateSubscriptionMetadata(); + coordinator.maybeUpdateSubscriptionMetadata(); - assertEquals(includeInternalTopics, subscriptions.subscription().contains(Topic.GROUP_METADATA_TOPIC_NAME)); + assertEquals(includeInternalTopics, subscriptions.subscription().contains(Topic.GROUP_METADATA_TOPIC_NAME)); + } } @Test @@ -1381,157 +1396,162 @@ private void testInFlightRequestsFailedAfterCoordinatorMarkedDead(Errors error) public void testAutoCommitDynamicAssignment() { final String consumerId = "consumer"; - ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, groupInstanceId); - - subscriptions.subscribe(singleton(topic1), rebalanceListener); - joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p)); - subscriptions.seek(t1p, 100); + try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, + true, groupInstanceId) + ) { + subscriptions.subscribe(singleton(topic1), rebalanceListener); + joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p)); + subscriptions.seek(t1p, 100); - prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE); - time.sleep(autoCommitIntervalMs); - coordinator.poll(time.timer(Long.MAX_VALUE)); - assertFalse(client.hasPendingResponses()); + prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE); + time.sleep(autoCommitIntervalMs); + coordinator.poll(time.timer(Long.MAX_VALUE)); + assertFalse(client.hasPendingResponses()); + } } @Test public void testAutoCommitRetryBackoff() { final String consumerId = "consumer"; - ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, groupInstanceId); - subscriptions.subscribe(singleton(topic1), rebalanceListener); - joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p)); - subscriptions.seek(t1p, 100); - time.sleep(autoCommitIntervalMs); + try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, + true, groupInstanceId)) { + subscriptions.subscribe(singleton(topic1), rebalanceListener); + joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p)); - // Send an offset commit, but let it fail with a retriable error - prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NOT_COORDINATOR); - coordinator.poll(time.timer(Long.MAX_VALUE)); - assertTrue(coordinator.coordinatorUnknown()); + subscriptions.seek(t1p, 100); + time.sleep(autoCommitIntervalMs); - // After the disconnect, we should rediscover the coordinator - client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); - coordinator.poll(time.timer(Long.MAX_VALUE)); + // Send an offset commit, but let it fail with a retriable error + prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NOT_COORDINATOR); + coordinator.poll(time.timer(Long.MAX_VALUE)); + assertTrue(coordinator.coordinatorUnknown()); - subscriptions.seek(t1p, 200); + // After the disconnect, we should rediscover the coordinator + client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); + coordinator.poll(time.timer(Long.MAX_VALUE)); - // Until the retry backoff has expired, we should not retry the offset commit - time.sleep(retryBackoffMs / 2); - coordinator.poll(time.timer(Long.MAX_VALUE)); - assertEquals(0, client.inFlightRequestCount()); + subscriptions.seek(t1p, 200); - // Once the backoff expires, we should retry - time.sleep(retryBackoffMs / 2); - coordinator.poll(time.timer(Long.MAX_VALUE)); - assertEquals(1, client.inFlightRequestCount()); - respondToOffsetCommitRequest(singletonMap(t1p, 200L), Errors.NONE); + // Until the retry backoff has expired, we should not retry the offset commit + time.sleep(retryBackoffMs / 2); + coordinator.poll(time.timer(Long.MAX_VALUE)); + assertEquals(0, client.inFlightRequestCount()); + + // Once the backoff expires, we should retry + time.sleep(retryBackoffMs / 2); + coordinator.poll(time.timer(Long.MAX_VALUE)); + assertEquals(1, client.inFlightRequestCount()); + respondToOffsetCommitRequest(singletonMap(t1p, 200L), Errors.NONE); + } } @Test public void testAutoCommitAwaitsInterval() { final String consumerId = "consumer"; - ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, true, groupInstanceId); - subscriptions.subscribe(singleton(topic1), rebalanceListener); - joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p)); - subscriptions.seek(t1p, 100); - time.sleep(autoCommitIntervalMs); + try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, true, groupInstanceId)) { + subscriptions.subscribe(singleton(topic1), rebalanceListener); + joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p)); - // Send the offset commit request, but do not respond - coordinator.poll(time.timer(Long.MAX_VALUE)); - assertEquals(1, client.inFlightRequestCount()); + subscriptions.seek(t1p, 100); + time.sleep(autoCommitIntervalMs); - time.sleep(autoCommitIntervalMs / 2); + // Send the offset commit request, but do not respond + coordinator.poll(time.timer(Long.MAX_VALUE)); + assertEquals(1, client.inFlightRequestCount()); - // Ensure that no additional offset commit is sent - coordinator.poll(time.timer(Long.MAX_VALUE)); - assertEquals(1, client.inFlightRequestCount()); + time.sleep(autoCommitIntervalMs / 2); - respondToOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE); - coordinator.poll(time.timer(Long.MAX_VALUE)); - assertEquals(0, client.inFlightRequestCount()); + // Ensure that no additional offset commit is sent + coordinator.poll(time.timer(Long.MAX_VALUE)); + assertEquals(1, client.inFlightRequestCount()); + + respondToOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE); + coordinator.poll(time.timer(Long.MAX_VALUE)); + assertEquals(0, client.inFlightRequestCount()); - subscriptions.seek(t1p, 200); + subscriptions.seek(t1p, 200); - // If we poll again before the auto-commit interval, there should be no new sends - coordinator.poll(time.timer(Long.MAX_VALUE)); - assertEquals(0, client.inFlightRequestCount()); + // If we poll again before the auto-commit interval, there should be no new sends + coordinator.poll(time.timer(Long.MAX_VALUE)); + assertEquals(0, client.inFlightRequestCount()); - // After the remainder of the interval passes, we send a new offset commit - time.sleep(autoCommitIntervalMs / 2); - coordinator.poll(time.timer(Long.MAX_VALUE)); - assertEquals(1, client.inFlightRequestCount()); - respondToOffsetCommitRequest(singletonMap(t1p, 200L), Errors.NONE); + // After the remainder of the interval passes, we send a new offset commit + time.sleep(autoCommitIntervalMs / 2); + coordinator.poll(time.timer(Long.MAX_VALUE)); + assertEquals(1, client.inFlightRequestCount()); + respondToOffsetCommitRequest(singletonMap(t1p, 200L), Errors.NONE); + } } @Test public void testAutoCommitDynamicAssignmentRebalance() { final String consumerId = "consumer"; - ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, groupInstanceId); + try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, + true, groupInstanceId)) { + subscriptions.subscribe(singleton(topic1), rebalanceListener); - subscriptions.subscribe(singleton(topic1), rebalanceListener); + client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); + coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); - client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); - coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); - - // haven't joined, so should not cause a commit - time.sleep(autoCommitIntervalMs); - consumerClient.poll(time.timer(0)); + // haven't joined, so should not cause a commit + time.sleep(autoCommitIntervalMs); + consumerClient.poll(time.timer(0)); - client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE)); - client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE)); - coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE)); + client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE)); + client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE)); + coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE)); - subscriptions.seek(t1p, 100); + subscriptions.seek(t1p, 100); - prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE); - time.sleep(autoCommitIntervalMs); - coordinator.poll(time.timer(Long.MAX_VALUE)); - assertFalse(client.hasPendingResponses()); + prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE); + time.sleep(autoCommitIntervalMs); + coordinator.poll(time.timer(Long.MAX_VALUE)); + assertFalse(client.hasPendingResponses()); + } } @Test public void testAutoCommitManualAssignment() { - ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, groupInstanceId); + try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, + true, groupInstanceId)) { + subscriptions.assignFromUser(singleton(t1p)); + subscriptions.seek(t1p, 100); - subscriptions.assignFromUser(singleton(t1p)); - subscriptions.seek(t1p, 100); + client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); + coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); - client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); - coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); - - prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE); - time.sleep(autoCommitIntervalMs); - coordinator.poll(time.timer(Long.MAX_VALUE)); - assertFalse(client.hasPendingResponses()); + prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE); + time.sleep(autoCommitIntervalMs); + coordinator.poll(time.timer(Long.MAX_VALUE)); + assertFalse(client.hasPendingResponses()); + } } @Test public void testAutoCommitManualAssignmentCoordinatorUnknown() { - ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, groupInstanceId); - - subscriptions.assignFromUser(singleton(t1p)); - subscriptions.seek(t1p, 100); + try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, + true, groupInstanceId)) { + subscriptions.assignFromUser(singleton(t1p)); + subscriptions.seek(t1p, 100); - // no commit initially since coordinator is unknown - consumerClient.poll(time.timer(0)); - time.sleep(autoCommitIntervalMs); - consumerClient.poll(time.timer(0)); + // no commit initially since coordinator is unknown + consumerClient.poll(time.timer(0)); + time.sleep(autoCommitIntervalMs); + consumerClient.poll(time.timer(0)); - // now find the coordinator - client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); - coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); + // now find the coordinator + client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); + coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); - // sleep only for the retry backoff - time.sleep(retryBackoffMs); - prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE); - coordinator.poll(time.timer(Long.MAX_VALUE)); - assertFalse(client.hasPendingResponses()); + // sleep only for the retry backoff + time.sleep(retryBackoffMs); + prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE); + coordinator.poll(time.timer(Long.MAX_VALUE)); + assertFalse(client.hasPendingResponses()); + } } @Test @@ -2023,116 +2043,126 @@ public boolean conditionMet() { @Test public void testCloseDynamicAssignment() throws Exception { - ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, true, Optional.empty()); - gracefulCloseTest(coordinator, true); + try (ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, true, Optional.empty())) { + gracefulCloseTest(coordinator, true); + } } @Test public void testCloseManualAssignment() throws Exception { - ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(false, true, Optional.empty()); - gracefulCloseTest(coordinator, false); + try (ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, true, Optional.empty())) { + gracefulCloseTest(coordinator, false); + } } @Test public void testCloseCoordinatorNotKnownManualAssignment() throws Exception { - ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(false, true, Optional.empty()); - makeCoordinatorUnknown(coordinator, Errors.NOT_COORDINATOR); - time.sleep(autoCommitIntervalMs); - closeVerifyTimeout(coordinator, 1000, 1000, 1000); + try (ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(false, true, Optional.empty())) { + makeCoordinatorUnknown(coordinator, Errors.NOT_COORDINATOR); + time.sleep(autoCommitIntervalMs); + closeVerifyTimeout(coordinator, 1000, 1000, 1000); + } } @Test public void testCloseCoordinatorNotKnownNoCommits() throws Exception { - ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, false, Optional.empty()); - makeCoordinatorUnknown(coordinator, Errors.NOT_COORDINATOR); - closeVerifyTimeout(coordinator, 1000, 0, 0); + try (ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, false, Optional.empty())) { + makeCoordinatorUnknown(coordinator, Errors.NOT_COORDINATOR); + closeVerifyTimeout(coordinator, 1000, 0, 0); + } } @Test public void testCloseCoordinatorNotKnownWithCommits() throws Exception { - ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, true, groupInstanceId); - makeCoordinatorUnknown(coordinator, Errors.NOT_COORDINATOR); - time.sleep(autoCommitIntervalMs); - closeVerifyTimeout(coordinator, 1000, 1000, 1000); + try (ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, true, Optional.empty())) { + makeCoordinatorUnknown(coordinator, Errors.NOT_COORDINATOR); + time.sleep(autoCommitIntervalMs); + closeVerifyTimeout(coordinator, 1000, 1000, 1000); + } } @Test public void testCloseCoordinatorUnavailableNoCommits() throws Exception { - ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, false, Optional.empty()); - makeCoordinatorUnknown(coordinator, Errors.COORDINATOR_NOT_AVAILABLE); - closeVerifyTimeout(coordinator, 1000, 0, 0); + try (ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, false, Optional.empty())) { + makeCoordinatorUnknown(coordinator, Errors.COORDINATOR_NOT_AVAILABLE); + closeVerifyTimeout(coordinator, 1000, 0, 0); + } } @Test public void testCloseTimeoutCoordinatorUnavailableForCommit() throws Exception { - ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, true, groupInstanceId); - makeCoordinatorUnknown(coordinator, Errors.COORDINATOR_NOT_AVAILABLE); - time.sleep(autoCommitIntervalMs); - closeVerifyTimeout(coordinator, 1000, 1000, 1000); + try (ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, true, groupInstanceId)) { + makeCoordinatorUnknown(coordinator, Errors.COORDINATOR_NOT_AVAILABLE); + time.sleep(autoCommitIntervalMs); + closeVerifyTimeout(coordinator, 1000, 1000, 1000); + } } @Test public void testCloseMaxWaitCoordinatorUnavailableForCommit() throws Exception { - ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, true, groupInstanceId); - makeCoordinatorUnknown(coordinator, Errors.COORDINATOR_NOT_AVAILABLE); - time.sleep(autoCommitIntervalMs); - closeVerifyTimeout(coordinator, Long.MAX_VALUE, requestTimeoutMs, requestTimeoutMs); + try (ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, true, groupInstanceId)) { + makeCoordinatorUnknown(coordinator, Errors.COORDINATOR_NOT_AVAILABLE); + time.sleep(autoCommitIntervalMs); + closeVerifyTimeout(coordinator, Long.MAX_VALUE, requestTimeoutMs, requestTimeoutMs); + } } @Test public void testCloseNoResponseForCommit() throws Exception { - ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, true, groupInstanceId); - time.sleep(autoCommitIntervalMs); - closeVerifyTimeout(coordinator, Long.MAX_VALUE, requestTimeoutMs, requestTimeoutMs); + try (ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, true, groupInstanceId)) { + time.sleep(autoCommitIntervalMs); + closeVerifyTimeout(coordinator, Long.MAX_VALUE, requestTimeoutMs, requestTimeoutMs); + } } @Test public void testCloseNoResponseForLeaveGroup() throws Exception { - ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, false, Optional.empty()); - closeVerifyTimeout(coordinator, Long.MAX_VALUE, requestTimeoutMs, requestTimeoutMs); + try (ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, false, Optional.empty())) { + closeVerifyTimeout(coordinator, Long.MAX_VALUE, requestTimeoutMs, requestTimeoutMs); + } } @Test public void testCloseNoWait() throws Exception { - ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, true, groupInstanceId); - time.sleep(autoCommitIntervalMs); - closeVerifyTimeout(coordinator, 0, 0, 0); + try (ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, true, groupInstanceId)) { + time.sleep(autoCommitIntervalMs); + closeVerifyTimeout(coordinator, 0, 0, 0); + } } @Test public void testHeartbeatThreadClose() throws Exception { - ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, true, groupInstanceId); - coordinator.ensureActiveGroup(); - time.sleep(heartbeatIntervalMs + 100); - Thread.yield(); // Give heartbeat thread a chance to attempt heartbeat - closeVerifyTimeout(coordinator, Long.MAX_VALUE, requestTimeoutMs, requestTimeoutMs); - Thread[] threads = new Thread[Thread.activeCount()]; - int threadCount = Thread.enumerate(threads); - for (int i = 0; i < threadCount; i++) { - System.out.println(threads[i].getName()); - - assertFalse("Heartbeat thread active after close", threads[i].getName().contains(groupId)); + try (ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, true, groupInstanceId)) { + coordinator.ensureActiveGroup(); + time.sleep(heartbeatIntervalMs + 100); + Thread.yield(); // Give heartbeat thread a chance to attempt heartbeat + closeVerifyTimeout(coordinator, Long.MAX_VALUE, requestTimeoutMs, requestTimeoutMs); + Thread[] threads = new Thread[Thread.activeCount()]; + int threadCount = Thread.enumerate(threads); + for (int i = 0; i < threadCount; i++) { + assertFalse("Heartbeat thread active after close", threads[i].getName().contains(groupId)); + } } } @Test public void testAutoCommitAfterCoordinatorBackToService() { - ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, groupInstanceId); - - subscriptions.assignFromUser(Collections.singleton(t1p)); - subscriptions.seek(t1p, 100L); - - coordinator.markCoordinatorUnknown(); - assertTrue(coordinator.coordinatorUnknown()); - client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); - prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE); - - // async commit offset should find coordinator - time.sleep(autoCommitIntervalMs); // sleep for a while to ensure auto commit does happen - coordinator.maybeAutoCommitOffsetsAsync(time.milliseconds()); - assertFalse(coordinator.coordinatorUnknown()); - assertEquals(100L, subscriptions.position(t1p).offset); + try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, + true, groupInstanceId)) { + subscriptions.assignFromUser(Collections.singleton(t1p)); + subscriptions.seek(t1p, 100L); + + coordinator.markCoordinatorUnknown(); + assertTrue(coordinator.coordinatorUnknown()); + client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); + prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE); + + // async commit offset should find coordinator + time.sleep(autoCommitIntervalMs); // sleep for a while to ensure auto commit does happen + coordinator.maybeAutoCommitOffsetsAsync(time.milliseconds()); + assertFalse(coordinator.coordinatorUnknown()); + assertEquals(100L, subscriptions.position(t1p).offset); + } } @Test(expected = FencedInstanceIdException.class) From 72e70d6fc1f80f2f82d59c210b96c211b4bcb8cd Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 21 May 2019 10:30:49 -0700 Subject: [PATCH 22/59] add more unit tests --- .../clients/consumer/ConsumerConfig.java | 1 - .../internals/ConsumerCoordinator.java | 8 +++++ .../internals/ConsumerCoordinatorTest.java | 33 +++++++++++-------- .../internals/MockPartitionAssignor.java | 8 ++--- 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java index e5c019489fb49..ff2e5cd8cc57d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerConfig.java @@ -18,7 +18,6 @@ import org.apache.kafka.clients.ClientDnsLookup; import org.apache.kafka.clients.CommonClientConfigs; -import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; import org.apache.kafka.common.config.AbstractConfig; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigDef.Importance; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index af425907acd8c..4d5af3ccafe5b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -181,6 +181,10 @@ public ConsumerCoordinator(LogContext logContext, supportedProtocols.retainAll(assignor.supportedProtocols()); } + if (supportedProtocols.isEmpty()) { + throw new IllegalArgumentException("Specified assignors do not have commonly supported rebalance protocol"); + } + Collections.sort(supportedProtocols); protocol = supportedProtocols.get(supportedProtocols.size() - 1); @@ -1177,4 +1181,8 @@ public void invoke() { } } + /* test-only classes below */ + RebalanceProtocol getProtocol() { + return protocol; + } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index 86c6dbdee53d9..ec47f8460acd9 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -145,7 +145,7 @@ public class ConsumerCoordinatorTest { public ConsumerCoordinatorTest(final PartitionAssignor.RebalanceProtocol protocol) { this.protocol = protocol; - this.partitionAssignor = new MockPartitionAssignor(protocol); + this.partitionAssignor = new MockPartitionAssignor(Collections.singletonList(protocol)); this.assignors = Collections.singletonList(partitionAssignor); } @@ -178,21 +178,26 @@ public void setup() { @After public void teardown() { - - Thread[] threads = new Thread[Thread.activeCount()]; - int threadCount = Thread.enumerate(threads); - for (int i = 0; i < threadCount; i++) { - System.out.println(threads[i].getName()); - } - this.metrics.close(); this.coordinator.close(time.timer(0)); + } + + @Test + public void testSelectRebalanceProtcol() { + List assignors = new ArrayList<>(); + assignors.add(new MockPartitionAssignor(Collections.singletonList(PartitionAssignor.RebalanceProtocol.EAGER))); + assignors.add(new MockPartitionAssignor(Collections.singletonList(PartitionAssignor.RebalanceProtocol.COOPERATIVE))); - System.out.println("-----------------------------------"); - threads = new Thread[Thread.activeCount()]; - threadCount = Thread.enumerate(threads); - for (int i = 0; i < threadCount; i++) { - System.out.println(threads[i].getName()); + // no commonly supported protocols + assertThrows(IllegalArgumentException.class, () -> buildCoordinator(new Metrics(), assignors, false, Optional.empty())); + + assignors.clear(); + assignors.add(new MockPartitionAssignor(Arrays.asList(PartitionAssignor.RebalanceProtocol.EAGER, PartitionAssignor.RebalanceProtocol.COOPERATIVE))); + assignors.add(new MockPartitionAssignor(Arrays.asList(PartitionAssignor.RebalanceProtocol.EAGER, PartitionAssignor.RebalanceProtocol.COOPERATIVE))); + + // select higher indexed (more advanced) protocols + try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, false, Optional.empty())) { + assertEquals(PartitionAssignor.RebalanceProtocol.COOPERATIVE, coordinator.getProtocol()); } } @@ -2050,7 +2055,7 @@ public void testCloseDynamicAssignment() throws Exception { @Test public void testCloseManualAssignment() throws Exception { - try (ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(true, true, Optional.empty())) { + try (ConsumerCoordinator coordinator = prepareCoordinatorForCloseTest(false, true, Optional.empty())) { gracefulCloseTest(coordinator, false); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MockPartitionAssignor.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MockPartitionAssignor.java index fe29b9c2a2d32..eb3f618e13cf9 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MockPartitionAssignor.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MockPartitionAssignor.java @@ -24,12 +24,12 @@ public class MockPartitionAssignor extends AbstractPartitionAssignor { - private final RebalanceProtocol supportedProtocol; + private final List supportedProtocols; private Map> result = null; - public MockPartitionAssignor(final RebalanceProtocol supportedProtocol) { - this.supportedProtocol = supportedProtocol; + public MockPartitionAssignor(final List supportedProtocols) { + this.supportedProtocols = supportedProtocols; } @Override @@ -47,7 +47,7 @@ public String name() { @Override public List supportedProtocols() { - return Collections.singletonList(supportedProtocol); + return supportedProtocols; } public void clear() { From ba97f991fe872736b5d247f6cc04e6fc3ce01ec8 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 21 May 2019 10:52:51 -0700 Subject: [PATCH 23/59] fix checkstyle --- .../clients/consumer/internals/MockPartitionAssignor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MockPartitionAssignor.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MockPartitionAssignor.java index eb3f618e13cf9..ca7cb34ddd414 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MockPartitionAssignor.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/MockPartitionAssignor.java @@ -18,7 +18,6 @@ import org.apache.kafka.common.TopicPartition; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -28,7 +27,7 @@ public class MockPartitionAssignor extends AbstractPartitionAssignor { private Map> result = null; - public MockPartitionAssignor(final List supportedProtocols) { + MockPartitionAssignor(final List supportedProtocols) { this.supportedProtocols = supportedProtocols; } From cc4c2920e4535ebc99ab1a2f12d958ce847d1b87 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 4 Jun 2019 18:46:13 -0700 Subject: [PATCH 24/59] lost partition whenever resetGeneration or metadata / subscription change --- .../internals/AbstractCoordinator.java | 14 +-- .../internals/ConsumerCoordinator.java | 88 +++++++++++-------- 2 files changed, 57 insertions(+), 45 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java index 54678f7ada3eb..0ed39b0694c63 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java @@ -553,7 +553,7 @@ public void handle(JoinGroupResponse joinResponse, RequestFuture fut future.raise(error); } else if (error == Errors.UNKNOWN_MEMBER_ID) { // reset the member id and retry immediately - resetGeneration(); + resetGeneration(this.getClass().getName() + " encountering " + error); log.debug("Attempt to join group failed due to unknown member id."); future.raise(Errors.UNKNOWN_MEMBER_ID); } else if (error == Errors.COORDINATOR_NOT_AVAILABLE @@ -675,7 +675,7 @@ public void handle(SyncGroupResponse syncResponse, } else if (error == Errors.UNKNOWN_MEMBER_ID || error == Errors.ILLEGAL_GENERATION) { log.debug("SyncGroup failed: {}", error.message()); - resetGeneration(); + resetGeneration(this.getClass().getName() + " encountering " + error); future.raise(error); } else if (error == Errors.COORDINATOR_NOT_AVAILABLE || error == Errors.NOT_COORDINATOR) { @@ -824,7 +824,9 @@ final synchronized boolean hasValidMemberId() { /** * Reset the generation and memberId because we have fallen out of the group. */ - protected synchronized void resetGeneration() { + protected synchronized void resetGeneration(String errorMessage) { + log.debug("Resetting generation and requesting re-join group due to {}", errorMessage); + this.generation = Generation.NO_GENERATION; this.rejoinNeeded = true; this.state = MemberState.UNJOINED; @@ -884,7 +886,7 @@ public synchronized void maybeLeaveGroup() { client.pollNoWakeup(); } - resetGeneration(); + resetGeneration("consumer proactively leaving group"); } protected boolean isDynamicMember() { @@ -938,14 +940,14 @@ public void handle(HeartbeatResponse heartbeatResponse, RequestFuture futu future.raise(Errors.REBALANCE_IN_PROGRESS); } else if (error == Errors.ILLEGAL_GENERATION) { log.info("Attempt to heartbeat failed since generation {} is not current", generation.generationId); - resetGeneration(); + resetGeneration(this.getClass().getName() + " encountering " + error); future.raise(Errors.ILLEGAL_GENERATION); } else if (error == Errors.FENCED_INSTANCE_ID) { log.error("Received fatal exception: group.instance.id gets fenced"); future.raise(error); } else if (error == Errors.UNKNOWN_MEMBER_ID) { log.info("Attempt to heartbeat failed for since member id {} is not valid.", generation.memberId); - resetGeneration(); + resetGeneration(this.getClass().getName() + " encountering " + error); future.raise(Errors.UNKNOWN_MEMBER_ID); } else if (error == Errors.GROUP_AUTHORIZATION_FAILED) { future.raise(new GroupAuthorizationException(groupId)); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 6df0e9df44760..cb66b5b87135a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -46,7 +46,6 @@ import org.apache.kafka.common.metrics.stats.Max; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.record.RecordBatch; -import org.apache.kafka.common.requests.JoinGroupResponse; import org.apache.kafka.common.requests.OffsetCommitRequest; import org.apache.kafka.common.requests.OffsetCommitResponse; import org.apache.kafka.common.requests.OffsetFetchRequest; @@ -584,66 +583,77 @@ protected void onJoinPrepare(int generation, String memberId) { // execute the user's callback before rebalance ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); - // if member.id is unknown, it means a reset generation, - // hence we should always call onPartitionsLost to all - // previously assigned partitions regardless of the rebalance protocol - if (memberId.equals(JoinGroupResponse.UNKNOWN_MEMBER_ID)) { - Set emigrated = new HashSet<>(subscriptions.assignedPartitions()); - log.info("Emigrating previously assigned partitions {}", emigrated); - - try { - listener.onPartitionsLost(emigrated); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition emigration", listener.getClass().getName(), e); - } - - // also clear the assigned partitions since all have been emigrated - subscriptions.assignFromSubscribed(Collections.emptySet()); - } else { - switch (protocol) { - case EAGER: - Set revoked = new HashSet<>(subscriptions.assignedPartitions()); - log.info("Revoking previously assigned partitions {}", revoked); - try { - listener.onPartitionsLost(revoked); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e); - } + switch (protocol) { + case EAGER: + Set revokedPartitions = new HashSet<>(subscriptions.assignedPartitions()); + log.info("Revoking previously assigned partitions {}", revokedPartitions); + try { + listener.onPartitionsRevoked(revokedPartitions); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e); + } - // also clear the assigned partitions since all have been revoked - subscriptions.assignFromSubscribed(Collections.emptySet()); + // also clear the assigned partitions since all have been revoked + subscriptions.assignFromSubscribed(Collections.emptySet()); - break; + break; - case COOPERATIVE: - break; - } + case COOPERATIVE: + break; } isLeader = false; subscriptions.resetGroupSubscription(); } + @Override + public void resetGeneration(String errorMessage) { + maybeLosePartitions(errorMessage); + super.resetGeneration(errorMessage); + } + @Override public boolean rejoinNeededOrPending() { if (!subscriptions.partitionsAutoAssigned()) return false; // we need to rejoin if we performed the assignment and metadata has changed - if (assignmentSnapshot != null && !assignmentSnapshot.matches(metadataSnapshot)) + if (assignmentSnapshot != null && !assignmentSnapshot.matches(metadataSnapshot)) { + maybeLosePartitions("topic metadata has changed and therefore some topics may not exist any moreq"); return true; + } // we need to join if our subscription has changed since the last join - if (joinedSubscription != null && !joinedSubscription.equals(subscriptions.subscription())) + if (joinedSubscription != null && !joinedSubscription.equals(subscriptions.subscription())) { + maybeLosePartitions("consumer subscription has changed since last joined generation"); return true; + } return super.rejoinNeededOrPending(); } + private void maybeLosePartitions(String rootCause) { + Set lostPartitions = new HashSet<>(subscriptions.assignedPartitions()); + + if (!lostPartitions.isEmpty()) { + log.info("Lost previously assigned partitions {} due to {}", lostPartitions, rootCause); + + ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); + try { + listener.onPartitionsLost(lostPartitions); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition being lost", listener.getClass().getName(), e); + } + + // also clear the assigned partitions since all have been lost + subscriptions.assignFromSubscribed(Collections.emptySet()); + } + } + /** * Refresh the committed offsets for provided partitions. * @@ -1035,7 +1045,7 @@ public void handle(OffsetCommitResponse commitResponse, RequestFuture futu || error == Errors.ILLEGAL_GENERATION || error == Errors.REBALANCE_IN_PROGRESS) { // need to re-join group - resetGeneration(); + resetGeneration(this.getClass().getName() + " encountering " + error); future.raise(new CommitFailedException()); return; } else { From 99cafcd92cc0c17fde46e01bebccbb09fab6882b Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Wed, 5 Jun 2019 18:49:22 -0700 Subject: [PATCH 25/59] github comments --- .../consumer/internals/ConsumerCoordinator.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index ac1ada1a6eb5e..43cc6666cb653 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -174,7 +174,7 @@ public ConsumerCoordinator(LogContext logContext, // 1. only consider protocols that are supported by all the assignors. If there is no common protocols supported // across all the assignors, throw an exception. // 2. if there are multiple protocols that are commonly supported, select the one with the highest id (i.e. the - // id number indicates how advanced is the protocol). + // id number indicates how advanced the protocol is). // we know there are at least one assignor in the list, no need to double check for NPE List supportedProtocols = new ArrayList<>(assignors.get(0).supportedProtocols()); @@ -183,7 +183,9 @@ public ConsumerCoordinator(LogContext logContext, } if (supportedProtocols.isEmpty()) { - throw new IllegalArgumentException("Specified assignors do not have commonly supported rebalance protocol"); + throw new IllegalArgumentException("Specified assignors " + + assignors.stream().map(PartitionAssignor::name).collect(Collectors.toSet()) + + " do not have commonly supported rebalance protocol"); } Collections.sort(supportedProtocols); @@ -300,8 +302,8 @@ protected void onJoinComplete(int generation, switch (protocol) { case EAGER: if (!ownedPartitions.isEmpty()) { - log.warn("Some partitions are not revoked with " + protocol + " protocol, " + - "it is likely that the previous rebalance did not complete due to some errors"); + throw new IllegalStateException("Coordinator has some partitions are not revoked with " + + protocol + " protocol, it is likely that the previous rebalance did not complete due to some errors"); } log.info("Setting newly assigned partitions: {}", Utils.join(assignedPartitions, ", ")); From cf66222d137f75df12a2f79b6cb39c6d426c4bd1 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 18 Jun 2019 13:39:52 -0700 Subject: [PATCH 26/59] update tests --- .../clients/consumer/KafkaConsumerTest.java | 2 +- .../internals/ConsumerCoordinatorTest.java | 127 +++--------------- 2 files changed, 16 insertions(+), 113 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index 0f9b956663f16..c1adf1932ec6b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -1692,7 +1692,7 @@ public boolean matches(AbstractRequest body) { assertTrue(protocolIterator.hasNext()); ByteBuffer protocolMetadata = ByteBuffer.wrap(protocolIterator.next().metadata()); - PartitionAssignor.Subscription subscription = ConsumerProtocol.buildSubscription(protocolMetadata, Optional.empty()); + PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(protocolMetadata); return subscribedTopics.equals(new HashSet<>(subscription.topics())); } }, joinGroupFollowerResponse(assignor, 1, "memberId", "leaderId", Errors.NONE), coordinator); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index d9e16824282ae..b6262e5c70a6a 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -203,14 +203,14 @@ public void testSelectRebalanceProtcol() { assignors.add(new MockPartitionAssignor(Collections.singletonList(PartitionAssignor.RebalanceProtocol.COOPERATIVE))); // no commonly supported protocols - assertThrows(IllegalArgumentException.class, () -> buildCoordinator(new Metrics(), assignors, false, Optional.empty())); + assertThrows(IllegalArgumentException.class, () -> buildCoordinator(rebalanceConfig, new Metrics(), assignors, false)); assignors.clear(); assignors.add(new MockPartitionAssignor(Arrays.asList(PartitionAssignor.RebalanceProtocol.EAGER, PartitionAssignor.RebalanceProtocol.COOPERATIVE))); assignors.add(new MockPartitionAssignor(Arrays.asList(PartitionAssignor.RebalanceProtocol.EAGER, PartitionAssignor.RebalanceProtocol.COOPERATIVE))); // select higher indexed (more advanced) protocols - try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, false, Optional.empty())) { + try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, false)) { assertEquals(PartitionAssignor.RebalanceProtocol.COOPERATIVE, coordinator.getProtocol()); } } @@ -696,7 +696,7 @@ public boolean matches(AbstractRequest body) { JoinGroupRequestData.JoinGroupRequestProtocol protocolMetadata = protocolIterator.next(); ByteBuffer metadata = ByteBuffer.wrap(protocolMetadata.metadata()); - PartitionAssignor.Subscription subscription = ConsumerProtocol.buildSubscription(metadata, Optional.empty()); + PartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(metadata); metadata.rewind(); return subscription.topics().containsAll(updatedSubscription); } @@ -1276,16 +1276,8 @@ private void testInternalTopicInclusion(boolean includeInternalTopics) { metadata = new ConsumerMetadata(0, Long.MAX_VALUE, includeInternalTopics, false, subscriptions, new LogContext(), new ClusterResourceListeners()); client = new MockClient(time, metadata); -<<<<<<< HEAD - try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, false, Optional.empty())) { + try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, false)) { subscriptions.subscribe(Pattern.compile(".*"), rebalanceListener); -======= - coordinator = buildCoordinator(rebalanceConfig, - new Metrics(), - assignors, - false); ->>>>>>> 33e39de4ad5ff5cdea918e076565416e9295f46c - Node node = new Node(0, "localhost", 9999); MetadataResponse.PartitionMetadata partitionMetadata = new MetadataResponse.PartitionMetadata(Errors.NONE, 0, node, Optional.empty(), @@ -1418,24 +1410,12 @@ private void testInFlightRequestsFailedAfterCoordinatorMarkedDead(Errors error) public void testAutoCommitDynamicAssignment() { final String consumerId = "consumer"; -<<<<<<< HEAD - try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, groupInstanceId) + try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, + true) ) { subscriptions.subscribe(singleton(topic1), rebalanceListener); joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p)); subscriptions.seek(t1p, 100); -======= - ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, - new Metrics(), - assignors, - true); - - subscriptions.subscribe(singleton(topic1), rebalanceListener); - joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p)); - subscriptions.seek(t1p, 100); ->>>>>>> 33e39de4ad5ff5cdea918e076565416e9295f46c - prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE); time.sleep(autoCommitIntervalMs); coordinator.poll(time.timer(Long.MAX_VALUE)); @@ -1446,18 +1426,9 @@ public void testAutoCommitDynamicAssignment() { @Test public void testAutoCommitRetryBackoff() { final String consumerId = "consumer"; -<<<<<<< HEAD -======= - ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, - new Metrics(), - assignors, - true); - subscriptions.subscribe(singleton(topic1), rebalanceListener); - joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p)); ->>>>>>> 33e39de4ad5ff5cdea918e076565416e9295f46c - try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, groupInstanceId)) { + try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, + true)) { subscriptions.subscribe(singleton(topic1), rebalanceListener); joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p)); @@ -1491,17 +1462,7 @@ public void testAutoCommitRetryBackoff() { @Test public void testAutoCommitAwaitsInterval() { final String consumerId = "consumer"; -<<<<<<< HEAD -======= - ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, - new Metrics(), - assignors, - true); - subscriptions.subscribe(singleton(topic1), rebalanceListener); - joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p)); ->>>>>>> 33e39de4ad5ff5cdea918e076565416e9295f46c - - try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, true, groupInstanceId)) { + try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, true)) { subscriptions.subscribe(singleton(topic1), rebalanceListener); joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p)); @@ -1540,17 +1501,9 @@ public void testAutoCommitAwaitsInterval() { public void testAutoCommitDynamicAssignmentRebalance() { final String consumerId = "consumer"; -<<<<<<< HEAD - try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, groupInstanceId)) { + try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, + true)) { subscriptions.subscribe(singleton(topic1), rebalanceListener); -======= - ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, - new Metrics(), - assignors, - true); ->>>>>>> 33e39de4ad5ff5cdea918e076565416e9295f46c - client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); @@ -1573,21 +1526,9 @@ public void testAutoCommitDynamicAssignmentRebalance() { @Test public void testAutoCommitManualAssignment() { -<<<<<<< HEAD - try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, groupInstanceId)) { + try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, true)) { subscriptions.assignFromUser(singleton(t1p)); subscriptions.seek(t1p, 100); -======= - ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, - new Metrics(), - assignors, - true); - - subscriptions.assignFromUser(singleton(t1p)); - subscriptions.seek(t1p, 100); ->>>>>>> 33e39de4ad5ff5cdea918e076565416e9295f46c - client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); @@ -1600,20 +1541,9 @@ public void testAutoCommitManualAssignment() { @Test public void testAutoCommitManualAssignmentCoordinatorUnknown() { -<<<<<<< HEAD - try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, groupInstanceId)) { + try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, true)) { subscriptions.assignFromUser(singleton(t1p)); subscriptions.seek(t1p, 100); -======= - ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, - new Metrics(), - assignors, - true); - - subscriptions.assignFromUser(singleton(t1p)); - subscriptions.seek(t1p, 100); ->>>>>>> 33e39de4ad5ff5cdea918e076565416e9295f46c // no commit initially since coordinator is unknown consumerClient.poll(time.timer(0)); @@ -2268,9 +2198,7 @@ public void testHeartbeatThreadClose() throws Exception { @Test public void testAutoCommitAfterCoordinatorBackToService() { -<<<<<<< HEAD - try (ConsumerCoordinator coordinator = buildCoordinator(new Metrics(), assignors, - true, groupInstanceId)) { + try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, true)) { subscriptions.assignFromUser(Collections.singleton(t1p)); subscriptions.seek(t1p, 100L); @@ -2285,26 +2213,6 @@ public void testAutoCommitAfterCoordinatorBackToService() { assertFalse(coordinator.coordinatorUnknown()); assertEquals(100L, subscriptions.position(t1p).offset); } -======= - ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, - new Metrics(), - assignors, - true); - - subscriptions.assignFromUser(Collections.singleton(t1p)); - subscriptions.seek(t1p, 100L); - - coordinator.markCoordinatorUnknown(); - assertTrue(coordinator.coordinatorUnknown()); - client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); - prepareOffsetCommitRequest(singletonMap(t1p, 100L), Errors.NONE); - - // async commit offset should find coordinator - time.sleep(autoCommitIntervalMs); // sleep for a while to ensure auto commit does happen - coordinator.maybeAutoCommitOffsetsAsync(time.milliseconds()); - assertFalse(coordinator.coordinatorUnknown()); - assertEquals(100L, subscriptions.position(t1p).offset); ->>>>>>> 33e39de4ad5ff5cdea918e076565416e9295f46c } @Test(expected = FencedInstanceIdException.class) @@ -2458,9 +2366,7 @@ private ConsumerCoordinator buildCoordinator(final GroupRebalanceConfig rebalanc time, autoCommitEnabled, autoCommitIntervalMs, -<<<<<<< HEAD - null, - !groupInstanceId.isPresent()); + null); } private Collection getRevoked(final List owned, @@ -2489,9 +2395,6 @@ private Collection getAdded(final List owned, default: throw new IllegalStateException("This should not happen"); } -======= - null); ->>>>>>> 33e39de4ad5ff5cdea918e076565416e9295f46c } private FindCoordinatorResponse groupCoordinatorResponse(Node node, Errors error) { From d56b1f36952806fa83e88519c5a20c465f3e7b38 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 18 Jun 2019 14:42:59 -0700 Subject: [PATCH 27/59] minor fixes --- .../internals/ConsumerCoordinator.java | 8 ++--- .../consumer/internals/ConsumerProtocol.java | 31 +++++++++++++----- .../consumer/internals/PartitionAssignor.java | 18 +++++------ .../internals/ConsumerCoordinatorTest.java | 32 ++----------------- .../internals/ConsumerProtocolTest.java | 1 - 5 files changed, 39 insertions(+), 51 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 442a822d50316..6891ae02be884 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -293,8 +293,8 @@ protected void onJoinComplete(int generation, switch (protocol) { case EAGER: if (!ownedPartitions.isEmpty()) { - throw new IllegalStateException("Coordinator has some partitions are not revoked with " + - protocol + " protocol, it is likely that the previous rebalance did not complete due to some errors"); + log.info("Coordinator has owned partitions {} that are not revoked with {} protocol, " + + "it is likely client is woken up before a previous pending rebalance completes its callback", ownedPartitions, protocol); } log.info("Setting newly assigned partitions: {}", Utils.join(assignedPartitions, ", ")); @@ -310,7 +310,7 @@ protected void onJoinComplete(int generation, case COOPERATIVE: assignAndRevoke(listener, assignedPartitions, ownedPartitions); - if (assignment.error() == ConsumerProtocol.Errors.NEED_REJOIN) { + if (assignment.error() == ConsumerProtocol.AssignmentError.NEED_REJOIN) { requestRejoin(); } @@ -564,7 +564,7 @@ private void adjustAssignment(final Map ownedPartitions, // if revocations are triggered, tell everyone to re-join immediately. if (revocationsNeeded) { for (final Assignment assignment : assignments.values()) { - assignment.setError(ConsumerProtocol.Errors.NEED_REJOIN); + assignment.setError(ConsumerProtocol.AssignmentError.NEED_REJOIN); } } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java index b4ad4514eb60b..d05d5b06906a2 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java @@ -30,8 +30,6 @@ import java.util.List; import java.util.Map; -import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE; - /** * ConsumerProtocol contains the schemas for consumer subscriptions and assignments for use with * Kafka's generalized group management protocol. Below is the version 1 format: @@ -54,7 +52,22 @@ * ErrorCode => [int16] * * - * Older versioned formats can be inferred by reading the code below. + * Version 0 format: + * + * 
+ * Subscription => Version Topics
+ *   Version    => Int16
+ *   Topics     => [String]
+ *   UserData   => Bytes
+ *
+ * Assignment => Version TopicPartitions
+ *   Version            => int16
+ *   AssignedPartitions => [Topic Partitions]
+ *     Topic            => String
+ *     Partitions       => [int32]
+ *   UserData           => Bytes
+ *
+ * * * The current implementation assumes that future versions will not break compatibility. When * it encounters a newer version, it parses it using the current format. This basically means @@ -72,6 +85,8 @@ public class ConsumerProtocol { public static final String TOPIC_PARTITIONS_KEY_NAME = "topic_partitions"; public static final String USER_DATA_KEY_NAME = "user_data"; + public static final Field.Int16 ERROR_CODE = new Field.Int16("error_code", "Assignment error code"); + public static final short CONSUMER_PROTOCOL_V0 = 0; public static final short CONSUMER_PROTOCOL_V1 = 1; @@ -104,13 +119,13 @@ public class ConsumerProtocol { new Field(USER_DATA_KEY_NAME, Type.NULLABLE_BYTES), ERROR_CODE); - public enum Errors { + public enum AssignmentError { NONE(0), NEED_REJOIN(1); private final short code; - Errors(final int code) { + AssignmentError(final int code) { this.code = (short) code; } @@ -118,7 +133,7 @@ public short code() { return code; } - public static Errors fromCode(final short code) { + public static AssignmentError fromCode(final short code) { switch (code) { case 0: return NONE; @@ -258,7 +273,7 @@ public static ByteBuffer serializeAssignmentV1(PartitionAssignor.Assignment assi topicAssignments.add(topicAssignment); } struct.set(TOPIC_PARTITIONS_KEY_NAME, topicAssignments.toArray()); - struct.set(ERROR_CODE.name, assignment.error().code); + struct.set(ERROR_CODE, assignment.error().code); ByteBuffer buffer = ByteBuffer.allocate(CONSUMER_PROTOCOL_HEADER_V1.sizeOf() + ASSIGNMENT_V1.sizeOf(struct)); CONSUMER_PROTOCOL_HEADER_V1.writeTo(buffer); @@ -307,7 +322,7 @@ public static PartitionAssignor.Assignment deserializeAssignmentV1(ByteBuffer bu } } - Errors error = Errors.fromCode(struct.get(ERROR_CODE)); + AssignmentError error = AssignmentError.fromCode(struct.get(ERROR_CODE)); return new PartitionAssignor.Assignment(CONSUMER_PROTOCOL_V1, partitions, userData, error); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java index f3eca65f18d95..b8b6247517fba 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java @@ -80,7 +80,7 @@ default void onAssignment(Assignment assignment, int generation) { } /** - * Indicate which rebalance protocol this assignor can would work with; + * Indicate which rebalance protocol this assignor works with; * By default it should always work with {@link RebalanceProtocol#EAGER}. */ default List supportedProtocols() { @@ -88,7 +88,7 @@ default List supportedProtocols() { } /** - * Return the version of the assignor which indicate how the user metadata encodings + * Return the version of the assignor which indicates how the user metadata encodings * and the assignment algorithm gets evolved. */ default short version() { @@ -204,9 +204,9 @@ class Assignment { private final Short version; private final List partitions; private final ByteBuffer userData; - private ConsumerProtocol.Errors error; + private ConsumerProtocol.AssignmentError error; - Assignment(Short version, List partitions, ByteBuffer userData, ConsumerProtocol.Errors error) { + Assignment(Short version, List partitions, ByteBuffer userData, ConsumerProtocol.AssignmentError error) { this.version = version; this.partitions = partitions; this.userData = userData; @@ -215,15 +215,15 @@ class Assignment { if (version < CONSUMER_PROTOCOL_V0) throw new SchemaException("Unsupported subscription version: " + version); - if (version < CONSUMER_PROTOCOL_V1 && error != ConsumerProtocol.Errors.NONE) + if (version < CONSUMER_PROTOCOL_V1 && error != ConsumerProtocol.AssignmentError.NONE) throw new IllegalArgumentException("Assignment version smaller than 1 should not have error code."); } Assignment(Short version, List partitions, ByteBuffer userData) { - this(version, partitions, userData, ConsumerProtocol.Errors.NONE); + this(version, partitions, userData, ConsumerProtocol.AssignmentError.NONE); } - public Assignment(List partitions, ByteBuffer userData, ConsumerProtocol.Errors error) { + public Assignment(List partitions, ByteBuffer userData, ConsumerProtocol.AssignmentError error) { this(CONSUMER_PROTOCOL_V1, partitions, userData, error); } @@ -243,11 +243,11 @@ public List partitions() { return partitions; } - public ConsumerProtocol.Errors error() { + public ConsumerProtocol.AssignmentError error() { return error; } - public void setError(ConsumerProtocol.Errors error) { + public void setError(ConsumerProtocol.AssignmentError error) { this.error = error; } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index b6262e5c70a6a..98659a20b3420 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -566,32 +566,6 @@ public boolean matches(AbstractRequest body) { assertEquals(assigned, rebalanceListener.assigned); } - @Test - public void testInvalidCoordinatorAssignment() { - final String consumerId = "invalid_assignment"; - - subscriptions.subscribe(singleton(topic1), rebalanceListener); - - client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); - coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); - - // normal join group - Map> memberSubscriptions = singletonMap(consumerId, singletonList(topic2)); - partitionAssignor.prepare(singletonMap(consumerId, Arrays.asList(t2p))); - - client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE)); - client.prepareResponse(new MockClient.RequestMatcher() { - @Override - public boolean matches(AbstractRequest body) { - SyncGroupRequest sync = (SyncGroupRequest) body; - return sync.data.memberId().equals(consumerId) && - sync.data.generationId() == 1 && - sync.groupAssignments().containsKey(consumerId); - } - }, syncGroupResponse(Collections.singletonList(t2p), Errors.NONE)); - assertThrows(IllegalStateException.class, () -> coordinator.poll(time.timer(Long.MAX_VALUE))); - } - @Test public void testPatternJoinGroupLeader() { final String consumerId = "leader"; @@ -1186,10 +1160,10 @@ public void onPartitionsAssigned(Collection partitions) { coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); // prepare initial rebalance - partitionAssignor.prepare(singletonMap(consumerId, Collections.singletonList(t1p))); + partitionAssignor.prepare(singletonMap(consumerId, Collections.singletonList(partition))); client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE)); - client.prepareResponse(syncGroupResponse(Collections.singletonList(t1p), Errors.NONE)); + client.prepareResponse(syncGroupResponse(Collections.singletonList(partition), Errors.NONE)); // The first call to poll should raise the exception from the rebalance listener try { @@ -2425,7 +2399,7 @@ private JoinGroupResponse joinGroupLeaderResponse(int generationId, .setProtocolName(partitionAssignor.name()) .setLeader(memberId) .setMemberId(memberId) - .setMembers(Collections.emptyList()) + .setMembers(metadata) ); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java index a78f966ffc8a1..27c68deb6e584 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java @@ -16,7 +16,6 @@ */ package org.apache.kafka.clients.consumer.internals; -import org.apache.kafka.clients.consumer.internals.ConsumerProtocol.Errors; import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Assignment; import org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription; import org.apache.kafka.common.TopicPartition; From f67942fbb87a3af9312848232956e192d929a11c Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 18 Jun 2019 14:55:53 -0700 Subject: [PATCH 28/59] github comments --- .../internals/ConsumerCoordinator.java | 33 ++++++++++++++----- .../consumer/internals/PartitionAssignor.java | 10 +++--- .../internals/ConsumerProtocolTest.java | 10 +++--- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 6891ae02be884..8b28cb34bfc85 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -548,7 +548,11 @@ private void adjustAssignment(final Map ownedPartitions, assignedPartitions.addAll(assignment.partitions()); // update the assignment if the partition is owned by another different owner - if (assignment.partitions().removeIf(tp -> ownedPartitions.containsKey(tp) && !entry.getKey().equals(ownedPartitions.get(tp)))) { + List updatedPartitions = assignment.partitions().stream() + .filter(tp -> ownedPartitions.containsKey(tp) && !entry.getKey().equals(ownedPartitions.get(tp))) + .collect(Collectors.toList()); + if (!updatedPartitions.equals(assignment.partitions())) { + assignment.updatePartitions(updatedPartitions); revocationsNeeded = true; } } @@ -577,14 +581,25 @@ protected void onJoinPrepare(int generation, String memberId) { // execute the user's callback before rebalance ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); - Set revoked = subscriptions.assignedPartitions(); - log.info("Revoking previously assigned partitions {}", revoked); - try { - listener.onPartitionsRevoked(revoked); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e); + switch (protocol) { + case EAGER: + Set revokedPartitions = new HashSet<>(subscriptions.assignedPartitions()); + log.info("Revoking previously assigned partitions {}", revokedPartitions); + try { + listener.onPartitionsRevoked(revokedPartitions); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e); + } + + // also clear the assigned partitions since all have been revoked + subscriptions.assignFromSubscribed(Collections.emptySet()); + + break; + + case COOPERATIVE: + break; } isLeader = false; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java index b8b6247517fba..c26f68462ea4b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java @@ -202,7 +202,7 @@ public String toString() { class Assignment { private final Short version; - private final List partitions; + private List partitions; private final ByteBuffer userData; private ConsumerProtocol.AssignmentError error; @@ -223,10 +223,6 @@ class Assignment { this(version, partitions, userData, ConsumerProtocol.AssignmentError.NONE); } - public Assignment(List partitions, ByteBuffer userData, ConsumerProtocol.AssignmentError error) { - this(CONSUMER_PROTOCOL_V1, partitions, userData, error); - } - public Assignment(List partitions, ByteBuffer userData) { this(CONSUMER_PROTOCOL_V1, partitions, userData); } @@ -247,6 +243,10 @@ public ConsumerProtocol.AssignmentError error() { return error; } + public void updatePartitions(List partitions) { + this.partitions = partitions; + } + public void setError(ConsumerProtocol.AssignmentError error) { this.error = error; } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java index 27c68deb6e584..9e601b034e529 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocolTest.java @@ -167,20 +167,20 @@ public void deserializeOldAssignmentVersion() { Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer); assertEquals(toSet(partitions), toSet(parsedAssignment.partitions())); assertNull(parsedAssignment.userData()); - assertEquals(Errors.NONE, parsedAssignment.error()); + assertEquals(ConsumerProtocol.AssignmentError.NONE, parsedAssignment.error()); } @Test public void deserializeNewAssignmentWithOldVersion() { List partitions = Collections.singletonList(tp1); - ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment((short) 1, partitions, null, Errors.NEED_REJOIN)); + ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment((short) 1, partitions, null, ConsumerProtocol.AssignmentError.NEED_REJOIN)); // ignore the version assuming it is the old byte code, as it will blindly deserialize as 0 Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer); header.getShort(VERSION_KEY_NAME); Assignment parsedAssignment = ConsumerProtocol.deserializeAssignmentV0(buffer); assertEquals(toSet(partitions), toSet(parsedAssignment.partitions())); assertNull(parsedAssignment.userData()); - assertEquals(Errors.NONE, parsedAssignment.error()); + assertEquals(ConsumerProtocol.AssignmentError.NONE, parsedAssignment.error()); } @Test @@ -200,7 +200,7 @@ public void deserializeFutureAssignmentVersion() { .set(ConsumerProtocol.TOPIC_KEY_NAME, tp1.topic()) .set(ConsumerProtocol.PARTITIONS_KEY_NAME, new Object[]{tp1.partition()})}); assignmentV100.set(USER_DATA_KEY_NAME, ByteBuffer.wrap(new byte[0])); - assignmentV100.set(ERROR_CODE.name, Errors.NEED_REJOIN.code()); + assignmentV100.set(ERROR_CODE.name, ConsumerProtocol.AssignmentError.NEED_REJOIN.code()); assignmentV100.set("foo", "bar"); Struct headerV100 = new Struct(CONSUMER_PROTOCOL_HEADER_SCHEMA); @@ -214,6 +214,6 @@ public void deserializeFutureAssignmentVersion() { PartitionAssignor.Assignment assignment = ConsumerProtocol.deserializeAssignment(buffer); assertEquals(toSet(Collections.singletonList(tp1)), toSet(assignment.partitions())); - assertEquals(Errors.NEED_REJOIN, assignment.error()); + assertEquals(ConsumerProtocol.AssignmentError.NEED_REJOIN, assignment.error()); } } From 11b1915f1c6d745cf6fe616c147b59947c6feafc Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 18 Jun 2019 17:24:16 -0700 Subject: [PATCH 29/59] set protocol with assignors configured only --- .../internals/ConsumerCoordinator.java | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 8b28cb34bfc85..44108f4723672 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -167,21 +167,25 @@ public ConsumerCoordinator(GroupRebalanceConfig rebalanceConfig, // 2. if there are multiple protocols that are commonly supported, select the one with the highest id (i.e. the // id number indicates how advanced the protocol is). // we know there are at least one assignor in the list, no need to double check for NPE - List supportedProtocols = new ArrayList<>(assignors.get(0).supportedProtocols()); + if (!assignors.isEmpty()) { + List supportedProtocols = new ArrayList<>(assignors.get(0).supportedProtocols()); - for (PartitionAssignor assignor : assignors) { - supportedProtocols.retainAll(assignor.supportedProtocols()); - } + for (PartitionAssignor assignor : assignors) { + supportedProtocols.retainAll(assignor.supportedProtocols()); + } - if (supportedProtocols.isEmpty()) { - throw new IllegalArgumentException("Specified assignors " + - assignors.stream().map(PartitionAssignor::name).collect(Collectors.toSet()) + - " do not have commonly supported rebalance protocol"); - } + if (supportedProtocols.isEmpty()) { + throw new IllegalArgumentException("Specified assignors " + + assignors.stream().map(PartitionAssignor::name).collect(Collectors.toSet()) + + " do not have commonly supported rebalance protocol"); + } - Collections.sort(supportedProtocols); + Collections.sort(supportedProtocols); - protocol = supportedProtocols.get(supportedProtocols.size() - 1); + protocol = supportedProtocols.get(supportedProtocols.size() - 1); + } else { + protocol = null; + } this.metadata.requestUpdate(); } From 8440e7defc7a3b3c0fa6c378969b33035ad1a21d Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 18 Jun 2019 17:38:19 -0700 Subject: [PATCH 30/59] maintain consistent behavior --- .../kafka/clients/consumer/internals/ConsumerCoordinator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 44108f4723672..48d0c95bbdba7 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -385,6 +385,10 @@ public boolean poll(Timer timer) { invokeCompletedOffsetCommitCallbacks(); if (subscriptions.partitionsAutoAssigned()) { + if (protocol == null) { + throw new IllegalStateException("User confingure ConsumerConfig#PARTITION_ASSIGNMENT_STRATEGY_CONFIG to empty " + + "while trying to subscribe for group protocol to auto assign partitions"); + } // Always update the heartbeat last poll time so that the heartbeat thread does not leave the // group proactively due to application inactivity even if (say) the coordinator cannot be found. pollHeartbeat(timer.currentTimeMs()); From 091062e5c14872c5fb760c3276e3877e48f744d2 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Tue, 18 Jun 2019 18:26:44 -0700 Subject: [PATCH 31/59] refactor the callback logic --- .../consumer/ConsumerRebalanceListener.java | 2 +- .../internals/ConsumerCoordinator.java | 178 ++++++++++-------- 2 files changed, 97 insertions(+), 83 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java index 508cdf30bbc2e..bfb33b036c2c9 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java @@ -125,7 +125,7 @@ public interface ConsumerRebalanceListener { /** * A callback method the user can implement to provide handling of cleaning up resources for partitions that have already - * been re-assigned to other consumers. This method will be called during normal execution as the owned partitions would + * been re-assigned to other consumers. This method will not be called during normal execution as the owned partitions would * first be revoked by calling the {@link ConsumerRebalanceListener#onPartitionsRevoked} first, before being re-assigned * to other consumers. However, when the consumer is being kicked out of the group and hence were not aware when the new * group is formed without itself, this function will then be called when the consumer finally be notified about the diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 4d6911564dd2d..9fc90e29d1317 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -70,6 +70,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; import java.util.stream.Collectors; /** @@ -234,7 +235,7 @@ private void maybeUpdateJoinedSubscription(Set assignedPartition // into the subscriptions as long as they still match the subscribed pattern Set addedTopics = new HashSet<>(); - //this is a copy because its handed to listener below + // this is a copy because its handed to listener below for (TopicPartition tp : assignedPartitions) { if (!joinedSubscription.contains(tp.topic())) addedTopics.add(tp.topic()); @@ -252,6 +253,69 @@ private void maybeUpdateJoinedSubscription(Set assignedPartition } } + private void maybeAssignPartitions(final Set assignedPartitions) { + log.info("Setting newly assigned partitions: {}", Utils.join(assignedPartitions, ", ")); + + ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); + try { + listener.onPartitionsAssigned(assignedPartitions); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); + } + } + + private void maybeLosePartitions(final String rootCause, final Predicate predicate) { + Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions()); + + if (subscriptions.partitionsAutoAssigned() && !ownedPartitions.isEmpty()) { + Set lostPartitions = metadataSnapshot == null ? ownedPartitions : + ownedPartitions.stream().filter(predicate).collect(Collectors.toSet()); + + log.info("Lost previously assigned partitions {} due to {}", lostPartitions, rootCause); + + ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); + try { + listener.onPartitionsLost(lostPartitions); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition being lost", listener.getClass().getName(), e); + } + + Set leftPartitions = new HashSet<>(ownedPartitions); + leftPartitions.removeAll(lostPartitions); + + subscriptions.assignFromSubscribed(leftPartitions); + } + } + + private void maybeRevokePartitions(final Predicate predicate) { + Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions()); + + if (subscriptions.partitionsAutoAssigned() && !ownedPartitions.isEmpty()) { + Set revokedPartitions = metadataSnapshot == null ? ownedPartitions : + ownedPartitions.stream().filter(predicate).collect(Collectors.toSet()); + + log.info("Revoke previously assigned partitions {}", revokedPartitions); + + ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); + try { + listener.onPartitionsRevoked(revokedPartitions); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition being revocation", listener.getClass().getName(), e); + } + + Set leftPartitions = new HashSet<>(ownedPartitions); + leftPartitions.removeAll(revokedPartitions); + + subscriptions.assignFromSubscribed(leftPartitions); + } + } + @Override protected void onJoinComplete(int generation, String memberId, @@ -292,8 +356,6 @@ protected void onJoinComplete(int generation, this.nextAutoCommitTimer.updateAndReset(autoCommitIntervalMs); // execute the user's callback after rebalance - ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); - switch (protocol) { case EAGER: if (!ownedPartitions.isEmpty()) { @@ -301,19 +363,29 @@ protected void onJoinComplete(int generation, "it is likely client is woken up before a previous pending rebalance completes its callback", ownedPartitions, protocol); } - log.info("Setting newly assigned partitions: {}", Utils.join(assignedPartitions, ", ")); - try { - listener.onPartitionsAssigned(assignedPartitions); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); - } + maybeAssignPartitions(assignedPartitions); break; case COOPERATIVE: - assignAndRevoke(listener, assignedPartitions, ownedPartitions); + Set addedPartitions = new HashSet<>(assignedPartitions); + Set revokedPartitions = new HashSet<>(ownedPartitions); + addedPartitions.removeAll(ownedPartitions); + revokedPartitions.removeAll(assignedPartitions); + + log.info("Updating with newly assigned partitions: {}, compare with already owned partitions: {}, " + + "newly added partitions: {}, revoking partitions: {}", + Utils.join(assignedPartitions, ", "), + Utils.join(ownedPartitions, ", "), + Utils.join(addedPartitions, ", "), + Utils.join(revokedPartitions, ", ")); + + // add partitions that was not previously owned but are now assigned + maybeAssignPartitions(addedPartitions); + + // revoked partitions that was previously owned but no longer assigned + maybeRevokePartitions(tp -> !assignedPartitions.contains(tp)); + // request re-join based on leader communicated error code if (assignment.error() == ConsumerProtocol.AssignmentError.NEED_REJOIN) { requestRejoin(); } @@ -323,38 +395,6 @@ protected void onJoinComplete(int generation, } - private void assignAndRevoke(final ConsumerRebalanceListener listener, - final Set assignedPartitions, - final Set ownedPartitions) { - Set addedPartitions = new HashSet<>(assignedPartitions); - Set revokedPartitions = new HashSet<>(ownedPartitions); - addedPartitions.removeAll(ownedPartitions); - revokedPartitions.removeAll(assignedPartitions); - - log.info("Updating with newly assigned partitions: {}, compare with already owned partitions: {}, " + - "newly added partitions: {}, revoking partitions: {}", - Utils.join(assignedPartitions, ", "), - Utils.join(ownedPartitions, ", "), - Utils.join(addedPartitions, ", "), - Utils.join(revokedPartitions, ", ")); - - try { - listener.onPartitionsAssigned(addedPartitions); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); - } - - try { - listener.onPartitionsRevoked(revokedPartitions); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e); - } - } - void maybeUpdateSubscriptionMetadata() { int version = metadata.updateVersion(); if (version > metadataSnapshot.version) { @@ -587,26 +627,15 @@ protected void onJoinPrepare(int generation, String memberId) { maybeAutoCommitOffsetsSync(time.timer(rebalanceConfig.rebalanceTimeoutMs)); // execute the user's callback before rebalance - ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); - switch (protocol) { case EAGER: - Set revokedPartitions = new HashSet<>(subscriptions.assignedPartitions()); - log.info("Revoking previously assigned partitions {}", revokedPartitions); - try { - listener.onPartitionsRevoked(revokedPartitions); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition revocation", listener.getClass().getName(), e); - } - - // also clear the assigned partitions since all have been revoked - subscriptions.assignFromSubscribed(Collections.emptySet()); - + // revoke all partitions + maybeRevokePartitions(tp -> true); break; case COOPERATIVE: + // only revoke those partitions that are not in the subscription any more. + maybeRevokePartitions(tp -> !subscriptions.subscription().contains(tp.topic())); break; } @@ -616,7 +645,8 @@ protected void onJoinPrepare(int generation, String memberId) { @Override public void resetGeneration(String errorMessage) { - maybeLosePartitions(errorMessage); + // revoke all partitions + maybeLosePartitions(errorMessage, tp -> true); super.resetGeneration(errorMessage); } @@ -627,39 +657,19 @@ public boolean rejoinNeededOrPending() { // we need to rejoin if we performed the assignment and metadata has changed if (assignmentSnapshot != null && !assignmentSnapshot.matches(metadataSnapshot)) { - maybeLosePartitions("topic metadata has changed and therefore some topics may not exist any moreq"); + maybeLosePartitions("topic metadata has changed and therefore some topics may not exist any more", + tp -> !metadataSnapshot.partitionsPerTopic().containsKey(tp.topic())); return true; } // we need to join if our subscription has changed since the last join if (joinedSubscription != null && !joinedSubscription.equals(subscriptions.subscription())) { - maybeLosePartitions("consumer subscription has changed since last joined generation"); return true; } return super.rejoinNeededOrPending(); } - private void maybeLosePartitions(String rootCause) { - Set lostPartitions = new HashSet<>(subscriptions.assignedPartitions()); - - if (subscriptions.partitionsAutoAssigned() && !lostPartitions.isEmpty()) { - log.info("Lost previously assigned partitions {} due to {}", lostPartitions, rootCause); - - ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); - try { - listener.onPartitionsLost(lostPartitions); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition being lost", listener.getClass().getName(), e); - } - - // also clear the assigned partitions since all have been lost - subscriptions.assignFromSubscribed(Collections.emptySet()); - } - } - /** * Refresh the committed offsets for provided partitions. * @@ -1203,6 +1213,10 @@ private MetadataSnapshot(SubscriptionState subscription, Cluster cluster, int ve boolean matches(MetadataSnapshot other) { return version == other.version || partitionsPerTopic.equals(other.partitionsPerTopic); } + + Map partitionsPerTopic() { + return partitionsPerTopic; + } } private static class OffsetCommitCompletion { From c95e1765158b3e0314deec0589674cb485273e50 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Wed, 19 Jun 2019 10:20:19 -0700 Subject: [PATCH 32/59] change behavior of listener callback --- .../internals/ConsumerCoordinator.java | 66 +++++++++---------- .../internals/ConsumerCoordinatorTest.java | 23 ++++--- 2 files changed, 47 insertions(+), 42 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 9fc90e29d1317..51fe183849ec4 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -266,53 +266,53 @@ private void maybeAssignPartitions(final Set assignedPartitions) } } - private void maybeLosePartitions(final String rootCause, final Predicate predicate) { + private void maybeRevokePartitions(final Predicate predicate) { Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions()); - if (subscriptions.partitionsAutoAssigned() && !ownedPartitions.isEmpty()) { - Set lostPartitions = metadataSnapshot == null ? ownedPartitions : - ownedPartitions.stream().filter(predicate).collect(Collectors.toSet()); + Set revokedPartitions = metadataSnapshot == null ? ownedPartitions : + ownedPartitions.stream().filter(predicate).collect(Collectors.toSet()); - log.info("Lost previously assigned partitions {} due to {}", lostPartitions, rootCause); + log.info("Revoke previously assigned partitions {}", revokedPartitions); - ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); - try { - listener.onPartitionsLost(lostPartitions); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition being lost", listener.getClass().getName(), e); - } + ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); + try { + listener.onPartitionsRevoked(revokedPartitions); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition being revocation", listener.getClass().getName(), e); + } - Set leftPartitions = new HashSet<>(ownedPartitions); - leftPartitions.removeAll(lostPartitions); + Set leftPartitions = new HashSet<>(ownedPartitions); + leftPartitions.removeAll(revokedPartitions); - subscriptions.assignFromSubscribed(leftPartitions); - } + subscriptions.assignFromSubscribed(leftPartitions); } - private void maybeRevokePartitions(final Predicate predicate) { - Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions()); + private void maybeLosePartitions(final String rootCause, final Predicate predicate) { + if (subscriptions.partitionsAutoAssigned()) { + Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions()); - if (subscriptions.partitionsAutoAssigned() && !ownedPartitions.isEmpty()) { - Set revokedPartitions = metadataSnapshot == null ? ownedPartitions : + Set lostPartitions = metadataSnapshot == null ? ownedPartitions : ownedPartitions.stream().filter(predicate).collect(Collectors.toSet()); - log.info("Revoke previously assigned partitions {}", revokedPartitions); + if (!lostPartitions.isEmpty()) { + log.info("Lost previously assigned partitions {} due to {}", lostPartitions, rootCause); - ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); - try { - listener.onPartitionsRevoked(revokedPartitions); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition being revocation", listener.getClass().getName(), e); - } + ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); + try { + listener.onPartitionsLost(lostPartitions); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition being lost", listener.getClass().getName(), e); + } - Set leftPartitions = new HashSet<>(ownedPartitions); - leftPartitions.removeAll(revokedPartitions); + Set leftPartitions = new HashSet<>(ownedPartitions); + leftPartitions.removeAll(lostPartitions); - subscriptions.assignFromSubscribed(leftPartitions); + subscriptions.assignFromSubscribed(leftPartitions); + } } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index 98659a20b3420..273c884367f1c 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -549,14 +549,9 @@ public boolean matches(AbstractRequest body) { final Collection revoked = getRevoked(owned, newAssignment); final Collection assigned = getAdded(owned, newAssignment); - int revokeCount = 1; + final int revokeCount = 2; final int addCount = 1; - // with eager protocol we will call revoke on the old assignment as well - if (protocol == PartitionAssignor.RebalanceProtocol.EAGER) { - revokeCount += 1; - } - assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(toSet(newAssignment), subscriptions.assignedPartitions()); assertEquals(toSet(newSubscription), subscriptions.groupSubscription()); @@ -600,11 +595,13 @@ public boolean matches(AbstractRequest body) { coordinator.poll(time.timer(Long.MAX_VALUE)); + final int revokeCount = protocol == PartitionAssignor.RebalanceProtocol.EAGER ? 1 : 2; + assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(2, subscriptions.numAssignedPartitions()); assertEquals(2, subscriptions.groupSubscription().size()); assertEquals(2, subscriptions.subscription().size()); - assertEquals(1, rebalanceListener.revokedCount); + assertEquals(revokeCount, rebalanceListener.revokedCount); assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); @@ -1313,9 +1310,11 @@ public void testDisconnectInJoin() { client.prepareResponse(syncGroupResponse(assigned, Errors.NONE)); coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE)); + final int revokeCount = protocol == PartitionAssignor.RebalanceProtocol.EAGER ? 1 : 2; + assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(toSet(assigned), subscriptions.assignedPartitions()); - assertEquals(1, rebalanceListener.revokedCount); + assertEquals(revokeCount, rebalanceListener.revokedCount); assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); @@ -2539,12 +2538,13 @@ public void onComplete(Map offsets, Exception } private static class MockRebalanceListener implements ConsumerRebalanceListener { + public Collection lost; public Collection revoked; public Collection assigned; + public int lostCount = 0; public int revokedCount = 0; public int assignedCount = 0; - @Override public void onPartitionsAssigned(Collection partitions) { this.assigned = partitions; @@ -2557,5 +2557,10 @@ public void onPartitionsRevoked(Collection partitions) { revokedCount++; } + @Override + public void onPartitionsLost(Collection partitions) { + this.lost = partitions; + lostCount++; + } } } From d112c5cf59a6f52d61bcbd3dfbf6d5bf2399d795 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Fri, 21 Jun 2019 09:56:26 -0700 Subject: [PATCH 33/59] update unit tests --- .../internals/ConsumerCoordinator.java | 49 +++++++------- .../internals/ConsumerCoordinatorTest.java | 66 ++++++++++--------- 2 files changed, 62 insertions(+), 53 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 51fe183849ec4..5be2bbae676f3 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -254,39 +254,42 @@ private void maybeUpdateJoinedSubscription(Set assignedPartition } private void maybeAssignPartitions(final Set assignedPartitions) { - log.info("Setting newly assigned partitions: {}", Utils.join(assignedPartitions, ", ")); + if (!assignedPartitions.isEmpty()) { + log.info("Setting newly assigned partitions: {}", Utils.join(assignedPartitions, ", ")); - ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); - try { - listener.onPartitionsAssigned(assignedPartitions); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); + ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); + try { + listener.onPartitionsAssigned(assignedPartitions); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); + } } } private void maybeRevokePartitions(final Predicate predicate) { Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions()); - Set revokedPartitions = metadataSnapshot == null ? ownedPartitions : - ownedPartitions.stream().filter(predicate).collect(Collectors.toSet()); + Set revokedPartitions = ownedPartitions.stream().filter(predicate).collect(Collectors.toSet()); - log.info("Revoke previously assigned partitions {}", revokedPartitions); + if (!revokedPartitions.isEmpty()) { + log.info("Revoke previously assigned partitions {}", revokedPartitions); - ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); - try { - listener.onPartitionsRevoked(revokedPartitions); - } catch (WakeupException | InterruptException e) { - throw e; - } catch (Exception e) { - log.error("User provided listener {} failed on partition being revocation", listener.getClass().getName(), e); - } + ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); + try { + listener.onPartitionsRevoked(revokedPartitions); + } catch (WakeupException | InterruptException e) { + throw e; + } catch (Exception e) { + log.error("User provided listener {} failed on partition being revocation", listener.getClass().getName(), e); + } - Set leftPartitions = new HashSet<>(ownedPartitions); - leftPartitions.removeAll(revokedPartitions); + Set leftPartitions = new HashSet<>(ownedPartitions); + leftPartitions.removeAll(revokedPartitions); - subscriptions.assignFromSubscribed(leftPartitions); + subscriptions.assignFromSubscribed(leftPartitions); + } } private void maybeLosePartitions(final String rootCause, final Predicate predicate) { @@ -363,6 +366,8 @@ protected void onJoinComplete(int generation, "it is likely client is woken up before a previous pending rebalance completes its callback", ownedPartitions, protocol); } + // TODO: after refactored the error handling, we need to exclude owned partitions from assigned partitions + maybeAssignPartitions(assignedPartitions); break; diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index 273c884367f1c..4b6fc935f428b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -486,8 +486,8 @@ public boolean matches(AbstractRequest body) { assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(toSet(assigned), subscriptions.assignedPartitions()); assertEquals(subscription, subscriptions.groupSubscription()); - assertEquals(1, rebalanceListener.revokedCount); - assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); + assertEquals(0, rebalanceListener.revokedCount); + assertNull(rebalanceListener.revoked); assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @@ -549,14 +549,13 @@ public boolean matches(AbstractRequest body) { final Collection revoked = getRevoked(owned, newAssignment); final Collection assigned = getAdded(owned, newAssignment); - final int revokeCount = 2; final int addCount = 1; assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(toSet(newAssignment), subscriptions.assignedPartitions()); assertEquals(toSet(newSubscription), subscriptions.groupSubscription()); - assertEquals(revokeCount, rebalanceListener.revokedCount); - assertEquals(revoked, rebalanceListener.revoked); + assertEquals(0, rebalanceListener.revokedCount); + assertNull(rebalanceListener.revoked); assertEquals(addCount, rebalanceListener.assignedCount); assertEquals(assigned, rebalanceListener.assigned); } @@ -595,14 +594,13 @@ public boolean matches(AbstractRequest body) { coordinator.poll(time.timer(Long.MAX_VALUE)); - final int revokeCount = protocol == PartitionAssignor.RebalanceProtocol.EAGER ? 1 : 2; - assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(2, subscriptions.numAssignedPartitions()); assertEquals(2, subscriptions.groupSubscription().size()); assertEquals(2, subscriptions.subscription().size()); - assertEquals(revokeCount, rebalanceListener.revokedCount); - assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); + // callback not triggered at all since there's nothing to be revoked + assertEquals(0, rebalanceListener.revokedCount); + assertNull(rebalanceListener.revoked); assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @@ -613,7 +611,6 @@ public void testMetadataRefreshDuringRebalance() { final List owned = Collections.emptyList(); final List oldAssigned = Arrays.asList(t1p); - subscriptions.subscribe(Pattern.compile(".*"), rebalanceListener); client.updateMetadata(TestUtils.metadataUpdateWith(1, singletonMap(topic1, 1))); coordinator.maybeUpdateSubscriptionMetadata(); @@ -646,8 +643,9 @@ public boolean matches(AbstractRequest body) { assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(singleton(topic1), subscriptions.subscription()); assertEquals(toSet(oldAssigned), subscriptions.assignedPartitions()); - assertEquals(1, rebalanceListener.revokedCount); - assertEquals(getRevoked(owned, oldAssigned), rebalanceListener.revoked); + // nothing to be revoked and hence no callback triggered + assertEquals(0, rebalanceListener.revokedCount); + assertNull(rebalanceListener.revoked); assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, oldAssigned), rebalanceListener.assigned); @@ -676,11 +674,13 @@ public boolean matches(AbstractRequest body) { coordinator.poll(time.timer(Long.MAX_VALUE)); + Collection revoked = getRevoked(oldAssigned, newAssigned); + assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(toSet(updatedSubscription), subscriptions.subscription()); assertEquals(toSet(newAssigned), subscriptions.assignedPartitions()); - assertEquals(2, rebalanceListener.revokedCount); - assertEquals(getRevoked(oldAssigned, newAssigned), rebalanceListener.revoked); + assertEquals(revoked.isEmpty() ? 0 : 1, rebalanceListener.revokedCount); + assertEquals(revoked.isEmpty() ? null : revoked, rebalanceListener.revoked); assertEquals(2, rebalanceListener.assignedCount); assertEquals(getAdded(oldAssigned, newAssigned), rebalanceListener.assigned); } @@ -762,8 +762,8 @@ public void testWakeupDuringJoin() { assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(toSet(assigned), subscriptions.assignedPartitions()); - assertEquals(1, rebalanceListener.revokedCount); - assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); + assertEquals(0, rebalanceListener.revokedCount); + assertNull(rebalanceListener.revoked); assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @@ -797,8 +797,8 @@ public boolean matches(AbstractRequest body) { assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(toSet(assigned), subscriptions.assignedPartitions()); assertEquals(subscription, subscriptions.groupSubscription()); - assertEquals(1, rebalanceListener.revokedCount); - assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); + assertEquals(0, rebalanceListener.revokedCount); + assertNull(rebalanceListener.revoked); assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @@ -867,8 +867,8 @@ public boolean matches(AbstractRequest body) { assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(assigned.size(), subscriptions.numAssignedPartitions()); assertEquals(subscription, subscriptions.subscription()); - assertEquals(1, rebalanceListener.revokedCount); - assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); + assertEquals(0, rebalanceListener.revokedCount); + assertNull(rebalanceListener.revoked); assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } @@ -1173,8 +1173,8 @@ public void onPartitionsAssigned(Collection partitions) { coordinator.poll(time.timer(Long.MAX_VALUE)); assertFalse(coordinator.rejoinNeededOrPending()); - assertEquals(1, rebalanceListener.revokedCount); - assertEquals(2, rebalanceListener.assignedCount); + assertEquals(0, rebalanceListener.revokedCount); + assertEquals(protocol == PartitionAssignor.RebalanceProtocol.EAGER ? 2 : 1, rebalanceListener.assignedCount); } @Test @@ -1213,7 +1213,8 @@ private void unavailableTopicTest(boolean patternSubscribe, Set unavaila client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.NONE)); coordinator.poll(time.timer(Long.MAX_VALUE)); assertFalse(coordinator.rejoinNeededOrPending()); - assertEquals(Collections.emptySet(), rebalanceListener.assigned); + // callback not triggered since there's nothing to be assigned + assertNull(rebalanceListener.assigned); assertTrue("Metadata refresh not requested for unavailable partitions", metadata.updateRequested()); Map topicErrors = new HashMap<>(); @@ -1275,8 +1276,8 @@ public void testRejoinGroup() { // join the group once joinAsFollowerAndReceiveAssignment("consumer", coordinator, assigned); - assertEquals(1, rebalanceListener.revokedCount); - assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); + assertEquals(0, rebalanceListener.revokedCount); + assertNull(rebalanceListener.revoked); assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); @@ -1288,10 +1289,12 @@ public void testRejoinGroup() { client.prepareResponse(syncGroupResponse(assigned, Errors.NONE)); coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE)); - assertEquals(2, rebalanceListener.revokedCount); - assertEquals(getRevoked(assigned, assigned), rebalanceListener.revoked); - assertEquals(2, rebalanceListener.assignedCount); - assertEquals(getAdded(assigned, assigned), rebalanceListener.assigned); + Collection revoked = getRevoked(assigned, assigned); + Collection added = getAdded(assigned, assigned); + assertEquals(revoked.isEmpty() ? 0 : 1, rebalanceListener.revokedCount); + assertEquals(revoked.isEmpty() ? null : revoked, rebalanceListener.revoked); + assertEquals(added.isEmpty() ? 1 : 2, rebalanceListener.assignedCount); + assertEquals(added.isEmpty() ? null : added, rebalanceListener.assigned); } @Test @@ -1314,8 +1317,9 @@ public void testDisconnectInJoin() { assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(toSet(assigned), subscriptions.assignedPartitions()); - assertEquals(revokeCount, rebalanceListener.revokedCount); - assertEquals(getRevoked(owned, assigned), rebalanceListener.revoked); + // nothing to be revoked hence callback not triggered + assertEquals(0, rebalanceListener.revokedCount); + assertNull(rebalanceListener.revoked); assertEquals(1, rebalanceListener.assignedCount); assertEquals(getAdded(owned, assigned), rebalanceListener.assigned); } From 9ceb83f1befd4878209e80e5a4e78ba7498d75d7 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 24 Jun 2019 15:25:10 -0700 Subject: [PATCH 34/59] throw exception to the KafkaConsumer.poll caller --- .../kafka/clients/consumer/KafkaConsumer.java | 3 +- .../internals/ConsumerCoordinator.java | 75 +++++++++++++------ .../internals/ConsumerCoordinatorTest.java | 27 +++---- 3 files changed, 65 insertions(+), 40 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index fa5cc99a18879..970cfd74e7602 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -1176,7 +1176,8 @@ public ConsumerRecords poll(final long timeoutMs) { * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed * topics or to the configured groupId. See the exception for more details * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or - * session timeout, errors deserializing key/value pairs, or any new error cases in future versions) + * session timeout, errors deserializing key/value pairs, user's rebalance callback thrown exceptions, + * or any new error cases in future versions) * @throws java.lang.IllegalArgumentException if the timeout value is negative * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any * partitions to consume from diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index 5be2bbae676f3..1be91d78dbdc1 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -70,6 +70,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -253,7 +254,7 @@ private void maybeUpdateJoinedSubscription(Set assignedPartition } } - private void maybeAssignPartitions(final Set assignedPartitions) { + private Exception maybeAssignPartitions(final Set assignedPartitions) { if (!assignedPartitions.isEmpty()) { log.info("Setting newly assigned partitions: {}", Utils.join(assignedPartitions, ", ")); @@ -263,12 +264,16 @@ private void maybeAssignPartitions(final Set assignedPartitions) } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { - log.error("User provided listener {} failed on partition assignment", listener.getClass().getName(), e); + log.error("User provided listener {} failed on partition assignment of {}", + listener.getClass().getName(), assignedPartitions, e); + return e; } } + + return null; } - private void maybeRevokePartitions(final Predicate predicate) { + private Exception maybeRevokePartitions(final Predicate predicate) { Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions()); Set revokedPartitions = ownedPartitions.stream().filter(predicate).collect(Collectors.toSet()); @@ -276,23 +281,26 @@ private void maybeRevokePartitions(final Predicate predicate) { if (!revokedPartitions.isEmpty()) { log.info("Revoke previously assigned partitions {}", revokedPartitions); + Set leftPartitions = new HashSet<>(ownedPartitions); + leftPartitions.removeAll(revokedPartitions); + subscriptions.assignFromSubscribed(leftPartitions); + ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); try { listener.onPartitionsRevoked(revokedPartitions); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { - log.error("User provided listener {} failed on partition being revocation", listener.getClass().getName(), e); + log.error("User provided listener {} failed on partition being revocation of {}", + listener.getClass().getName(), revokedPartitions, e); + return e; } - - Set leftPartitions = new HashSet<>(ownedPartitions); - leftPartitions.removeAll(revokedPartitions); - - subscriptions.assignFromSubscribed(leftPartitions); } + + return null; } - private void maybeLosePartitions(final String rootCause, final Predicate predicate) { + private Exception maybeLostPartitions(final String rootCause, final Predicate predicate) { if (subscriptions.partitionsAutoAssigned()) { Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions()); @@ -302,21 +310,24 @@ private void maybeLosePartitions(final String rootCause, final Predicate leftPartitions = new HashSet<>(ownedPartitions); + leftPartitions.removeAll(lostPartitions); + subscriptions.assignFromSubscribed(leftPartitions); + ConsumerRebalanceListener listener = subscriptions.rebalanceListener(); try { listener.onPartitionsLost(lostPartitions); } catch (WakeupException | InterruptException e) { throw e; } catch (Exception e) { - log.error("User provided listener {} failed on partition being lost", listener.getClass().getName(), e); + log.error("User provided listener {} failed on partition being lost of {}", + listener.getClass().getName(), lostPartitions, e); + return e; } - - Set leftPartitions = new HashSet<>(ownedPartitions); - leftPartitions.removeAll(lostPartitions); - - subscriptions.assignFromSubscribed(leftPartitions); } } + + return null; } @Override @@ -359,6 +370,7 @@ protected void onJoinComplete(int generation, this.nextAutoCommitTimer.updateAndReset(autoCommitIntervalMs); // execute the user's callback after rebalance + final AtomicReference firstException = new AtomicReference<>(null); switch (protocol) { case EAGER: if (!ownedPartitions.isEmpty()) { @@ -368,7 +380,8 @@ protected void onJoinComplete(int generation, // TODO: after refactored the error handling, we need to exclude owned partitions from assigned partitions - maybeAssignPartitions(assignedPartitions); + firstException.compareAndSet(null, maybeAssignPartitions(assignedPartitions)); + break; case COOPERATIVE: @@ -385,10 +398,10 @@ protected void onJoinComplete(int generation, Utils.join(revokedPartitions, ", ")); // add partitions that was not previously owned but are now assigned - maybeAssignPartitions(addedPartitions); + firstException.compareAndSet(null, maybeAssignPartitions(addedPartitions)); // revoked partitions that was previously owned but no longer assigned - maybeRevokePartitions(tp -> !assignedPartitions.contains(tp)); + firstException.compareAndSet(null, maybeRevokePartitions(tp -> !assignedPartitions.contains(tp))); // request re-join based on leader communicated error code if (assignment.error() == ConsumerProtocol.AssignmentError.NEED_REJOIN) { @@ -398,6 +411,8 @@ protected void onJoinComplete(int generation, break; } + if (firstException.get() != null) + throw new KafkaException("User rebalance callback throws an error", firstException.get()); } void maybeUpdateSubscriptionMetadata() { @@ -632,27 +647,36 @@ protected void onJoinPrepare(int generation, String memberId) { maybeAutoCommitOffsetsSync(time.timer(rebalanceConfig.rebalanceTimeoutMs)); // execute the user's callback before rebalance + final AtomicReference firstException = new AtomicReference<>(null); switch (protocol) { case EAGER: // revoke all partitions - maybeRevokePartitions(tp -> true); + firstException.compareAndSet(null, maybeRevokePartitions(tp -> true)); break; case COOPERATIVE: // only revoke those partitions that are not in the subscription any more. - maybeRevokePartitions(tp -> !subscriptions.subscription().contains(tp.topic())); + firstException.compareAndSet(null, maybeRevokePartitions(tp -> !subscriptions.subscription().contains(tp.topic()))); break; } isLeader = false; subscriptions.resetGroupSubscription(); + + if (firstException.get() != null) { + throw new KafkaException("User rebalance callback throws an error", firstException.get()); + } } @Override public void resetGeneration(String errorMessage) { // revoke all partitions - maybeLosePartitions(errorMessage, tp -> true); + Exception e = maybeLostPartitions(errorMessage, tp -> true); super.resetGeneration(errorMessage); + + if (e != null) { + throw new KafkaException("User rebalance callback throws an error", e); + } } @Override @@ -662,8 +686,13 @@ public boolean rejoinNeededOrPending() { // we need to rejoin if we performed the assignment and metadata has changed if (assignmentSnapshot != null && !assignmentSnapshot.matches(metadataSnapshot)) { - maybeLosePartitions("topic metadata has changed and therefore some topics may not exist any more", + Exception e = maybeLostPartitions("topic metadata has changed and therefore some topics may not exist any more", tp -> !metadataSnapshot.partitionsPerTopic().containsKey(tp.topic())); + + if (e != null) { + throw new KafkaException("User rebalance callback throws an error", e); + } + return true; } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index d14c0587e9745..a0aa1ba09d013 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -246,7 +246,7 @@ public void testGroupReadUnauthorized() { client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); - client.prepareResponse(joinGroupLeaderResponse(0, "memberId", Collections.>emptyMap(), + client.prepareResponse(joinGroupLeaderResponse(0, "memberId", Collections.emptyMap(), Errors.GROUP_AUTHORIZATION_FAILED)); coordinator.poll(time.timer(Long.MAX_VALUE)); } @@ -273,7 +273,7 @@ public void testCoordinatorNotAvailable() { } @Test - public void testManyInFlightAsyncCommitsWithCoordinatorDisconnect() throws Exception { + public void testManyInFlightAsyncCommitsWithCoordinatorDisconnect() { client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); @@ -301,7 +301,7 @@ public void onComplete(Map offsets, Exception } @Test - public void testCoordinatorUnknownInUnsentCallbacksAfterCoordinatorDead() throws Exception { + public void testCoordinatorUnknownInUnsentCallbacksAfterCoordinatorDead() { // When the coordinator is marked dead, all unsent or in-flight requests are cancelled // with a disconnect error. This test case ensures that the corresponding callbacks see // the coordinator as unknown which prevents additional retries to the same coordinator. @@ -447,7 +447,7 @@ public void testJoinGroupInvalidGroupId() { client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); - client.prepareResponse(joinGroupLeaderResponse(0, consumerId, Collections.>emptyMap(), + client.prepareResponse(joinGroupLeaderResponse(0, consumerId, Collections.emptyMap(), Errors.INVALID_GROUP_ID)); coordinator.poll(time.timer(Long.MAX_VALUE)); } @@ -546,17 +546,14 @@ public boolean matches(AbstractRequest body) { coordinator.poll(time.timer(Long.MAX_VALUE)); - final Collection revoked = getRevoked(owned, newAssignment); final Collection assigned = getAdded(owned, newAssignment); - final int addCount = 1; - assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(toSet(newAssignment), subscriptions.assignedPartitions()); assertEquals(toSet(newSubscription), subscriptions.groupSubscription()); assertEquals(0, rebalanceListener.revokedCount); assertNull(rebalanceListener.revoked); - assertEquals(addCount, rebalanceListener.assignedCount); + assertEquals(1, rebalanceListener.assignedCount); assertEquals(assigned, rebalanceListener.assigned); } @@ -979,7 +976,7 @@ public void testUnknownMemberIdOnSyncGroup() { // join initially, but let coordinator returns unknown member id client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE)); - client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.UNKNOWN_MEMBER_ID)); + client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.UNKNOWN_MEMBER_ID)); // now we should see a new join with the empty UNKNOWN_MEMBER_ID client.prepareResponse(new MockClient.RequestMatcher() { @@ -1008,7 +1005,7 @@ public void testRebalanceInProgressOnSyncGroup() { // join initially, but let coordinator rebalance on sync client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE)); - client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.REBALANCE_IN_PROGRESS)); + client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.REBALANCE_IN_PROGRESS)); // then let the full join/sync finish successfully client.prepareResponse(joinGroupFollowerResponse(2, consumerId, "leader", Errors.NONE)); @@ -1031,7 +1028,7 @@ public void testIllegalGenerationOnSyncGroup() { // join initially, but let coordinator rebalance on sync client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE)); - client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.ILLEGAL_GENERATION)); + client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.ILLEGAL_GENERATION)); // then let the full join/sync finish successfully client.prepareResponse(new MockClient.RequestMatcher() { @@ -1313,8 +1310,6 @@ public void testDisconnectInJoin() { client.prepareResponse(syncGroupResponse(assigned, Errors.NONE)); coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE)); - final int revokeCount = protocol == PartitionAssignor.RebalanceProtocol.EAGER ? 1 : 2; - assertFalse(coordinator.rejoinNeededOrPending()); assertEquals(toSet(assigned), subscriptions.assignedPartitions()); // nothing to be revoked hence callback not triggered @@ -1699,7 +1694,7 @@ public void testAsyncCommitCallbacksInvokedPriorToSyncCommitCompletion() throws client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE)); - final List committedOffsets = Collections.synchronizedList(new ArrayList()); + final List committedOffsets = Collections.synchronizedList(new ArrayList<>()); final OffsetAndMetadata firstOffset = new OffsetAndMetadata(0L); final OffsetAndMetadata secondOffset = new OffsetAndMetadata(1L); @@ -1971,7 +1966,7 @@ public void testRefreshOffsetWithNoFetchableOffsets() { assertEquals(Collections.singleton(t1p), subscriptions.missingFetchPositions()); assertEquals(Collections.emptySet(), subscriptions.partitionsNeedingReset(time.milliseconds())); assertFalse(subscriptions.hasAllFetchPositions()); - assertEquals(null, subscriptions.position(t1p)); + assertNull(subscriptions.position(t1p)); } @Test @@ -2019,7 +2014,7 @@ public void testAuthenticationFailureInEnsureActiveGroup() { public void testThreadSafeAssignedPartitionsMetric() throws Exception { // Get the assigned-partitions metric final Metric metric = metrics.metric(new MetricName("assigned-partitions", "consumer" + groupId + "-coordinator-metrics", - "", Collections.emptyMap())); + "", Collections.emptyMap())); // Start polling the metric in the background final AtomicBoolean doStop = new AtomicBoolean(); From dc4ad4db4f4178a8175a8a42de4cab7f6c11c1c2 Mon Sep 17 00:00:00 2001 From: Guozhang Wang Date: Mon, 24 Jun 2019 16:03:26 -0700 Subject: [PATCH 35/59] Update Javadoc --- .../consumer/ConsumerRebalanceListener.java | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java index bfb33b036c2c9..27051b8f4460a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java @@ -47,11 +47,21 @@ * This callback will only execute in the user thread as part of the {@link Consumer#poll(java.time.Duration) poll(long)} call * whenever partition assignment changes. *

- * It is guaranteed that all consumer processes will invoke {@link #onPartitionsRevoked(Collection) onPartitionsRevoked} prior to - * any process invoking {@link #onPartitionsAssigned(Collection) onPartitionsAssigned}. So if offsets or other state is saved in the + * It is guaranteed that for any partition, if a partition is reassigned from one consumer to another, then the old consumer will + * always invoke {@link #onPartitionsRevoked(Collection) onPartitionsRevoked} for that partition prior to the new consumer + * invoking {@link #onPartitionsAssigned(Collection) onPartitionsAssigned} for the same partition. So if offsets or other state is saved in the * {@link #onPartitionsRevoked(Collection) onPartitionsRevoked} call it is guaranteed to be saved by the time the process taking over that * partition has their {@link #onPartitionsAssigned(Collection) onPartitionsAssigned} callback called to load the state. *

+ * In addition, a third {@link #onPartitionsLost(Collection)} callback will be invoked when the partition previously owned by the consumer + * are lost. It is different to the {@link #onPartitionsRevoked(Collection)} function such that the latter is invoked when a partition is revoked + * from the consumer during a rebalance event, while the former is invoked when a partition is lost not during a rebalance event, but due to + * an exceptional event, such as a consumer finding out it is no longer part of the consumer group, or that some of its assigned partitions no longer + * exist due to administrative operations like delete topics. Users then could implement these two functions differently (by default, + * {@link #onPartitionsLost(Collection)} will be calling {@link #onPartitionsRevoked(Collection)} directly); for example, in the + * {@link #onPartitionsLost(Collection)} we would not need to store the offsets since we know these partitions are no longer owned by the consumer + * at that time. + *

* Here is pseudo-code for a callback implementation for saving offsets: * 

  * {@code
@@ -68,6 +78,10 @@
  *              saveOffsetInExternalStore(consumer.position(partition));
  *       }
  *
+ *       public void onPartitionsLost(Collection partitions) {
+ *           // do not need to save the offsets since these partitions are probably owned by other consumers already
+ *       }
+ *
  *       public void onPartitionsAssigned(Collection partitions) {
  *           // read the offsets from an external store using some custom code not described here
  *           for(TopicPartition partition: partitions)
@@ -126,13 +140,16 @@ public interface ConsumerRebalanceListener {
     /**
      * A callback method the user can implement to provide handling of cleaning up resources for partitions that have already
      * been re-assigned to other consumers. This method will not be called during normal execution as the owned partitions would
-     * first be revoked by calling the {@link ConsumerRebalanceListener#onPartitionsRevoked} first, before being re-assigned
-     * to other consumers. However, when the consumer is being kicked out of the group and hence were not aware when the new
-     * group is formed without itself, this function will then be called when the consumer finally be notified about the
-     * partitions that have already been emigrated to other consumers.
+     * first be revoked by calling the {@link ConsumerRebalanceListener#onPartitionsRevoked}, before being re-assigned
+     * to other consumers during a rebalance event. However, during exceptional scenarios when the consumer realized that it
+     * does not own this partition any longer, i.e. not revoked via a normal rebalance event, then this method would be invoked.
+     *
+     * For example, if a consumer is kicked out of the group in the previous generation and hence were not aware when the new
+     * group is formed and partitions assigned to other members; later when it finally be notified about the illegal generation
+     * error and are going to reset its generation and re-join, this function will be called.
      *
      * By default it will just trigger {@link ConsumerRebalanceListener#onPartitionsRevoked}; for advanced users who want to distinguish
-     * the handling logic of revoked partitions v.s. emigrated partitions, they can override the default implementation.
+     * the handling logic of revoked partitions v.s. lost partitions, they can override the default implementation.
      *
      * It is possible
      * for a {@link org.apache.kafka.common.errors.WakeupException} or {@link org.apache.kafka.common.errors.InterruptException}

From 3b0b667394162bca5a1405e80ebf194d7591c5c1 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Wed, 26 Jun 2019 17:10:31 -0700
Subject: [PATCH 36/59] refactor the caller to always change the owned
 partitions first

---
 .../internals/ConsumerCoordinator.java        | 101 ++++++++++--------
 1 file changed, 54 insertions(+), 47 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 1be91d78dbdc1..9771867bdb671 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -71,7 +71,6 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 /**
@@ -273,18 +272,10 @@ private Exception maybeAssignPartitions(final Set assignedPartit
         return null;
     }
 
-    private Exception maybeRevokePartitions(final Predicate predicate) {
-        Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions());
-
-        Set revokedPartitions = ownedPartitions.stream().filter(predicate).collect(Collectors.toSet());
-
+    private Exception maybeRevokePartitions(final Set revokedPartitions) {
         if (!revokedPartitions.isEmpty()) {
             log.info("Revoke previously assigned partitions {}", revokedPartitions);
 
-            Set leftPartitions = new HashSet<>(ownedPartitions);
-            leftPartitions.removeAll(revokedPartitions);
-            subscriptions.assignFromSubscribed(leftPartitions);
-
             ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
             try {
                 listener.onPartitionsRevoked(revokedPartitions);
@@ -300,30 +291,19 @@ private Exception maybeRevokePartitions(final Predicate predicat
         return null;
     }
 
-    private Exception maybeLostPartitions(final String rootCause, final Predicate predicate) {
-        if (subscriptions.partitionsAutoAssigned()) {
-            Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions());
+    private Exception maybeLostPartitions(final String rootCause, final Set lostPartitions) {
+        if (subscriptions.partitionsAutoAssigned() && !lostPartitions.isEmpty()) {
+            log.info("Lost previously assigned partitions {} due to {}", lostPartitions, rootCause);
 
-            Set lostPartitions = metadataSnapshot == null ? ownedPartitions :
-                ownedPartitions.stream().filter(predicate).collect(Collectors.toSet());
-
-            if (!lostPartitions.isEmpty()) {
-                log.info("Lost previously assigned partitions {} due to {}", lostPartitions, rootCause);
-
-                Set leftPartitions = new HashSet<>(ownedPartitions);
-                leftPartitions.removeAll(lostPartitions);
-                subscriptions.assignFromSubscribed(leftPartitions);
-
-                ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
-                try {
-                    listener.onPartitionsLost(lostPartitions);
-                } catch (WakeupException | InterruptException e) {
-                    throw e;
-                } catch (Exception e) {
-                    log.error("User provided listener {} failed on partition being lost of {}",
-                        listener.getClass().getName(), lostPartitions, e);
-                    return e;
-                }
+            ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
+            try {
+                listener.onPartitionsLost(lostPartitions);
+            } catch (WakeupException | InterruptException e) {
+                throw e;
+            } catch (Exception e) {
+                log.error("User provided listener {} failed on partition being lost of {}",
+                    listener.getClass().getName(), lostPartitions, e);
+                return e;
             }
         }
 
@@ -371,6 +351,8 @@ protected void onJoinComplete(int generation,
 
         // execute the user's callback after rebalance
         final AtomicReference firstException = new AtomicReference<>(null);
+        Set addedPartitions = new HashSet<>(assignedPartitions);
+        addedPartitions.removeAll(ownedPartitions);
         switch (protocol) {
             case EAGER:
                 if (!ownedPartitions.isEmpty()) {
@@ -378,16 +360,13 @@ protected void onJoinComplete(int generation,
                         "it is likely client is woken up before a previous pending rebalance completes its callback", ownedPartitions, protocol);
                 }
 
-                // TODO: after refactored the error handling, we need to exclude owned partitions from assigned partitions
-
-                firstException.compareAndSet(null, maybeAssignPartitions(assignedPartitions));
+                // assign partitions that are not yet owned
+                firstException.compareAndSet(null, maybeAssignPartitions(addedPartitions));
 
                 break;
 
             case COOPERATIVE:
-                Set addedPartitions = new HashSet<>(assignedPartitions);
                 Set revokedPartitions = new HashSet<>(ownedPartitions);
-                addedPartitions.removeAll(ownedPartitions);
                 revokedPartitions.removeAll(assignedPartitions);
 
                 log.info("Updating with newly assigned partitions: {}, compare with already owned partitions: {}, " +
@@ -401,7 +380,7 @@ protected void onJoinComplete(int generation,
                 firstException.compareAndSet(null, maybeAssignPartitions(addedPartitions));
 
                 // revoked partitions that was previously owned but no longer assigned
-                firstException.compareAndSet(null, maybeRevokePartitions(tp -> !assignedPartitions.contains(tp)));
+                firstException.compareAndSet(null, maybeRevokePartitions(revokedPartitions));
 
                 // request re-join based on leader communicated error code
                 if (assignment.error() == ConsumerProtocol.AssignmentError.NEED_REJOIN) {
@@ -647,16 +626,31 @@ protected void onJoinPrepare(int generation, String memberId) {
         maybeAutoCommitOffsetsSync(time.timer(rebalanceConfig.rebalanceTimeoutMs));
 
         // execute the user's callback before rebalance
+        final Set revokedPartitions;
         final AtomicReference firstException = new AtomicReference<>(null);
         switch (protocol) {
             case EAGER:
                 // revoke all partitions
-                firstException.compareAndSet(null, maybeRevokePartitions(tp -> true));
+                revokedPartitions = new HashSet<>(subscriptions.assignedPartitions());
+                subscriptions.assignFromSubscribed(Collections.emptySet());
+
+                firstException.compareAndSet(null, maybeRevokePartitions(revokedPartitions));
                 break;
 
             case COOPERATIVE:
                 // only revoke those partitions that are not in the subscription any more.
-                firstException.compareAndSet(null, maybeRevokePartitions(tp -> !subscriptions.subscription().contains(tp.topic())));
+                Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions());
+                revokedPartitions = ownedPartitions.stream()
+                    .filter(tp -> !subscriptions.subscription().contains(tp.topic()))
+                    .collect(Collectors.toSet());
+
+                if (!revokedPartitions.isEmpty()) {
+                    ownedPartitions.removeAll(revokedPartitions);
+                    subscriptions.assignFromSubscribed(ownedPartitions);
+
+                    firstException.compareAndSet(null, maybeRevokePartitions(revokedPartitions));
+                }
+
                 break;
         }
 
@@ -670,8 +664,11 @@ protected void onJoinPrepare(int generation, String memberId) {
 
     @Override
     public void resetGeneration(String errorMessage) {
-        // revoke all partitions
-        Exception e = maybeLostPartitions(errorMessage, tp -> true);
+        // lost all partitions
+        Set lostPartitions = new HashSet<>(subscriptions.assignedPartitions());
+        subscriptions.assignFromSubscribed(Collections.emptySet());
+
+        Exception e = maybeLostPartitions(errorMessage, lostPartitions);
         super.resetGeneration(errorMessage);
 
         if (e != null) {
@@ -686,11 +683,21 @@ public boolean rejoinNeededOrPending() {
 
         // we need to rejoin if we performed the assignment and metadata has changed
         if (assignmentSnapshot != null && !assignmentSnapshot.matches(metadataSnapshot)) {
-            Exception e = maybeLostPartitions("topic metadata has changed and therefore some topics may not exist any more",
-                tp -> !metadataSnapshot.partitionsPerTopic().containsKey(tp.topic()));
+            Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions());
+            Set lostPartitions = ownedPartitions.stream()
+                .filter(tp -> !metadataSnapshot.partitionsPerTopic().containsKey(tp.topic()))
+                .collect(Collectors.toSet());
 
-            if (e != null) {
-                throw new KafkaException("User rebalance callback throws an error", e);
+            if (!lostPartitions.isEmpty()) {
+                ownedPartitions.removeAll(lostPartitions);
+                subscriptions.assignFromSubscribed(ownedPartitions);
+
+                Exception e = maybeLostPartitions("topic metadata has changed and therefore some topics may not exist any more",
+                    lostPartitions);
+
+                if (e != null) {
+                    throw new KafkaException("User rebalance callback throws an error", e);
+                }
             }
 
             return true;

From 65f7c6816d593f5e2ea88b09560124af9c563cd5 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Thu, 27 Jun 2019 14:24:16 -0700
Subject: [PATCH 37/59] Update
 clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java

Co-Authored-By: John Roesler 
---
 .../kafka/clients/consumer/ConsumerRebalanceListener.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
index 27051b8f4460a..f8257b4e05dca 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
@@ -138,7 +138,7 @@ public interface ConsumerRebalanceListener {
     void onPartitionsAssigned(Collection partitions);
 
     /**
-     * A callback method the user can implement to provide handling of cleaning up resources for partitions that have already
+     * A callback method you can implement to provide handling of cleaning up resources for partitions that have already
      * been re-assigned to other consumers. This method will not be called during normal execution as the owned partitions would
      * first be revoked by calling the {@link ConsumerRebalanceListener#onPartitionsRevoked}, before being re-assigned
      * to other consumers during a rebalance event. However, during exceptional scenarios when the consumer realized that it

From a4f584fa417d116ec9ef5167904c26d6b7f8c5b8 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Thu, 27 Jun 2019 14:28:36 -0700
Subject: [PATCH 38/59] Update
 clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java

Co-Authored-By: John Roesler 
---
 .../java/org/apache/kafka/clients/consumer/KafkaConsumer.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index 970cfd74e7602..95f4706b31faf 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -1176,7 +1176,7 @@ public ConsumerRecords poll(final long timeoutMs) {
      * @throws org.apache.kafka.common.errors.AuthorizationException if caller lacks Read access to any of the subscribed
      *             topics or to the configured groupId. See the exception for more details
      * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. invalid groupId or
-     *             session timeout, errors deserializing key/value pairs, user's rebalance callback thrown exceptions,
+     *             session timeout, errors deserializing key/value pairs, your rebalance callback thrown exceptions,
      *             or any new error cases in future versions)
      * @throws java.lang.IllegalArgumentException if the timeout value is negative
      * @throws java.lang.IllegalStateException if the consumer is not subscribed to any topics or manually assigned any

From fb5b1e1f9fcece8f3f1a664c5892e20f6b9aa844 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Thu, 27 Jun 2019 18:42:16 -0700
Subject: [PATCH 39/59] github comments

---
 .../consumer/ConsumerRebalanceListener.java   |  24 ++--
 .../internals/ConsumerCoordinator.java        | 108 ++++++++++--------
 .../consumer/internals/ConsumerProtocol.java  |   5 +-
 .../consumer/internals/PartitionAssignor.java |  41 +++----
 .../internals/ConsumerCoordinatorTest.java    |  14 +--
 5 files changed, 101 insertions(+), 91 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
index f8257b4e05dca..a7c5855222a70 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
@@ -28,8 +28,9 @@
  * those partitions will never be reassigned and this callback is not applicable.
  * 

  * When Kafka is managing the group membership, a partition re-assignment will be triggered any time the members of the group change or the subscription
- * of the members changes. This can occur when processes die, new process instances are added or old instances come back to life after failure.
- * Rebalances can also be triggered by changes affecting the subscribed topics (e.g. when the number of partitions is
+ * of the members changes. This can occur when processes die, new process instances are added or old instances come back to life after failure causing
+ * a rebalance event.
+ * Partition re-assignments can also be triggered by changes affecting the subscribed topics (e.g. when the number of partitions is
  * administratively adjusted).
  * 

  * There are many uses for this functionality. One common use is saving offsets in a custom store. By saving offsets in
@@ -50,16 +51,19 @@
  * It is guaranteed that for any partition, if a partition is reassigned from one consumer to another, then the old consumer will
  * always invoke {@link #onPartitionsRevoked(Collection) onPartitionsRevoked} for that partition prior to the new consumer
  * invoking {@link #onPartitionsAssigned(Collection) onPartitionsAssigned} for the same partition. So if offsets or other state is saved in the
- * {@link #onPartitionsRevoked(Collection) onPartitionsRevoked} call it is guaranteed to be saved by the time the process taking over that
- * partition has their {@link #onPartitionsAssigned(Collection) onPartitionsAssigned} callback called to load the state.
+ * {@link #onPartitionsRevoked(Collection) onPartitionsRevoked} call by one consumer member, it will be always accessible by the time the
+ * other consumer member taking over that partition and triggering its {@link #onPartitionsAssigned(Collection) onPartitionsAssigned} callback to load the state.
  * 

- * In addition, a third {@link #onPartitionsLost(Collection)} callback will be invoked when the partition previously owned by the consumer
- * are lost. It is different to the {@link #onPartitionsRevoked(Collection)} function such that the latter is invoked when a partition is revoked
- * from the consumer during a rebalance event, while the former is invoked when a partition is lost not during a rebalance event, but due to
- * an exceptional event, such as a consumer finding out it is no longer part of the consumer group, or that some of its assigned partitions no longer
- * exist due to administrative operations like delete topics. Users then could implement these two functions differently (by default,
+ * In addition, a third {@link #onPartitionsLost(Collection)} callback will be invoked when some partitions previously owned by the consumer
+ * are lost due to unexpected events, for example, the consumer finding some of its assigned partitions no longer
+ * exist from its topic metadata due to administrative operations like delete topics, or the consumer realizing it is
+ * no longer part of the consumer group due to soft failures and therefore all of its previously owned partitions have reassigned to other consumers already.
+ * Note this function is very different
+ * to the {@link #onPartitionsRevoked(Collection)} function: the latter is invoked when a partition is revoked
+ * from the consumer who still owns it right before the invocation, while the former is invoked when a partition is already lost
+ * and not owned by the consumer anymore during the invocation. Users then could implement these two functions differently (by default,
  * {@link #onPartitionsLost(Collection)} will be calling {@link #onPartitionsRevoked(Collection)} directly); for example, in the
- * {@link #onPartitionsLost(Collection)} we would not need to store the offsets since we know these partitions are no longer owned by the consumer
+ * {@link #onPartitionsLost(Collection)} we should not need to store the offsets since we know these partitions are no longer owned by the consumer
  * at that time.
  * 

  * Here is pseudo-code for a callback implementation for saving offsets:
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 9771867bdb671..8fd61fbc3515b 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -292,7 +292,7 @@ private Exception maybeRevokePartitions(final Set revokedPartiti
     }
 
     private Exception maybeLostPartitions(final String rootCause, final Set lostPartitions) {
-        if (subscriptions.partitionsAutoAssigned() && !lostPartitions.isEmpty()) {
+        if (!lostPartitions.isEmpty()) {
             log.info("Lost previously assigned partitions {} due to {}", lostPartitions, rootCause);
 
             ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
@@ -326,6 +326,15 @@ protected void onJoinComplete(int generation,
         Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions());
 
         Assignment assignment = ConsumerProtocol.deserializeAssignment(assignmentBuffer);
+
+        if (assignment.error() == ConsumerProtocol.AssignmentError.ASSIGNOR_ERROR) {
+            // leader's assignor has a fatal
+            throw new KafkaException("Leader's assignor logic has a fatal error that violates the rebalance protocol, should close this client now");
+        } else if (assignment.error() == ConsumerProtocol.AssignmentError.NEED_REJOIN) {
+            // request re-join based on leader communicated error code
+            requestRejoin();
+        }
+
         if (!subscriptions.assignFromSubscribed(assignment.partitions())) {
             log.warn("We received an assignment {} that doesn't match our current subscription {}; it is likely " +
                 "that the subscription has changed since we joined the group. Will try re-join the group with current subscription",
@@ -355,11 +364,6 @@ protected void onJoinComplete(int generation,
         addedPartitions.removeAll(ownedPartitions);
         switch (protocol) {
             case EAGER:
-                if (!ownedPartitions.isEmpty()) {
-                    log.info("Coordinator has owned partitions {} that are not revoked with {} protocol, " +
-                        "it is likely client is woken up before a previous pending rebalance completes its callback", ownedPartitions, protocol);
-                }
-
                 // assign partitions that are not yet owned
                 firstException.compareAndSet(null, maybeAssignPartitions(addedPartitions));
 
@@ -382,11 +386,6 @@ protected void onJoinComplete(int generation,
                 // revoked partitions that was previously owned but no longer assigned
                 firstException.compareAndSet(null, maybeRevokePartitions(revokedPartitions));
 
-                // request re-join based on leader communicated error code
-                if (assignment.error() == ConsumerProtocol.AssignmentError.NEED_REJOIN) {
-                    requestRejoin();
-                }
-
                 break;
         }
 
@@ -425,7 +424,7 @@ public boolean poll(Timer timer) {
 
         if (subscriptions.partitionsAutoAssigned()) {
             if (protocol == null) {
-                throw new IllegalStateException("User confingure ConsumerConfig#PARTITION_ASSIGNMENT_STRATEGY_CONFIG to empty " +
+                throw new IllegalStateException("User configure ConsumerConfig#PARTITION_ASSIGNMENT_STRATEGY_CONFIG to empty " +
                     "while trying to subscribe for group protocol to auto assign partitions");
             }
             // Always update the heartbeat last poll time so that the heartbeat thread does not leave the
@@ -516,13 +515,13 @@ protected Map performAssignment(String leaderId,
         Set allSubscribedTopics = new HashSet<>();
         Map subscriptions = new HashMap<>();
         // collect all the owned partitions
-        Map ownedPartitions = new HashMap<>();
+        Map> ownedPartitions = new HashMap<>();
         for (JoinGroupResponseData.JoinGroupResponseMember memberSubscription : allSubscriptions) {
             Subscription subscription = ConsumerProtocol.deserializeSubscription(ByteBuffer.wrap(memberSubscription.metadata()));
             subscription.setGroupInstanceId(Optional.ofNullable(memberSubscription.groupInstanceId()));
             subscriptions.put(memberSubscription.memberId(), subscription);
             allSubscribedTopics.addAll(subscription.topics());
-            ownedPartitions.putAll(subscription.ownedPartitions().stream().collect(Collectors.toMap(item -> item, item -> memberSubscription.memberId())));
+            ownedPartitions.put(memberSubscription.memberId(), subscription.ownedPartitions());
         }
 
         // the leader will begin watching for changes to any of the topics the group is interested in,
@@ -535,13 +534,8 @@ protected Map performAssignment(String leaderId,
 
         Map assignments = assignor.assign(metadata.fetch(), subscriptions);
 
-        switch (protocol) {
-            case EAGER:
-                break;
-
-            case COOPERATIVE:
-                adjustAssignment(ownedPartitions, assignments);
-                break;
+        if (protocol == RebalanceProtocol.COOPERATIVE) {
+            validateAssignment(ownedPartitions, assignments);
         }
 
         // user-customized assignor may have created some topics that are not in the subscription list
@@ -586,34 +580,47 @@ protected Map performAssignment(String leaderId,
         return groupAssignment;
     }
 
-    private void adjustAssignment(final Map ownedPartitions,
-                                  final Map assignments) {
-        boolean revocationsNeeded = false;
-        Set assignedPartitions = new HashSet<>();
+    /**
+     * Used by COOPERATIVE rebalance protocol only.
+     *
+     * Validate the user assignor returned assignments such that there are no partitions which are going to
+     * be assigned to a different consumer other than its current owner (i.e. who includes it in its owned
+     * partitions list encoded in subscription); also if there are any revoked partitions set the error code to let
+     * everyone retry.
+     */
+    private void validateAssignment(final Map> ownedPartitions,
+                                    final Map assignments) {
+        Set totalRevokedPartitions = new HashSet<>();
+        Set totalAddedPartitions = new HashSet<>();
         for (final Map.Entry entry : assignments.entrySet()) {
             final Assignment assignment = entry.getValue();
-            assignedPartitions.addAll(assignment.partitions());
-
-            // update the assignment if the partition is owned by another different owner
-            List updatedPartitions = assignment.partitions().stream()
-                .filter(tp -> ownedPartitions.containsKey(tp) && !entry.getKey().equals(ownedPartitions.get(tp)))
-                .collect(Collectors.toList());
-            if (!updatedPartitions.equals(assignment.partitions())) {
-                assignment.updatePartitions(updatedPartitions);
-                revocationsNeeded = true;
-            }
+            final Set addedPartitions = new HashSet<>(assignment.partitions());
+            addedPartitions.removeAll(ownedPartitions.get(entry.getKey()));
+            final Set revokedPartitions = new HashSet<>(ownedPartitions.get(entry.getKey()));
+            revokedPartitions.removeAll(assignment.partitions());
+
+            totalAddedPartitions.addAll(addedPartitions);
+            totalRevokedPartitions.addAll(revokedPartitions);
         }
 
-        // for all owned but not assigned partitions, blindly add them to assignment
-        for (final Map.Entry entry : ownedPartitions.entrySet()) {
-            final TopicPartition tp = entry.getKey();
-            if (!assignedPartitions.contains(tp)) {
-                assignments.get(entry.getValue()).partitions().add(tp);
+        // if there are overlap between revoked partitions and added partitions, it means some partitions
+        // immediately gets re-assigned to another member while it is still claimed by some member
+        totalAddedPartitions.retainAll(totalRevokedPartitions);
+        if (!totalAddedPartitions.isEmpty()) {
+            log.error("With COOPERATIVE protocol, all owned partitions should not be " +
+                "reassigned to other members; however the assignor has reassigned partitions {} which are still owned " +
+                "by some members; return the error code to all members to let them stop", totalAddedPartitions);
+
+            for (final Assignment assignment : assignments.values()) {
+                assignment.setError(ConsumerProtocol.AssignmentError.ASSIGNOR_ERROR);
             }
         }
 
-        // if revocations are triggered, tell everyone to re-join immediately.
-        if (revocationsNeeded) {
+        if (!totalRevokedPartitions.isEmpty()) {
+            log.info("With COOPERATIVE protocol there are some partitions {} that will be revoked by their current " +
+                "owners; return the error code to let all members to re-trigger another rebalance after revoking partitions",
+                totalRevokedPartitions);
+
             for (final Assignment assignment : assignments.values()) {
                 assignment.setError(ConsumerProtocol.AssignmentError.NEED_REJOIN);
             }
@@ -664,16 +671,19 @@ protected void onJoinPrepare(int generation, String memberId) {
 
     @Override
     public void resetGeneration(String errorMessage) {
-        // lost all partitions
-        Set lostPartitions = new HashSet<>(subscriptions.assignedPartitions());
-        subscriptions.assignFromSubscribed(Collections.emptySet());
+        if (subscriptions.partitionsAutoAssigned()) {
+            // lost all partitions if it is based on subscriptions
+            Set lostPartitions = new HashSet<>(subscriptions.assignedPartitions());
+            subscriptions.assignFromSubscribed(Collections.emptySet());
 
-        Exception e = maybeLostPartitions(errorMessage, lostPartitions);
-        super.resetGeneration(errorMessage);
+            Exception e = maybeLostPartitions(errorMessage, lostPartitions);
 
-        if (e != null) {
-            throw new KafkaException("User rebalance callback throws an error", e);
+            if (e != null) {
+                throw new KafkaException("User rebalance callback throws an error", e);
+            }
         }
+
+        super.resetGeneration(errorMessage);
     }
 
     @Override
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
index d05d5b06906a2..661a591205d5c 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
@@ -121,7 +121,8 @@ public class ConsumerProtocol {
 
     public enum AssignmentError {
         NONE(0),
-        NEED_REJOIN(1);
+        NEED_REJOIN(1),
+        ASSIGNOR_ERROR(2);
 
         private final short code;
 
@@ -139,6 +140,8 @@ public static AssignmentError fromCode(final short code) {
                     return NONE;
                 case 1:
                     return NEED_REJOIN;
+                case 2:
+                    return ASSIGNOR_ERROR;
                 default:
                     throw new IllegalArgumentException("Unknown error code: " + code);
             }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
index c26f68462ea4b..a39a652e03163 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
@@ -89,7 +89,9 @@ default List supportedProtocols() {
 
     /**
      * Return the version of the assignor which indicates how the user metadata encodings
-     * and the assignment algorithm gets evolved.
+     * and the assignment algorithm gets evolved. The broker-side group coordinator can then
+     * select the leader as the one with the highest assignor version, expecting it be able
+     * to decode other member assignor's subscription user metadata encoded with older versions.
      */
     default short version() {
         return (short) 0;
@@ -166,10 +168,6 @@ public Subscription(List topics) {
             this(topics, ByteBuffer.wrap(new byte[0]));
         }
 
-        Short version() {
-            return version;
-        }
-
         public List topics() {
             return topics;
         }
@@ -182,12 +180,16 @@ public ByteBuffer userData() {
             return userData;
         }
 
-        public void setGroupInstanceId(Optional groupInstanceId) {
+        public Optional groupInstanceId() {
+            return groupInstanceId;
+        }
+
+        void setGroupInstanceId(Optional groupInstanceId) {
             this.groupInstanceId = groupInstanceId;
         }
 
-        public Optional groupInstanceId() {
-            return groupInstanceId;
+        Short version() {
+            return version;
         }
 
         @Override
@@ -202,7 +204,7 @@ public String toString() {
 
     class Assignment {
         private final Short version;
-        private List partitions;
+        private final List partitions;
         private final ByteBuffer userData;
         private ConsumerProtocol.AssignmentError error;
 
@@ -231,28 +233,23 @@ public Assignment(List partitions) {
             this(partitions, ByteBuffer.wrap(new byte[0]));
         }
 
-        Short version() {
-            return version;
-        }
-
         public List partitions() {
             return partitions;
         }
 
-        public ConsumerProtocol.AssignmentError error() {
-            return error;
+        public ByteBuffer userData() {
+            return userData;
         }
 
-        public void updatePartitions(List partitions) {
-            this.partitions = partitions;
+        Short version() {
+            return version;
         }
 
-        public void setError(ConsumerProtocol.AssignmentError error) {
-            this.error = error;
+        ConsumerProtocol.AssignmentError error() {
+            return error;
         }
-
-        public ByteBuffer userData() {
-            return userData;
+        void setError(ConsumerProtocol.AssignmentError error) {
+            this.error = error;
         }
 
         @Override
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index a0aa1ba09d013..9580abb7dcb4c 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -197,7 +197,7 @@ public void teardown() {
     }
 
     @Test
-    public void testSelectRebalanceProtcol() {
+    public void testSelectRebalanceProtocol() {
         List assignors = new ArrayList<>();
         assignors.add(new MockPartitionAssignor(Collections.singletonList(PartitionAssignor.RebalanceProtocol.EAGER)));
         assignors.add(new MockPartitionAssignor(Collections.singletonList(PartitionAssignor.RebalanceProtocol.COOPERATIVE)));
@@ -607,7 +607,6 @@ public void testMetadataRefreshDuringRebalance() {
         final String consumerId = "leader";
         final List owned = Collections.emptyList();
         final List oldAssigned = Arrays.asList(t1p);
-
         subscriptions.subscribe(Pattern.compile(".*"), rebalanceListener);
         client.updateMetadata(TestUtils.metadataUpdateWith(1, singletonMap(topic1, 1)));
         coordinator.maybeUpdateSubscriptionMetadata();
@@ -1171,7 +1170,7 @@ public void onPartitionsAssigned(Collection partitions) {
 
         assertFalse(coordinator.rejoinNeededOrPending());
         assertEquals(0, rebalanceListener.revokedCount);
-        assertEquals(protocol == PartitionAssignor.RebalanceProtocol.EAGER ? 2 : 1, rebalanceListener.assignedCount);
+        assertEquals(1, rebalanceListener.assignedCount);
     }
 
     @Test
@@ -1382,8 +1381,7 @@ private void testInFlightRequestsFailedAfterCoordinatorMarkedDead(Errors error)
     public void testAutoCommitDynamicAssignment() {
         final String consumerId = "consumer";
 
-        try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors,
-            true)
+        try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, true)
         ) {
             subscriptions.subscribe(singleton(topic1), rebalanceListener);
             joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p));
@@ -1399,8 +1397,7 @@ public void testAutoCommitDynamicAssignment() {
     public void testAutoCommitRetryBackoff() {
         final String consumerId = "consumer";
 
-        try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors,
-            true)) {
+        try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, true)) {
             subscriptions.subscribe(singleton(topic1), rebalanceListener);
             joinAsFollowerAndReceiveAssignment(consumerId, coordinator, singletonList(t1p));
 
@@ -1473,8 +1470,7 @@ public void testAutoCommitAwaitsInterval() {
     public void testAutoCommitDynamicAssignmentRebalance() {
         final String consumerId = "consumer";
 
-        try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors,
-            true)) {
+        try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, true)) {
             subscriptions.subscribe(singleton(topic1), rebalanceListener);
             client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
             coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));

From 10c1f976010193030409a3b22f9dd8f415b4831c Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Fri, 28 Jun 2019 18:00:18 -0700
Subject: [PATCH 40/59] github comments

---
 .../consumer/ConsumerRebalanceListener.java   | 41 +++++++++------
 .../internals/AbstractCoordinator.java        |  8 +--
 .../internals/ConsumerCoordinator.java        | 51 +++++++++----------
 .../consumer/internals/ConsumerProtocol.java  |  7 +--
 .../consumer/internals/PartitionAssignor.java |  5 +-
 .../consumer/RoundRobinAssignorTest.java      | 20 ++------
 6 files changed, 63 insertions(+), 69 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
index a7c5855222a70..be862834680ff 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
@@ -54,18 +54,31 @@
  * {@link #onPartitionsRevoked(Collection) onPartitionsRevoked} call by one consumer member, it will be always accessible by the time the
  * other consumer member taking over that partition and triggering its {@link #onPartitionsAssigned(Collection) onPartitionsAssigned} callback to load the state.
  * 

- * In addition, a third {@link #onPartitionsLost(Collection)} callback will be invoked when some partitions previously owned by the consumer
- * are lost due to unexpected events, for example, the consumer finding some of its assigned partitions no longer
- * exist from its topic metadata due to administrative operations like delete topics, or the consumer realizing it is
- * no longer part of the consumer group due to soft failures and therefore all of its previously owned partitions have reassigned to other consumers already.
- * Note this function is very different
- * to the {@link #onPartitionsRevoked(Collection)} function: the latter is invoked when a partition is revoked
- * from the consumer who still owns it right before the invocation, while the former is invoked when a partition is already lost
- * and not owned by the consumer anymore during the invocation. Users then could implement these two functions differently (by default,
+ * You can think of revocation as a graceful way to give up ownership of a partition. In some cases, the consumer may not have an opportunity to do so.
+ * For example, if the session times out, then the partitions may be reassigned before we have a chance to revoke them gracefully.
+ * For this case, we have a third callback {@link #onPartitionsLost(Collection)}. The difference between this function and
+ * {@link #onPartitionsRevoked(Collection)} is that upon invocation of {@link #onPartitionsLost(Collection)}, the partitions are no longer owned
+ * by the consumer but maybe by some other members in the group.
+ * Users could implement these two functions differently (by default,
  * {@link #onPartitionsLost(Collection)} will be calling {@link #onPartitionsRevoked(Collection)} directly); for example, in the
  * {@link #onPartitionsLost(Collection)} we should not need to store the offsets since we know these partitions are no longer owned by the consumer
  * at that time.
  * 

+ * It is possible
+ * for a {@link org.apache.kafka.common.errors.WakeupException} or {@link org.apache.kafka.common.errors.InterruptException}
+ * to be raised from one of these nested invocations. In this case, the exception will be propagated to the current
+ * invocation of {@link KafkaConsumer#poll(java.time.Duration)} in which this callback is being executed. This means it is not
+ * necessary to catch these exceptions and re-attempt to wakeup or interrupt the consumer thread.
+ * Also if the callback function implementation itself throws an exception, this exception will be propagated to the current
+ * invocation of {@link KafkaConsumer#poll(java.time.Duration)} as well.
+ *
+ * Note that the semantics of the callback is only for notifying the assignment change, and hence throwing any exceptions would
+ * not affect any of the notified assignment changes at all. That means, if user captures the exception and
+ * calls {@link KafkaConsumer#poll(java.time.Duration)} again, these callback functions would not be invoked any more unless there are
+ * new partitions reassigned.
+ *
+ * 

+ *
  * Here is pseudo-code for a callback implementation for saving offsets:
  * 
  * {@code
@@ -109,7 +122,7 @@ public interface ConsumerRebalanceListener {
      * 

      * It is common for the revocation callback to use the consumer instance in order to commit offsets. It is possible
      * for a {@link org.apache.kafka.common.errors.WakeupException} or {@link org.apache.kafka.common.errors.InterruptException}
-     * to be raised from one these nested invocations. In this case, the exception will be propagated to the current
+     * to be raised from one of these nested invocations. In this case, the exception will be propagated to the current
      * invocation of {@link KafkaConsumer#poll(java.time.Duration)} in which this callback is being executed. This means it is not
      * necessary to catch these exceptions and re-attempt to wakeup or interrupt the consumer thread.
      *
@@ -130,7 +143,7 @@ public interface ConsumerRebalanceListener {
      * 

      * It is common for the assignment callback to use the consumer instance in order to query offsets. It is possible
      * for a {@link org.apache.kafka.common.errors.WakeupException} or {@link org.apache.kafka.common.errors.InterruptException}
-     * to be raised from one these nested invocations. In this case, the exception will be propagated to the current
+     * to be raised from one of these nested invocations. In this case, the exception will be propagated to the current
      * invocation of {@link KafkaConsumer#poll(java.time.Duration)} in which this callback is being executed. This means it is not
      * necessary to catch these exceptions and re-attempt to wakeup or interrupt the consumer thread.
      *
@@ -148,16 +161,14 @@ public interface ConsumerRebalanceListener {
      * to other consumers during a rebalance event. However, during exceptional scenarios when the consumer realized that it
      * does not own this partition any longer, i.e. not revoked via a normal rebalance event, then this method would be invoked.
      *
-     * For example, if a consumer is kicked out of the group in the previous generation and hence were not aware when the new
-     * group is formed and partitions assigned to other members; later when it finally be notified about the illegal generation
-     * error and are going to reset its generation and re-join, this function will be called.
+     * For example, this function is called if a consumer's session timeout has expired.
      *
-     * By default it will just trigger {@link ConsumerRebalanceListener#onPartitionsRevoked}; for advanced users who want to distinguish
+     * By default it will just trigger {@link ConsumerRebalanceListener#onPartitionsRevoked}; for users who want to distinguish
      * the handling logic of revoked partitions v.s. lost partitions, they can override the default implementation.
      *
      * It is possible
      * for a {@link org.apache.kafka.common.errors.WakeupException} or {@link org.apache.kafka.common.errors.InterruptException}
-     * to be raised from one these nested invocations. In this case, the exception will be propagated to the current
+     * to be raised from one of these nested invocations. In this case, the exception will be propagated to the current
      * invocation of {@link KafkaConsumer#poll(java.time.Duration)} in which this callback is being executed. This means it is not
      * necessary to catch these exceptions and re-attempt to wakeup or interrupt the consumer thread.
      *
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 00aadfe416335..ec5ae5cb3a49c 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -521,7 +521,7 @@ public void handle(JoinGroupResponse joinResponse, RequestFuture fut
                 future.raise(error);
             } else if (error == Errors.UNKNOWN_MEMBER_ID) {
                 // reset the member id and retry immediately
-                resetGeneration(this.getClass().getName() + " encountering " + error);
+                resetGeneration("encountering " + error + " on join response");
                 log.debug("Attempt to join group failed due to unknown member id.");
                 future.raise(error);
             } else if (error == Errors.COORDINATOR_NOT_AVAILABLE
@@ -643,7 +643,7 @@ public void handle(SyncGroupResponse syncResponse,
                 } else if (error == Errors.UNKNOWN_MEMBER_ID
                         || error == Errors.ILLEGAL_GENERATION) {
                     log.debug("SyncGroup failed: {}", error.message());
-                    resetGeneration(this.getClass().getName() + " encountering " + error);
+                    resetGeneration("encountering " + error + " on sync group response");
                     future.raise(error);
                 } else if (error == Errors.COORDINATOR_NOT_AVAILABLE
                         || error == Errors.NOT_COORDINATOR) {
@@ -915,14 +915,14 @@ public void handle(HeartbeatResponse heartbeatResponse, RequestFuture futu
                 future.raise(error);
             } else if (error == Errors.ILLEGAL_GENERATION) {
                 log.info("Attempt to heartbeat failed since generation {} is not current", generation.generationId);
-                resetGeneration(this.getClass().getName() + " encountering " + error);
+                resetGeneration("encountering " + error + " on heartbeat response");
                 future.raise(error);
             } else if (error == Errors.FENCED_INSTANCE_ID) {
                 log.error("Received fatal exception: group.instance.id gets fenced");
                 future.raise(error);
             } else if (error == Errors.UNKNOWN_MEMBER_ID) {
                 log.info("Attempt to heartbeat failed for since member id {} is not valid.", generation.memberId);
-                resetGeneration(this.getClass().getName() + " encountering " + error);
+                resetGeneration("encountering " + error + " on heartbeat response");
                 future.raise(error);
             } else if (error == Errors.GROUP_AUTHORIZATION_FAILED) {
                 future.raise(new GroupAuthorizationException(rebalanceConfig.groupId));
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 8fd61fbc3515b..8d635df38d712 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -253,7 +253,7 @@ private void maybeUpdateJoinedSubscription(Set assignedPartition
         }
     }
 
-    private Exception maybeAssignPartitions(final Set assignedPartitions) {
+    private Exception maybeInvokePartitionsAssigned(final Set assignedPartitions) {
         if (!assignedPartitions.isEmpty()) {
             log.info("Setting newly assigned partitions: {}", Utils.join(assignedPartitions, ", "));
 
@@ -263,7 +263,7 @@ private Exception maybeAssignPartitions(final Set assignedPartit
             } catch (WakeupException | InterruptException e) {
                 throw e;
             } catch (Exception e) {
-                log.error("User provided listener {} failed on partition assignment of {}",
+                log.error("User provided listener {} failed on invocation of onPartitionsAssigned for partitions {}",
                     listener.getClass().getName(), assignedPartitions, e);
                 return e;
             }
@@ -272,7 +272,7 @@ private Exception maybeAssignPartitions(final Set assignedPartit
         return null;
     }
 
-    private Exception maybeRevokePartitions(final Set revokedPartitions) {
+    private Exception maybeInvokePartitionsRevoked(final Set revokedPartitions) {
         if (!revokedPartitions.isEmpty()) {
             log.info("Revoke previously assigned partitions {}", revokedPartitions);
 
@@ -282,7 +282,7 @@ private Exception maybeRevokePartitions(final Set revokedPartiti
             } catch (WakeupException | InterruptException e) {
                 throw e;
             } catch (Exception e) {
-                log.error("User provided listener {} failed on partition being revocation of {}",
+                log.error("User provided listener {} failed on invocation of onPartitionsRevoked for partitions {}",
                     listener.getClass().getName(), revokedPartitions, e);
                 return e;
             }
@@ -291,7 +291,7 @@ private Exception maybeRevokePartitions(final Set revokedPartiti
         return null;
     }
 
-    private Exception maybeLostPartitions(final String rootCause, final Set lostPartitions) {
+    private Exception maybeInvokePartitionsLost(final String rootCause, final Set lostPartitions) {
         if (!lostPartitions.isEmpty()) {
             log.info("Lost previously assigned partitions {} due to {}", lostPartitions, rootCause);
 
@@ -301,7 +301,7 @@ private Exception maybeLostPartitions(final String rootCause, final Set performAssignment(String leaderId,
         Map assignments = assignor.assign(metadata.fetch(), subscriptions);
 
         if (protocol == RebalanceProtocol.COOPERATIVE) {
-            validateAssignment(ownedPartitions, assignments);
+            validateCooperativeAssignment(ownedPartitions, assignments);
         }
 
         // user-customized assignor may have created some topics that are not in the subscription list
@@ -588,8 +585,8 @@ protected Map performAssignment(String leaderId,
      * partitions list encoded in subscription); also if there are any revoked partitions set the error code to let
      * everyone retry.
      */
-    private void validateAssignment(final Map> ownedPartitions,
-                                    final Map assignments) {
+    private void validateCooperativeAssignment(final Map> ownedPartitions,
+                                               final Map assignments) {
         Set totalRevokedPartitions = new HashSet<>();
         Set totalAddedPartitions = new HashSet<>();
         for (final Map.Entry entry : assignments.entrySet()) {
@@ -607,17 +604,15 @@ private void validateAssignment(final Map> ownedPar
         // immediately gets re-assigned to another member while it is still claimed by some member
         totalAddedPartitions.retainAll(totalRevokedPartitions);
         if (!totalAddedPartitions.isEmpty()) {
-            log.error("With COOPERATIVE protocol, all owned partitions should not be " +
+            log.error("With the COOPERATIVE protocol, owned partitions cannot be " +
                 "reassigned to other members; however the assignor has reassigned partitions {} which are still owned " +
                 "by some members; return the error code to all members to let them stop", totalAddedPartitions);
 
-            for (final Assignment assignment : assignments.values()) {
-                assignment.setError(ConsumerProtocol.AssignmentError.ASSIGNOR_ERROR);
-            }
+            throw new IllegalStateException("Assignor supporting the COOPERATIVE protocol violates its requirements");
         }
 
         if (!totalRevokedPartitions.isEmpty()) {
-            log.info("With COOPERATIVE protocol there are some partitions {} that will be revoked by their current " +
+            log.debug("With the COOPERATIVE protocol there are some partitions {} that will be revoked by their current " +
                 "owners; return the error code to let all members to re-trigger another rebalance after revoking partitions",
                 totalRevokedPartitions);
 
@@ -641,7 +636,7 @@ protected void onJoinPrepare(int generation, String memberId) {
                 revokedPartitions = new HashSet<>(subscriptions.assignedPartitions());
                 subscriptions.assignFromSubscribed(Collections.emptySet());
 
-                firstException.compareAndSet(null, maybeRevokePartitions(revokedPartitions));
+                firstException.compareAndSet(null, maybeInvokePartitionsRevoked(revokedPartitions));
                 break;
 
             case COOPERATIVE:
@@ -655,7 +650,7 @@ protected void onJoinPrepare(int generation, String memberId) {
                     ownedPartitions.removeAll(revokedPartitions);
                     subscriptions.assignFromSubscribed(ownedPartitions);
 
-                    firstException.compareAndSet(null, maybeRevokePartitions(revokedPartitions));
+                    firstException.compareAndSet(null, maybeInvokePartitionsRevoked(revokedPartitions));
                 }
 
                 break;
@@ -676,7 +671,7 @@ public void resetGeneration(String errorMessage) {
             Set lostPartitions = new HashSet<>(subscriptions.assignedPartitions());
             subscriptions.assignFromSubscribed(Collections.emptySet());
 
-            Exception e = maybeLostPartitions(errorMessage, lostPartitions);
+            Exception e = maybeInvokePartitionsLost(errorMessage, lostPartitions);
 
             if (e != null) {
                 throw new KafkaException("User rebalance callback throws an error", e);
@@ -702,7 +697,7 @@ public boolean rejoinNeededOrPending() {
                 ownedPartitions.removeAll(lostPartitions);
                 subscriptions.assignFromSubscribed(ownedPartitions);
 
-                Exception e = maybeLostPartitions("topic metadata has changed and therefore some topics may not exist any more",
+                Exception e = maybeInvokePartitionsLost("topic metadata has changed and therefore some topics may not exist any more",
                     lostPartitions);
 
                 if (e != null) {
@@ -1124,7 +1119,7 @@ public void handle(OffsetCommitResponse commitResponse, RequestFuture futu
                         } else if (error == Errors.UNKNOWN_MEMBER_ID
                                 || error == Errors.ILLEGAL_GENERATION) {
                             // need to reset generation and re-join group
-                            resetGeneration(this.getClass().getName() + " encountering " + error);
+                            resetGeneration("encountering " + error + " on offset commit response");
                             future.raise(new CommitFailedException());
                             return;
                         } else {
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
index 661a591205d5c..824a128533d30 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java
@@ -119,10 +119,9 @@ public class ConsumerProtocol {
         new Field(USER_DATA_KEY_NAME, Type.NULLABLE_BYTES),
         ERROR_CODE);
 
-    public enum AssignmentError {
+    enum AssignmentError {
         NONE(0),
-        NEED_REJOIN(1),
-        ASSIGNOR_ERROR(2);
+        NEED_REJOIN(1);
 
         private final short code;
 
@@ -140,8 +139,6 @@ public static AssignmentError fromCode(final short code) {
                     return NONE;
                 case 1:
                     return NEED_REJOIN;
-                case 2:
-                    return ASSIGNOR_ERROR;
                 default:
                     throw new IllegalArgumentException("Unknown error code: " + code);
             }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
index a39a652e03163..0f9f15cd7ddd6 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
@@ -156,7 +156,7 @@ class Subscription {
             this(version, topics, userData, Collections.emptyList());
         }
 
-        public Subscription(List topics, ByteBuffer userData, List ownedPartitions) {
+        Subscription(List topics, ByteBuffer userData, List ownedPartitions) {
             this(CONSUMER_PROTOCOL_V1, topics, userData, ownedPartitions);
         }
 
@@ -184,7 +184,7 @@ public Optional groupInstanceId() {
             return groupInstanceId;
         }
 
-        void setGroupInstanceId(Optional groupInstanceId) {
+        public void setGroupInstanceId(Optional groupInstanceId) {
             this.groupInstanceId = groupInstanceId;
         }
 
@@ -248,6 +248,7 @@ Short version() {
         ConsumerProtocol.AssignmentError error() {
             return error;
         }
+
         void setError(ConsumerProtocol.AssignmentError error) {
             this.error = error;
         }
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/RoundRobinAssignorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/RoundRobinAssignorTest.java
index fa6840649a167..2fd14c1dfedf5 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/RoundRobinAssignorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/RoundRobinAssignorTest.java
@@ -188,14 +188,10 @@ public void testTwoStaticConsumersTwoTopicsSixPartitions() {
         partitionsPerTopic.put(topic2, 3);
 
         Map consumers = new HashMap<>();
-        Subscription consumer1Subscription = new Subscription(topics(topic1, topic2),
-                                                              null,
-                                                              Collections.emptyList());
+        Subscription consumer1Subscription = new Subscription(topics(topic1, topic2), null);
         consumer1Subscription.setGroupInstanceId(Optional.of(instance1));
         consumers.put(consumer1, consumer1Subscription);
-        Subscription consumer2Subscription = new Subscription(topics(topic1, topic2),
-                                                              null,
-                                                              Collections.emptyList());
+        Subscription consumer2Subscription = new Subscription(topics(topic1, topic2), null);
         consumer2Subscription.setGroupInstanceId(Optional.of(instance2));
         consumers.put(consumer2, consumer2Subscription);
         Map> assignment = assignor.assign(partitionsPerTopic, consumers);
@@ -219,9 +215,7 @@ public void testOneStaticConsumerAndOneDynamicConsumerTwoTopicsSixPartitions() {
 
         Map consumers = new HashMap<>();
 
-        Subscription consumer1Subscription = new Subscription(topics(topic1, topic2),
-                                                              null,
-                                                              Collections.emptyList());
+        Subscription consumer1Subscription = new Subscription(topics(topic1, topic2), null);
         consumer1Subscription.setGroupInstanceId(Optional.of(instance1));
         consumers.put(consumer1, consumer1Subscription);
         consumers.put(consumer2, new Subscription(topics(topic1, topic2)));
@@ -257,9 +251,7 @@ public void testStaticMemberAssignmentPersistent() {
         partitionsPerTopic.put(topic2, 3);
         Map consumers = new HashMap<>();
         for (MemberInfo m : staticMemberInfos) {
-            Subscription subscription = new Subscription(topics(topic1, topic2),
-                                                         null,
-                                                         Collections.emptyList());
+            Subscription subscription = new Subscription(topics(topic1, topic2), null);
             subscription.setGroupInstanceId(m.groupInstanceId);
             consumers.put(m.memberId, subscription);
         }
@@ -333,9 +325,7 @@ private Map> checkStaticAssignment(String topic1,
         partitionsPerTopic.put(topic2, 3);
         Map consumers = new HashMap<>();
         for (MemberInfo m : staticMemberInfos) {
-            Subscription subscription = new Subscription(topics(topic1, topic2),
-                                                         null,
-                                                         Collections.emptyList());
+            Subscription subscription = new Subscription(topics(topic1, topic2), null);
             subscription.setGroupInstanceId(m.groupInstanceId);
             consumers.put(m.memberId, subscription);
         }

From ca7bf3284bb52bcaaf2398bb7311707099f2f9fe Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Fri, 28 Jun 2019 18:02:42 -0700
Subject: [PATCH 41/59] add owned-partitions to subscription

---
 .../consumer/internals/ConsumerCoordinator.java        |  1 +
 .../clients/consumer/internals/PartitionAssignor.java  | 10 +++++-----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 8d635df38d712..2d7c10da5409b 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -204,6 +204,7 @@ protected JoinGroupRequestData.JoinGroupRequestProtocolCollection metadata() {
 
         for (PartitionAssignor assignor : assignors) {
             Subscription subscription = assignor.subscription(joinedSubscription);
+            subscription.setOwnedPartitions(new ArrayList<>(subscriptions.assignedPartitions()));
             ByteBuffer metadata = ConsumerProtocol.serializeSubscription(subscription);
 
             protocolSet.add(new JoinGroupRequestData.JoinGroupRequestProtocol()
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
index 0f9f15cd7ddd6..ddff43f42581c 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PartitionAssignor.java
@@ -132,7 +132,7 @@ class Subscription {
         private final Short version;
         private final List topics;
         private final ByteBuffer userData;
-        private final List ownedPartitions;
+        private List ownedPartitions;
         private Optional groupInstanceId;
 
         Subscription(Short version,
@@ -156,10 +156,6 @@ class Subscription {
             this(version, topics, userData, Collections.emptyList());
         }
 
-        Subscription(List topics, ByteBuffer userData, List ownedPartitions) {
-            this(CONSUMER_PROTOCOL_V1, topics, userData, ownedPartitions);
-        }
-
         public Subscription(List topics, ByteBuffer userData) {
             this(CONSUMER_PROTOCOL_V1, topics, userData);
         }
@@ -188,6 +184,10 @@ public void setGroupInstanceId(Optional groupInstanceId) {
             this.groupInstanceId = groupInstanceId;
         }
 
+        void setOwnedPartitions(List ownedPartitions) {
+            this.ownedPartitions = ownedPartitions;
+        }
+
         Short version() {
             return version;
         }

From ddfe28f9dd5ce26f8f189946e5c5d90b627e33ca Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Thu, 25 Jul 2019 18:18:22 -0700
Subject: [PATCH 42/59] add unit tests

---
 .../kafka/clients/consumer/KafkaConsumer.java |  8 +++
 .../internals/AbstractCoordinator.java        |  6 ++
 .../internals/ConsumerCoordinator.java        | 17 ++++-
 .../internals/ConsumerCoordinatorTest.java    | 66 +++++++++++++++++--
 4 files changed, 90 insertions(+), 7 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index e8713a78df59f..675e13672b89f 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -1045,6 +1045,8 @@ public void subscribe(Pattern pattern) {
     /**
      * Unsubscribe from topics currently subscribed with {@link #subscribe(Collection)} or {@link #subscribe(Pattern)}.
      * This also clears any partitions directly assigned through {@link #assign(Collection)}.
+     *
+     * @throws org.apache.kafka.common.KafkaException for any other unrecoverable errors (e.g. rebalance callback errors)
      */
     public void unsubscribe() {
         acquireAndEnsureOpen();
@@ -1190,6 +1192,9 @@ public ConsumerRecords poll(final Duration timeout) {
         return poll(time.timer(timeout), true);
     }
 
+    /**
+     * @throws KafkaException if the rebalance callback throws exception
+     */
     private ConsumerRecords poll(final Timer timer, final boolean includeMetadataInTimeout) {
         acquireAndEnsureOpen();
         try {
@@ -1244,6 +1249,9 @@ boolean updateAssignmentMetadataIfNeeded(final Timer timer) {
         return updateFetchPositions(timer);
     }
 
+    /**
+     * @throws KafkaException if the rebalance callback throws exception
+     */
     private Map>> pollForFetches(Timer timer) {
         long pollTimeout = coordinator == null ? timer.remainingMs() :
                 Math.min(coordinator.timeToNextPoll(timer.currentTimeMs()), timer.remainingMs());
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 32556ee66cead..035968f882067 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -313,6 +313,7 @@ public void ensureActiveGroup() {
      * Ensure the group is active (i.e., joined and synced)
      *
      * @param timer Timer bounding how long this method can block
+     * @throws KafkaException if the callback throws exception
      * @return true iff the group is active
      */
     boolean ensureActiveGroup(final Timer timer) {
@@ -361,6 +362,7 @@ private void closeHeartbeatThread() {
      * Visible for testing.
      *
      * @param timer Timer bounding how long this method can block
+     * @throws KafkaException if the callback throws exception
      * @return true iff the operation succeeded
      */
     boolean joinGroupIfNeeded(final Timer timer) {
@@ -818,6 +820,9 @@ public final void close() {
         close(time.timer(0));
     }
 
+    /**
+     * @throws KafkaException if the rebalance callback throws exception
+     */
     protected void close(Timer timer) {
         try {
             closeHeartbeatThread();
@@ -844,6 +849,7 @@ protected void close(Timer timer) {
     /**
      * Leave the current group and reset local generation/memberId.
      * @param leaveReason reason to attempt leaving the group
+     * @throws KafkaException if the callback throws exception
      */
     public synchronized void maybeLeaveGroup(String leaveReason) {
         // Starting from 2.3, only dynamic members will send LeaveGroupRequest to the broker,
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 37696a8029c45..c4a9e3cf06a91 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -418,6 +418,7 @@ void maybeUpdateSubscriptionMetadata() {
      * Returns early if the timeout expires
      *
      * @param timer Timer bounding how long this method can block
+     * @throws KafkaException if the rebalance callback throws exception
      * @return true iff the operation succeeded
      */
     public boolean poll(Timer timer) {
@@ -662,6 +663,9 @@ protected void onJoinPrepare(int generation, String memberId) {
         }
     }
 
+    /**
+     * @throws KafkaException if the rebalance callback throws exception
+     */
     @Override
     public void resetGeneration(String errorMessage) {
         if (subscriptions.partitionsAutoAssigned()) {
@@ -679,6 +683,9 @@ public void resetGeneration(String errorMessage) {
         super.resetGeneration(errorMessage);
     }
 
+    /**
+     * @throws KafkaException if the callback throws exception
+     */
     @Override
     public boolean rejoinNeededOrPending() {
         if (!subscriptions.partitionsAutoAssigned())
@@ -788,6 +795,9 @@ public Map fetchCommittedOffsets(final Set partitionsPerTopic = new HashMap<>();
-            for (String topic : subscription.groupSubscription())
-                partitionsPerTopic.put(topic, cluster.partitionCountForTopic(topic));
+            for (String topic : subscription.groupSubscription()) {
+                Integer numPartitions = cluster.partitionCountForTopic(topic);
+                if (numPartitions != null)
+                    partitionsPerTopic.put(topic, numPartitions);
+            }
             this.partitionsPerTopic = partitionsPerTopic;
             this.version = version;
         }
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index e4aaf0e0cbaf8..3607964e018a3 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -389,10 +389,12 @@ public void testIllegalGeneration() {
         assertTrue(future.failed());
         assertEquals(Errors.ILLEGAL_GENERATION.exception(), future.exception());
         assertTrue(coordinator.rejoinNeededOrPending());
+        assertEquals(1, rebalanceListener.lostCount);
+        assertEquals(Collections.singleton(t1p), rebalanceListener.lost);
     }
 
     @Test
-    public void testUnknownConsumerId() {
+    public void testUnknownMemberId() {
         client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
         coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
 
@@ -413,6 +415,8 @@ public void testUnknownConsumerId() {
         assertTrue(future.failed());
         assertEquals(Errors.UNKNOWN_MEMBER_ID.exception(), future.exception());
         assertTrue(coordinator.rejoinNeededOrPending());
+        assertEquals(1, rebalanceListener.lostCount);
+        assertEquals(Collections.singleton(t1p), rebalanceListener.lost);
     }
 
     @Test
@@ -648,7 +652,7 @@ public boolean matches(AbstractRequest body) {
 
         List newAssigned = Arrays.asList(t1p, t2p);
 
-        Map> updatedSubscriptions = singletonMap(consumerId, Arrays.asList(topic1, topic2));
+        final Map> updatedSubscriptions = singletonMap(consumerId, Arrays.asList(topic1, topic2));
         partitionAssignor.prepare(singletonMap(consumerId, newAssigned));
 
         // we expect to see a second rebalance with the new-found topics
@@ -667,19 +671,65 @@ public boolean matches(AbstractRequest body) {
                 return subscription.topics().containsAll(updatedSubscription);
             }
         }, joinGroupLeaderResponse(2, consumerId, updatedSubscriptions, Errors.NONE));
-        client.prepareResponse(syncGroupResponse(newAssigned, Errors.NONE));
+        client.prepareResponse(new MockClient.RequestMatcher() {
+            // update the metadata again back to topic1
+            @Override
+            public boolean matches(AbstractRequest body) {
+                client.updateMetadata(TestUtils.metadataUpdateWith(1, singletonMap(topic1, 1)));
+                return true;
+            }
+        }, syncGroupResponse(newAssigned, Errors.NONE));
 
         coordinator.poll(time.timer(Long.MAX_VALUE));
 
         Collection revoked = getRevoked(oldAssigned, newAssigned);
+        int revokedCount = revoked.isEmpty() ? 0 : 1;
 
         assertFalse(coordinator.rejoinNeededOrPending());
         assertEquals(toSet(updatedSubscription), subscriptions.subscription());
         assertEquals(toSet(newAssigned), subscriptions.assignedPartitions());
-        assertEquals(revoked.isEmpty() ? 0 : 1, rebalanceListener.revokedCount);
+        assertEquals(revokedCount, rebalanceListener.revokedCount);
         assertEquals(revoked.isEmpty() ? null : revoked, rebalanceListener.revoked);
         assertEquals(2, rebalanceListener.assignedCount);
         assertEquals(getAdded(oldAssigned, newAssigned), rebalanceListener.assigned);
+
+        // we expect to see a third rebalance with the new-found topics
+        partitionAssignor.prepare(singletonMap(consumerId, oldAssigned));
+
+        client.prepareResponse(new MockClient.RequestMatcher() {
+            @Override
+            public boolean matches(AbstractRequest body) {
+                JoinGroupRequest join = (JoinGroupRequest) body;
+                Iterator protocolIterator =
+                    join.data().protocols().iterator();
+                assertTrue(protocolIterator.hasNext());
+                JoinGroupRequestData.JoinGroupRequestProtocol protocolMetadata = protocolIterator.next();
+
+                ByteBuffer metadata = ByteBuffer.wrap(protocolMetadata.metadata());
+                ConsumerPartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(metadata);
+                metadata.rewind();
+                return subscription.topics().contains(topic1);
+            }
+        }, joinGroupLeaderResponse(3, consumerId, initialSubscription, Errors.NONE));
+        client.prepareResponse(syncGroupResponse(oldAssigned, Errors.NONE));
+
+        coordinator.poll(time.timer(Long.MAX_VALUE));
+
+        Collection lost = Collections.singleton(t2p);
+        revoked = getRevoked(newAssigned, oldAssigned);
+        revoked.removeAll(lost);
+        revokedCount += revoked.isEmpty() ? 0 : 1;
+        Collection added = getAdded(newAssigned, oldAssigned);
+
+        assertFalse(coordinator.rejoinNeededOrPending());
+        assertEquals(singleton(topic1), subscriptions.subscription());
+        assertEquals(toSet(oldAssigned), subscriptions.assignedPartitions());
+        assertEquals(revokedCount, rebalanceListener.revokedCount);
+        assertEquals(revoked.isEmpty() ? null : revoked, rebalanceListener.revoked);
+        assertEquals(added.isEmpty() ? 2 : 3, rebalanceListener.assignedCount);
+        assertEquals(added.isEmpty() ? getAdded(oldAssigned, newAssigned) : added, rebalanceListener.assigned);
+        assertEquals(1, rebalanceListener.lostCount);
+        assertEquals(lost, rebalanceListener.lost);
     }
 
     @Test
@@ -2237,8 +2287,9 @@ private ConsumerCoordinator prepareCoordinatorForCloseTest(final boolean useGrou
             client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE));
             client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE));
             coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE));
-        } else
+        } else {
             subscriptions.assignFromUser(singleton(t1p));
+        }
 
         subscriptions.seek(t1p, 100);
         coordinator.poll(time.timer(Long.MAX_VALUE));
@@ -2317,6 +2368,11 @@ public boolean matches(AbstractRequest body) {
         coordinator.close();
         assertTrue("Commit not requested", commitRequested.get());
         assertEquals("leaveGroupRequested should be " + shouldLeaveGroup, shouldLeaveGroup, leaveGroupRequested.get());
+
+        if (shouldLeaveGroup) {
+            assertEquals(1, rebalanceListener.lostCount);
+            assertEquals(singleton(t1p), rebalanceListener.lost);
+        }
     }
 
     private ConsumerCoordinator buildCoordinator(final GroupRebalanceConfig rebalanceConfig,

From 4e466908057ddb8db517e54c5689bb64eaf411e3 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Mon, 29 Jul 2019 11:35:03 -0700
Subject: [PATCH 43/59] fix unit tests

---
 .../internals/ConsumerCoordinator.java        | 28 +++++++++----------
 .../kafka/api/AbstractConsumerTest.scala      | 10 +++----
 .../kafka/api/ConsumerBounceTest.scala        |  9 ++----
 .../kafka/api/PlaintextConsumerTest.scala     |  4 +--
 .../processor/internals/StreamThread.java     |  2 +-
 5 files changed, 22 insertions(+), 31 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index c4a9e3cf06a91..77e37873ce328 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -259,19 +259,17 @@ private void maybeUpdateJoinedSubscription(Set assignedPartition
     }
 
     private Exception maybeInvokePartitionsAssigned(final Set assignedPartitions) {
-        if (!assignedPartitions.isEmpty()) {
-            log.info("Adding newly assigned partitions: {}", Utils.join(assignedPartitions, ", "));
+        log.info("Adding newly assigned partitions: {}", Utils.join(assignedPartitions, ", "));
 
-            ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
-            try {
-                listener.onPartitionsAssigned(assignedPartitions);
-            } catch (WakeupException | InterruptException e) {
-                throw e;
-            } catch (Exception e) {
-                log.error("User provided listener {} failed on invocation of onPartitionsAssigned for partitions {}",
-                    listener.getClass().getName(), assignedPartitions, e);
-                return e;
-            }
+        ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
+        try {
+            listener.onPartitionsAssigned(assignedPartitions);
+        } catch (WakeupException | InterruptException e) {
+            throw e;
+        } catch (Exception e) {
+            log.error("User provided listener {} failed on invocation of onPartitionsAssigned for partitions {}",
+                listener.getClass().getName(), assignedPartitions, e);
+            return e;
         }
 
         return null;
@@ -378,12 +376,12 @@ protected void onJoinComplete(int generation,
                     Utils.join(addedPartitions, ", "),
                     Utils.join(revokedPartitions, ", "));
 
-                // add partitions that were not previously owned but are now assigned
-                firstException.compareAndSet(null, maybeInvokePartitionsAssigned(addedPartitions));
-
                 // revoked partitions that was previously owned but no longer assigned
                 firstException.compareAndSet(null, maybeInvokePartitionsRevoked(revokedPartitions));
 
+                // add partitions that were not previously owned but are now assigned
+                firstException.compareAndSet(null, maybeInvokePartitionsAssigned(addedPartitions));
+
                 // if revoked any partitions, need to re-join the group afterwards
                 if (!revokedPartitions.isEmpty()) {
                     requestRejoin();
diff --git a/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala b/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala
index 210ceb2d4d006..b616744628620 100644
--- a/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala
@@ -332,28 +332,28 @@ abstract class AbstractConsumerTest extends BaseRequestTest {
     @volatile var thrownException: Option[Throwable] = None
     @volatile var receivedMessages = 0
 
-    @volatile private var partitionAssignment: Set[TopicPartition] = partitionsToAssign
+    @volatile private var partitionAssignment: mutable.Set[TopicPartition] = new mutable.HashSet[TopicPartition]()
     @volatile private var subscriptionChanged = false
     private var topicsSubscription = topicsToSubscribe
 
     val rebalanceListener: ConsumerRebalanceListener = new ConsumerRebalanceListener {
       override def onPartitionsAssigned(partitions: util.Collection[TopicPartition]) = {
-        partitionAssignment = collection.immutable.Set(consumer.assignment().asScala.toArray: _*)
+        partitionAssignment ++= partitions.toArray(new Array[TopicPartition](0))
       }
 
       override def onPartitionsRevoked(partitions: util.Collection[TopicPartition]) = {
-        partitionAssignment = Set.empty[TopicPartition]
+        partitionAssignment --= partitions.toArray(new Array[TopicPartition](0))
       }
     }
 
-    if (partitionAssignment.isEmpty) {
+    if (partitionsToAssign.isEmpty) {
       consumer.subscribe(topicsToSubscribe.asJava, rebalanceListener)
     } else {
       consumer.assign(partitionAssignment.asJava)
     }
 
     def consumerAssignment(): Set[TopicPartition] = {
-      partitionAssignment
+      partitionAssignment.toSet
     }
 
     /**
diff --git a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
index 778e3b885ccac..1dbc20a6bc3b8 100644
--- a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
@@ -384,13 +384,8 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
 
     def subscribeAndPoll(consumer: KafkaConsumer[Array[Byte], Array[Byte]], revokeSemaphore: Option[Semaphore] = None): Future[Any] = {
       executor.submit(CoreUtils.runnable {
-          consumer.subscribe(Collections.singletonList(topic), new ConsumerRebalanceListener {
-            def onPartitionsAssigned(partitions: Collection[TopicPartition]) {
-            }
-            def onPartitionsRevoked(partitions: Collection[TopicPartition]) {
-              revokeSemaphore.foreach(s => s.release())
-            }
-          })
+        consumer.subscribe(Collections.singletonList(topic))
+        revokeSemaphore.foreach(s => s.release())
         // requires to used deprecated `poll(long)` to trigger metadata update
           consumer.poll(0L)
         }, 0)
diff --git a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
index 33c14ebe12ff0..613ff110d4ae9 100644
--- a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
@@ -185,14 +185,12 @@ class PlaintextConsumerTest extends BaseConsumerTest {
     // rebalance to get the initial assignment
     awaitRebalance(consumer, listener)
     assertEquals(1, listener.callsToAssigned)
-    assertEquals(1, listener.callsToRevoked)
 
     Thread.sleep(3500)
 
     // we should fall out of the group and need to rebalance
     awaitRebalance(consumer, listener)
     assertEquals(2, listener.callsToAssigned)
-    assertEquals(2, listener.callsToRevoked)
   }
 
   @Test
@@ -208,7 +206,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
 
     val listener = new TestConsumerReassignmentListener {
       override def onPartitionsRevoked(partitions: util.Collection[TopicPartition]): Unit = {
-        if (callsToRevoked > 0) {
+        if (!partitions.isEmpty) {
           // on the second rebalance (after we have joined the group initially), sleep longer
           // than session timeout and then try a commit. We should still be in the group,
           // so the commit should succeed
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index 1633b30223b62..02df3781708ed 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -305,7 +305,7 @@ public void onPartitionsAssigned(final Collection assignment) {
         }
 
         @Override
-        public void onPartitionsRevoked(final Collection assignment) {
+        public void  onPartitionsRevoked(final Collection assignment) {
             log.debug("at state {}: partitions {} revoked at the beginning of consumer rebalance.\n" +
                     "\tcurrent assigned active tasks: {}\n" +
                     "\tcurrent assigned standby tasks: {}\n",

From 0c2486ce6bd185e06ea83a537014b3dc503ad5b2 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Mon, 29 Jul 2019 14:20:05 -0700
Subject: [PATCH 44/59] change update assignment ordering

---
 .../internals/ConsumerCoordinator.java        | 60 +++++++++++--------
 .../consumer/internals/SubscriptionState.java | 46 +++++++-------
 .../internals/SubscriptionStateTest.java      | 40 +++++++++----
 .../kafka/api/PlaintextConsumerTest.scala     |  1 +
 4 files changed, 88 insertions(+), 59 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 77e37873ce328..109f2bd9b26bf 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -235,26 +235,28 @@ private ConsumerPartitionAssignor lookupAssignor(String name) {
     }
 
     private void maybeUpdateJoinedSubscription(Set assignedPartitions) {
-        // Check if the assignment contains some topics that were not in the original
-        // subscription, if yes we will obey what leader has decided and add these topics
-        // into the subscriptions as long as they still match the subscribed pattern
-
-        Set addedTopics = new HashSet<>();
-        // this is a copy because its handed to listener below
-        for (TopicPartition tp : assignedPartitions) {
-            if (!joinedSubscription.contains(tp.topic()))
-                addedTopics.add(tp.topic());
-        }
+        if (subscriptions.hasPatternSubscription()) {
+            // Check if the assignment contains some topics that were not in the original
+            // subscription, if yes we will obey what leader has decided and add these topics
+            // into the subscriptions as long as they still match the subscribed pattern
+
+            Set addedTopics = new HashSet<>();
+            // this is a copy because its handed to listener below
+            for (TopicPartition tp : assignedPartitions) {
+                if (!joinedSubscription.contains(tp.topic()))
+                    addedTopics.add(tp.topic());
+            }
 
-        if (!addedTopics.isEmpty()) {
-            Set newSubscription = new HashSet<>(subscriptions.subscription());
-            Set newJoinedSubscription = new HashSet<>(joinedSubscription);
-            newSubscription.addAll(addedTopics);
-            newJoinedSubscription.addAll(addedTopics);
+            if (!addedTopics.isEmpty()) {
+                Set newSubscription = new HashSet<>(subscriptions.subscription());
+                Set newJoinedSubscription = new HashSet<>(joinedSubscription);
+                newSubscription.addAll(addedTopics);
+                newJoinedSubscription.addAll(addedTopics);
 
-            if (this.subscriptions.subscribeFromPattern(newSubscription))
-                metadata.requestUpdateForNewTopics();
-            this.joinedSubscription = newJoinedSubscription;
+                if (this.subscriptions.subscribeFromPattern(newSubscription))
+                    metadata.requestUpdateForNewTopics();
+                this.joinedSubscription = newJoinedSubscription;
+            }
         }
     }
 
@@ -330,7 +332,9 @@ protected void onJoinComplete(int generation,
 
         Assignment assignment = ConsumerProtocol.deserializeAssignment(assignmentBuffer);
 
-        if (!subscriptions.assignFromSubscribed(assignment.partitions())) {
+        Set assignedPartitions = new HashSet<>(assignment.partitions());
+
+        if (!subscriptions.checkAssignmentMatchedSubscription(assignedPartitions)) {
             log.warn("We received an assignment {} that doesn't match our current subscription {}; it is likely " +
                 "that the subscription has changed since we joined the group. Will try re-join the group with current subscription",
                 assignment.partitions(), subscriptions.prettyString());
@@ -340,8 +344,6 @@ protected void onJoinComplete(int generation,
             return;
         }
 
-        Set assignedPartitions = subscriptions.assignedPartitions();
-
         // The leader may have assigned partitions which match our subscription pattern, but which
         // were not explicitly requested, so we update the joined subscription here.
         maybeUpdateJoinedSubscription(assignedPartitions);
@@ -358,9 +360,13 @@ protected void onJoinComplete(int generation,
         final AtomicReference firstException = new AtomicReference<>(null);
         Set addedPartitions = new HashSet<>(assignedPartitions);
         addedPartitions.removeAll(ownedPartitions);
+
+        // note that we should only change the assignment AFTER we've triggered the rebalance callback
         switch (protocol) {
             case EAGER:
                 // assign partitions that are not yet owned
+                subscriptions.assignFromSubscribed(assignedPartitions);
+
                 firstException.compareAndSet(null, maybeInvokePartitionsAssigned(addedPartitions));
 
                 break;
@@ -379,6 +385,8 @@ protected void onJoinComplete(int generation,
                 // revoked partitions that was previously owned but no longer assigned
                 firstException.compareAndSet(null, maybeInvokePartitionsRevoked(revokedPartitions));
 
+                subscriptions.assignFromSubscribed(assignedPartitions);
+
                 // add partitions that were not previously owned but are now assigned
                 firstException.compareAndSet(null, maybeInvokePartitionsAssigned(addedPartitions));
 
@@ -627,13 +635,17 @@ protected void onJoinPrepare(int generation, String memberId) {
         // execute the user's callback before rebalance
         final Set revokedPartitions;
         final AtomicReference firstException = new AtomicReference<>(null);
+
+        // note we should only change the assignment AFTER the callback is triggered
+        // so that users can still access the pre-assigned partitions to commit offsets etc.
         switch (protocol) {
             case EAGER:
                 // revoke all partitions
                 revokedPartitions = new HashSet<>(subscriptions.assignedPartitions());
+                firstException.compareAndSet(null, maybeInvokePartitionsRevoked(revokedPartitions));
+
                 subscriptions.assignFromSubscribed(Collections.emptySet());
 
-                firstException.compareAndSet(null, maybeInvokePartitionsRevoked(revokedPartitions));
                 break;
 
             case COOPERATIVE:
@@ -644,10 +656,10 @@ protected void onJoinPrepare(int generation, String memberId) {
                     .collect(Collectors.toSet());
 
                 if (!revokedPartitions.isEmpty()) {
+                    firstException.compareAndSet(null, maybeInvokePartitionsRevoked(revokedPartitions));
+
                     ownedPartitions.removeAll(revokedPartitions);
                     subscriptions.assignFromSubscribed(ownedPartitions);
-
-                    firstException.compareAndSet(null, maybeInvokePartitionsRevoked(revokedPartitions));
                 }
 
                 break;
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
index af834ce71b1db..3a33406ee57ce 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
@@ -238,42 +238,42 @@ public synchronized boolean assignFromUser(Set partitions) {
     }
 
     /**
-     * Change the assignment to the specified partitions returned from the coordinator, note this is
-     * different from {@link #assignFromUser(Set)} which directly set the assignment from user inputs.
-     *
      * @return true if assignments matches subscription, otherwise false
      */
-    public synchronized boolean assignFromSubscribed(Collection assignments) {
-        if (!this.partitionsAutoAssigned())
-            throw new IllegalArgumentException("Attempt to dynamically assign partitions while manual assignment in use");
-
-        boolean assignmentMatchedSubscription = true;
+    public synchronized boolean checkAssignmentMatchedSubscription(Collection assignments) {
         for (TopicPartition topicPartition : assignments) {
             if (this.subscribedPattern != null) {
-                assignmentMatchedSubscription = this.subscribedPattern.matcher(topicPartition.topic()).matches();
-                if (!assignmentMatchedSubscription) {
+                if (!this.subscribedPattern.matcher(topicPartition.topic()).matches()) {
                     log.info("Assigned partition {} for non-subscribed topic regex pattern; subscription pattern is {}",
-                            topicPartition,
-                            this.subscribedPattern);
-                    break;
+                        topicPartition,
+                        this.subscribedPattern);
+
+                    return false;
                 }
             } else {
-                assignmentMatchedSubscription = this.subscription.contains(topicPartition.topic());
-                if (!assignmentMatchedSubscription) {
+                if (!this.subscription.contains(topicPartition.topic())) {
                     log.info("Assigned partition {} for non-subscribed topic; subscription is {}", topicPartition, this.subscription);
-                    break;
+
+                    return false;
                 }
             }
         }
 
-        if (assignmentMatchedSubscription) {
-            Map assignedPartitionStates = partitionToStateMap(
-                    assignments);
-            assignmentId++;
-            this.assignment.set(assignedPartitionStates);
-        }
+        return true;
+    }
+
+    /**
+     * Change the assignment to the specified partitions returned from the coordinator, note this is
+     * different from {@link #assignFromUser(Set)} which directly set the assignment from user inputs.
+     */
+    public synchronized void assignFromSubscribed(Collection assignments) {
+        if (!this.partitionsAutoAssigned())
+            throw new IllegalArgumentException("Attempt to dynamically assign partitions while manual assignment in use");
 
-        return assignmentMatchedSubscription;
+
+        Map assignedPartitionStates = partitionToStateMap(assignments);
+        assignmentId++;
+        this.assignment.set(assignedPartitionStates);
     }
 
     private void registerRebalanceListener(ConsumerRebalanceListener listener) {
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java
index 484b9de0a9bce..c3ce02a79a155 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/SubscriptionStateTest.java
@@ -62,7 +62,7 @@ public void partitionAssignment() {
         state.seek(tp0, 1);
         assertTrue(state.isFetchable(tp0));
         assertEquals(1L, state.position(tp0).offset);
-        state.assignFromUser(Collections.emptySet());
+        state.assignFromUser(Collections.emptySet());
         assertTrue(state.assignedPartitions().isEmpty());
         assertEquals(0, state.numAssignedPartitions());
         assertFalse(state.isAssigned(tp0));
@@ -88,7 +88,8 @@ public void partitionAssignmentChangeOnTopicSubscription() {
         assertTrue(state.assignedPartitions().isEmpty());
         assertEquals(0, state.numAssignedPartitions());
 
-        assertTrue(state.assignFromSubscribed(singleton(t1p0)));
+        assertTrue(state.checkAssignmentMatchedSubscription(singleton(t1p0)));
+        state.assignFromSubscribed(singleton(t1p0));
         // assigned partitions should immediately change
         assertEquals(singleton(t1p0), state.assignedPartitions());
         assertEquals(1, state.numAssignedPartitions());
@@ -116,13 +117,17 @@ public void partitionAssignmentChangeOnPatternSubscription() {
         assertTrue(state.assignedPartitions().isEmpty());
         assertEquals(0, state.numAssignedPartitions());
 
-        assertTrue(state.assignFromSubscribed(singleton(tp1)));
+        assertTrue(state.checkAssignmentMatchedSubscription(singleton(tp1)));
+        state.assignFromSubscribed(singleton(tp1));
+
         // assigned partitions should immediately change
         assertEquals(singleton(tp1), state.assignedPartitions());
         assertEquals(1, state.numAssignedPartitions());
         assertEquals(singleton(topic), state.subscription());
 
-        assertTrue(state.assignFromSubscribed(Collections.singletonList(t1p0)));
+        assertTrue(state.checkAssignmentMatchedSubscription(singleton(t1p0)));
+        state.assignFromSubscribed(singleton(t1p0));
+
         // assigned partitions should immediately change
         assertEquals(singleton(t1p0), state.assignedPartitions());
         assertEquals(1, state.numAssignedPartitions());
@@ -138,7 +143,9 @@ public void partitionAssignmentChangeOnPatternSubscription() {
         assertEquals(singleton(t1p0), state.assignedPartitions());
         assertEquals(1, state.numAssignedPartitions());
 
-        assertTrue(state.assignFromSubscribed(Collections.singletonList(tp0)));
+        assertTrue(state.checkAssignmentMatchedSubscription(singleton(tp0)));
+        state.assignFromSubscribed(singleton(tp0));
+
         // assigned partitions should immediately change
         assertEquals(singleton(tp0), state.assignedPartitions());
         assertEquals(1, state.numAssignedPartitions());
@@ -164,7 +171,8 @@ public void verifyAssignmentId() {
 
         Set autoAssignment = Utils.mkSet(t1p0);
         state.subscribe(singleton(topic1), rebalanceListener);
-        assertTrue(state.assignFromSubscribed(autoAssignment));
+        assertTrue(state.checkAssignmentMatchedSubscription(autoAssignment));
+        state.assignFromSubscribed(autoAssignment);
         assertEquals(3, state.assignmentId());
         assertEquals(autoAssignment, state.assignedPartitions());
     }
@@ -192,10 +200,14 @@ public void topicSubscription() {
         assertTrue(state.assignedPartitions().isEmpty());
         assertEquals(0, state.numAssignedPartitions());
         assertTrue(state.partitionsAutoAssigned());
-        assertTrue(state.assignFromSubscribed(singleton(tp0)));
+        assertTrue(state.checkAssignmentMatchedSubscription(singleton(tp0)));
+        state.assignFromSubscribed(singleton(tp0));
+
         state.seek(tp0, 1);
         assertEquals(1L, state.position(tp0).offset);
-        assertTrue(state.assignFromSubscribed(singleton(tp1)));
+        assertTrue(state.checkAssignmentMatchedSubscription(singleton(tp1)));
+        state.assignFromSubscribed(singleton(tp1));
+
         assertTrue(state.isAssigned(tp1));
         assertFalse(state.isAssigned(tp0));
         assertFalse(state.isFetchable(tp1));
@@ -217,21 +229,23 @@ public void partitionPause() {
     @Test(expected = IllegalStateException.class)
     public void invalidPositionUpdate() {
         state.subscribe(singleton(topic), rebalanceListener);
-        assertTrue(state.assignFromSubscribed(singleton(tp0)));
+        assertTrue(state.checkAssignmentMatchedSubscription(singleton(tp0)));
+        state.assignFromSubscribed(singleton(tp0));
+
         state.position(tp0, new SubscriptionState.FetchPosition(0, Optional.empty(), leaderAndEpoch));
     }
 
     @Test
     public void cantAssignPartitionForUnsubscribedTopics() {
         state.subscribe(singleton(topic), rebalanceListener);
-        assertFalse(state.assignFromSubscribed(Collections.singletonList(t1p0)));
+        assertFalse(state.checkAssignmentMatchedSubscription(Collections.singletonList(t1p0)));
     }
 
     @Test
     public void cantAssignPartitionForUnmatchedPattern() {
         state.subscribe(Pattern.compile(".*t"), rebalanceListener);
         state.subscribeFromPattern(new HashSet<>(Collections.singletonList(topic)));
-        assertFalse(state.assignFromSubscribed(Collections.singletonList(t1p0)));
+        assertFalse(state.checkAssignmentMatchedSubscription(Collections.singletonList(t1p0)));
     }
 
     @Test(expected = IllegalStateException.class)
@@ -291,7 +305,9 @@ public void unsubscribeUserSubscribe() {
     public void unsubscription() {
         state.subscribe(Pattern.compile(".*"), rebalanceListener);
         state.subscribeFromPattern(new HashSet<>(Arrays.asList(topic, topic1)));
-        assertTrue(state.assignFromSubscribed(singleton(tp1)));
+        assertTrue(state.checkAssignmentMatchedSubscription(singleton(tp1)));
+        state.assignFromSubscribed(singleton(tp1));
+
         assertEquals(singleton(tp1), state.assignedPartitions());
         assertEquals(1, state.numAssignedPartitions());
 
diff --git a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
index 613ff110d4ae9..4ad59d5bacddd 100644
--- a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
@@ -205,6 +205,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
     var committedPosition: Long = -1
 
     val listener = new TestConsumerReassignmentListener {
+      override def onPartitionsLost(partitions: util.Collection[TopicPartition]): Unit = {}
       override def onPartitionsRevoked(partitions: util.Collection[TopicPartition]): Unit = {
         if (!partitions.isEmpty) {
           // on the second rebalance (after we have joined the group initially), sleep longer

From 60f8c9cb66d242151643b44e3164e714a7362e47 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Mon, 29 Jul 2019 19:20:32 -0700
Subject: [PATCH 45/59] add more java docs

---
 .../consumer/ConsumerPartitionAssignor.java   | 15 +++++
 .../consumer/ConsumerRebalanceListener.java   | 56 +++++++++++--------
 .../internals/AbstractCoordinator.java        | 20 +++++--
 .../internals/ConsumerCoordinator.java        | 38 ++++++++-----
 .../internals/ConsumerCoordinatorTest.java    | 16 +++---
 .../main/scala/kafka/tools/MirrorMaker.scala  |  2 +
 .../kafka/api/AbstractConsumerTest.scala      |  4 +-
 .../kafka/api/PlaintextConsumerTest.scala     |  1 +
 8 files changed, 99 insertions(+), 53 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
index 72d5d6e806bd7..be3b336d2af68 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
@@ -178,6 +178,21 @@ public Map groupAssignment() {
         }
     }
 
+    /**
+     * {@link RebalanceProtocol#EAGER} rebalance protocol will let a consumer to always revoke all its owned partitions
+     * before participating the rebalance event, and therefore allows a complete reshuffling of the assignment.
+     *
+     * {@link RebalanceProtocol#COOPERATIVE} rebalance protocol will let a consumer to retain its currently assigned
+     * partitions before participating the rebalance event, as a result the assignor would not reassign its owned
+     * partitions immediately but may only indicate it to revoke first before the next rebalance event, and therefore is
+     * suitable for cooperative adjustments of assignments.
+     * {@link ConsumerPartitionAssignor} implementors who claim to support {@link RebalanceProtocol#COOPERATIVE} protocol
+     * from {@link ConsumerPartitionAssignor#supportedProtocols()} needs to make sure its
+     * {@link ConsumerPartitionAssignor#assign(Cluster, GroupSubscription)} logic respects the specified
+     * {@link Subscription#ownedPartitions()} list and not reassign them to other consumers; instead it can decide
+     * whether to remove them from the new assignment of the current owner consumer so that they can be revoked first
+     * before reassigned to other consumers in future rebalances.
+     */
     enum RebalanceProtocol {
         EAGER((byte) 0), COOPERATIVE((byte) 1);
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
index be862834680ff..38e5d31f6019a 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
@@ -28,8 +28,7 @@
  * those partitions will never be reassigned and this callback is not applicable.
  * 

  * When Kafka is managing the group membership, a partition re-assignment will be triggered any time the members of the group change or the subscription
- * of the members changes. This can occur when processes die, new process instances are added or old instances come back to life after failure causing
- * a rebalance event.
+ * of the members changes. This can occur when processes die, new process instances are added or old instances come back to life after failure.
  * Partition re-assignments can also be triggered by changes affecting the subscribed topics (e.g. when the number of partitions is
  * administratively adjusted).
  * 

@@ -48,7 +47,7 @@
  * This callback will only execute in the user thread as part of the {@link Consumer#poll(java.time.Duration) poll(long)} call
  * whenever partition assignment changes.
  * 

- * It is guaranteed that for any partition, if a partition is reassigned from one consumer to another, then the old consumer will
+ * Under normal conditions, if a partition is reassigned from one consumer to another, then the old consumer will
  * always invoke {@link #onPartitionsRevoked(Collection) onPartitionsRevoked} for that partition prior to the new consumer
  * invoking {@link #onPartitionsAssigned(Collection) onPartitionsAssigned} for the same partition. So if offsets or other state is saved in the
  * {@link #onPartitionsRevoked(Collection) onPartitionsRevoked} call by one consumer member, it will be always accessible by the time the
@@ -57,13 +56,19 @@
  * You can think of revocation as a graceful way to give up ownership of a partition. In some cases, the consumer may not have an opportunity to do so.
  * For example, if the session times out, then the partitions may be reassigned before we have a chance to revoke them gracefully.
  * For this case, we have a third callback {@link #onPartitionsLost(Collection)}. The difference between this function and
- * {@link #onPartitionsRevoked(Collection)} is that upon invocation of {@link #onPartitionsLost(Collection)}, the partitions are no longer owned
- * by the consumer but maybe by some other members in the group.
+ * {@link #onPartitionsRevoked(Collection)} is that upon invocation of {@link #onPartitionsLost(Collection)}, the partitions
+ * may already be owned by some other members in the group and therefore users would not be able to commit its consumed offsets for example.
  * Users could implement these two functions differently (by default,
  * {@link #onPartitionsLost(Collection)} will be calling {@link #onPartitionsRevoked(Collection)} directly); for example, in the
  * {@link #onPartitionsLost(Collection)} we should not need to store the offsets since we know these partitions are no longer owned by the consumer
  * at that time.
  * 

+ * During a rebalance event, the {@link #onPartitionsAssigned(Collection) onPartitionsAssigned} function will always be triggered exactly once when
+ * the rebalance completes. That is, even if there is no newly assigned partitions for a consumer member, its {@link #onPartitionsAssigned(Collection) onPartitionsAssigned}
+ * will still be triggered with an empty collection of partitions. As a result this function can be used also to notify when a rebalance event has happened.
+ * On the other hand, {@link #onPartitionsRevoked(Collection)} and {@link #onPartitionsLost(Collection)}
+ * will only be triggered when there are non-empty partitions revoked or lost from this consumer member during a rebalance event.
+ * 

  * It is possible
  * for a {@link org.apache.kafka.common.errors.WakeupException} or {@link org.apache.kafka.common.errors.InterruptException}
  * to be raised from one of these nested invocations. In this case, the exception will be propagated to the current
@@ -71,12 +76,12 @@
  * necessary to catch these exceptions and re-attempt to wakeup or interrupt the consumer thread.
  * Also if the callback function implementation itself throws an exception, this exception will be propagated to the current
  * invocation of {@link KafkaConsumer#poll(java.time.Duration)} as well.
- *
- * Note that the semantics of the callback is only for notifying the assignment change, and hence throwing any exceptions would
- * not affect any of the notified assignment changes at all. That means, if user captures the exception and
- * calls {@link KafkaConsumer#poll(java.time.Duration)} again, these callback functions would not be invoked any more unless there are
- * new partitions reassigned.
- *
+ * 

+ * Note that callbacks only serve as notification of an assignment change.
+ * They cannot be used to express acceptance of the change.
+ * Hence throwing an exception from a callback does not affect the assignment in any way,
+ * as it will be propagated all the way up to the {@link KafkaConsumer#poll(java.time.Duration)} call.
+ * If user captures the exception in the caller, the callback is still assumed successful and no further retries will be attempted.
  * 

  *
  * Here is pseudo-code for a callback implementation for saving offsets:
@@ -126,7 +131,8 @@ public interface ConsumerRebalanceListener {
      * invocation of {@link KafkaConsumer#poll(java.time.Duration)} in which this callback is being executed. This means it is not
      * necessary to catch these exceptions and re-attempt to wakeup or interrupt the consumer thread.
      *
-     * @param partitions The list of partitions that were assigned to the consumer and now need to be revoked
+     * @param partitions The list of partitions that were assigned to the consumer and now need to be revoked (may not
+     *                   include all currently assigned partitions, i.e. there may still be some partitions left)
      * @throws org.apache.kafka.common.errors.WakeupException If raised from a nested call to {@link KafkaConsumer}
      * @throws org.apache.kafka.common.errors.InterruptException If raised from a nested call to {@link KafkaConsumer}
      */
@@ -137,9 +143,12 @@ public interface ConsumerRebalanceListener {
      * partition re-assignment. This method will be called after the partition re-assignment completes and before the
      * consumer starts fetching data, and only as the result of a {@link Consumer#poll(java.time.Duration) poll(long)} call.
      * 

-     * It is guaranteed that all the processes in a consumer group will execute their
+     * It is guaranteed that under normal conditions all the processes in a consumer group will execute their
      * {@link #onPartitionsRevoked(Collection)} callback before any instance executes its
-     * {@link #onPartitionsAssigned(Collection)} callback.
+     * {@link #onPartitionsAssigned(Collection)} callback. During exceptional scenarios, partitions may be migrated
+     * without the old owner being notified (i.e. their {@link #onPartitionsRevoked(Collection)} callback not triggered),
+     * and later when the old owner consumer realized this event, the {@link #onPartitionsLost(Collection)} (Collection)} callback
+     * will be triggered by the consumer then.
      * 

      * It is common for the assignment callback to use the consumer instance in order to query offsets. It is possible
      * for a {@link org.apache.kafka.common.errors.WakeupException} or {@link org.apache.kafka.common.errors.InterruptException}
@@ -147,8 +156,8 @@ public interface ConsumerRebalanceListener {
      * invocation of {@link KafkaConsumer#poll(java.time.Duration)} in which this callback is being executed. This means it is not
      * necessary to catch these exceptions and re-attempt to wakeup or interrupt the consumer thread.
      *
-     * @param partitions The list of partitions that are now assigned to the consumer (may include partitions previously
-     *            assigned to the consumer)
+     * @param partitions The list of partitions that are now assigned to the consumer (previously owned partitions will
+     *                   NOT be included, i.e. this list will only include newly added partitions)
      * @throws org.apache.kafka.common.errors.WakeupException If raised from a nested call to {@link KafkaConsumer}
      * @throws org.apache.kafka.common.errors.InterruptException If raised from a nested call to {@link KafkaConsumer}
      */
@@ -156,24 +165,25 @@ public interface ConsumerRebalanceListener {
 
     /**
      * A callback method you can implement to provide handling of cleaning up resources for partitions that have already
-     * been re-assigned to other consumers. This method will not be called during normal execution as the owned partitions would
-     * first be revoked by calling the {@link ConsumerRebalanceListener#onPartitionsRevoked}, before being re-assigned
+     * been reassigned to other consumers. This method will not be called during normal execution as the owned partitions would
+     * first be revoked by calling the {@link ConsumerRebalanceListener#onPartitionsRevoked}, before being reassigned
      * to other consumers during a rebalance event. However, during exceptional scenarios when the consumer realized that it
      * does not own this partition any longer, i.e. not revoked via a normal rebalance event, then this method would be invoked.
-     *
+     * 

      * For example, this function is called if a consumer's session timeout has expired.
-     *
+     * 

      * By default it will just trigger {@link ConsumerRebalanceListener#onPartitionsRevoked}; for users who want to distinguish
      * the handling logic of revoked partitions v.s. lost partitions, they can override the default implementation.
-     *
+     * 

      * It is possible
      * for a {@link org.apache.kafka.common.errors.WakeupException} or {@link org.apache.kafka.common.errors.InterruptException}
      * to be raised from one of these nested invocations. In this case, the exception will be propagated to the current
      * invocation of {@link KafkaConsumer#poll(java.time.Duration)} in which this callback is being executed. This means it is not
      * necessary to catch these exceptions and re-attempt to wakeup or interrupt the consumer thread.
      *
-     * @param partitions The list of partitions that were assigned to the consumer and now have been re-assigned
-     *                  to other consumers
+     * @param partitions The list of partitions that were assigned to the consumer and now have been reassigned
+     *                   to other consumers (may not include all currently assigned partitions, i.e. there may still
+     *                   be some partitions left)
      * @throws org.apache.kafka.common.errors.WakeupException If raised from a nested call to {@link KafkaConsumer}
      * @throws org.apache.kafka.common.errors.InterruptException If raised from a nested call to {@link KafkaConsumer}
      */
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 035968f882067..18941ce39e531 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -797,15 +797,23 @@ final synchronized boolean hasValidMemberId() {
     }
 
 
+    public synchronized void resetGeneration(String causeMessage, boolean dueToError) {
+        if (dueToError) {
+            log.debug("Resetting generation and requesting re-join group due to {}", causeMessage);
+        } else {
+            log.debug("Resetting generation pro-actively due to {}", causeMessage);
+        }
+
+        this.generation = Generation.NO_GENERATION;
+        this.state = MemberState.UNJOINED;
+        this.rejoinNeeded = true;
+    }
+
     /**
      * Reset the generation and memberId because we have fallen out of the group.
      */
     protected synchronized void resetGeneration(String errorMessage) {
-        log.debug("Resetting generation and requesting re-join group due to {}", errorMessage);
-
-        this.generation = Generation.NO_GENERATION;
-        this.rejoinNeeded = true;
-        this.state = MemberState.UNJOINED;
+        resetGeneration(errorMessage, true);
     }
 
     protected synchronized void requestRejoin() {
@@ -868,7 +876,7 @@ public synchronized void maybeLeaveGroup(String leaveReason) {
             client.pollNoWakeup();
         }
 
-        resetGeneration("consumer proactively leaving group");
+        resetGeneration("consumer proactively leaving group", false);
     }
 
     protected boolean isDynamicMember() {
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 109f2bd9b26bf..e0244858a1d06 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -424,7 +424,7 @@ void maybeUpdateSubscriptionMetadata() {
      * Returns early if the timeout expires
      *
      * @param timer Timer bounding how long this method can block
-     * @throws KafkaException if the rebalance callback throws exception
+     * @throws KafkaException if the rebalance callback throws an exception
      * @return true iff the operation succeeded
      */
     public boolean poll(Timer timer) {
@@ -595,10 +595,10 @@ protected Map performAssignment(String leaderId,
     /**
      * Used by COOPERATIVE rebalance protocol only.
      *
-     * Validate the user assignor returned assignments such that there are no partitions which are going to
-     * be assigned to a different consumer other than its current owner (i.e. who includes it in its owned
-     * partitions list encoded in subscription); also if there are any revoked partitions set the error code to let
-     * everyone retry.
+     * Validate the assignments returned by the assignor such that no owned partitions are going to
+     * be reassigned to a different consumer directly: if the assignor wants to reassign an owned partition,
+     * it must first remove it from the new assignment of the current owner so that it is not assigned to any
+     * member, and then in the next rebalance it can finally reassign those partitions not owned by anyone to consumers.
      */
     private void validateCooperativeAssignment(final Map> ownedPartitions,
                                                final Map assignments) {
@@ -677,20 +677,27 @@ protected void onJoinPrepare(int generation, String memberId) {
      * @throws KafkaException if the rebalance callback throws exception
      */
     @Override
-    public void resetGeneration(String errorMessage) {
+    public void resetGeneration(String causeMessage, boolean dueToError) {
         if (subscriptions.partitionsAutoAssigned()) {
-            // lost all partitions if it is based on subscriptions
-            Set lostPartitions = new HashSet<>(subscriptions.assignedPartitions());
+            // if reset due to error then we have to give up all owned partitions as if they've lost;
+            // otherwise we can give them up as revoked
+            Set droppedPartitions = new HashSet<>(subscriptions.assignedPartitions());
             subscriptions.assignFromSubscribed(Collections.emptySet());
 
-            Exception e = maybeInvokePartitionsLost(errorMessage, lostPartitions);
+            Exception e;
+
+            if (dueToError) {
+                e = maybeInvokePartitionsLost(causeMessage, droppedPartitions);
+            } else {
+                e = maybeInvokePartitionsRevoked(droppedPartitions);
+            }
 
             if (e != null) {
                 throw new KafkaException("User rebalance callback throws an error", e);
             }
         }
 
-        super.resetGeneration(errorMessage);
+        super.resetGeneration(causeMessage, dueToError);
     }
 
     /**
@@ -701,19 +708,20 @@ public boolean rejoinNeededOrPending() {
         if (!subscriptions.partitionsAutoAssigned())
             return false;
 
-        // we need to rejoin if we performed the assignment and metadata has changed
+        // we need to rejoin if we performed the assignment and metadata has changed;
+        // also for those owned-but-no-longer-existed partitions we should drop them as lost
         if (assignmentSnapshot != null && !assignmentSnapshot.matches(metadataSnapshot)) {
             Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions());
-            Set lostPartitions = ownedPartitions.stream()
+            Set noLongerExistingPartitions = ownedPartitions.stream()
                 .filter(tp -> !metadataSnapshot.partitionsPerTopic().containsKey(tp.topic()))
                 .collect(Collectors.toSet());
 
-            if (!lostPartitions.isEmpty()) {
-                ownedPartitions.removeAll(lostPartitions);
+            if (!noLongerExistingPartitions.isEmpty()) {
+                ownedPartitions.removeAll(noLongerExistingPartitions);
                 subscriptions.assignFromSubscribed(ownedPartitions);
 
                 Exception e = maybeInvokePartitionsLost("topic metadata has changed and therefore some topics may not exist any more",
-                    lostPartitions);
+                    noLongerExistingPartitions);
 
                 if (e != null) {
                     throw new KafkaException("User rebalance callback throws an error", e);
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index 3607964e018a3..a113a44072f24 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -726,8 +726,8 @@ public boolean matches(AbstractRequest body) {
         assertEquals(toSet(oldAssigned), subscriptions.assignedPartitions());
         assertEquals(revokedCount, rebalanceListener.revokedCount);
         assertEquals(revoked.isEmpty() ? null : revoked, rebalanceListener.revoked);
-        assertEquals(added.isEmpty() ? 2 : 3, rebalanceListener.assignedCount);
-        assertEquals(added.isEmpty() ? getAdded(oldAssigned, newAssigned) : added, rebalanceListener.assigned);
+        assertEquals(3, rebalanceListener.assignedCount);
+        assertEquals(added, rebalanceListener.assigned);
         assertEquals(1, rebalanceListener.lostCount);
         assertEquals(lost, rebalanceListener.lost);
     }
@@ -1221,7 +1221,7 @@ public void onPartitionsAssigned(Collection partitions) {
 
         assertFalse(coordinator.rejoinNeededOrPending());
         assertEquals(0, rebalanceListener.revokedCount);
-        assertEquals(1, rebalanceListener.assignedCount);
+        assertEquals(2, rebalanceListener.assignedCount);
     }
 
     @Test
@@ -1261,7 +1261,7 @@ private void unavailableTopicTest(boolean patternSubscribe, Set unavaila
         coordinator.poll(time.timer(Long.MAX_VALUE));
         assertFalse(coordinator.rejoinNeededOrPending());
         // callback not triggered since there's nothing to be assigned
-        assertNull(rebalanceListener.assigned);
+        assertEquals(Collections.emptySet(), rebalanceListener.assigned);
         assertTrue("Metadata refresh not requested for unavailable partitions", metadata.updateRequested());
 
         Map topicErrors = new HashMap<>();
@@ -1340,8 +1340,8 @@ public void testRejoinGroup() {
         Collection added = getAdded(assigned, assigned);
         assertEquals(revoked.isEmpty() ? 0 : 1, rebalanceListener.revokedCount);
         assertEquals(revoked.isEmpty() ? null : revoked, rebalanceListener.revoked);
-        assertEquals(added.isEmpty() ? 1 : 2, rebalanceListener.assignedCount);
-        assertEquals(added.isEmpty() ? null : added, rebalanceListener.assigned);
+        assertEquals(2, rebalanceListener.assignedCount);
+        assertEquals(added, rebalanceListener.assigned);
     }
 
     @Test
@@ -2370,8 +2370,8 @@ public boolean matches(AbstractRequest body) {
         assertEquals("leaveGroupRequested should be " + shouldLeaveGroup, shouldLeaveGroup, leaveGroupRequested.get());
 
         if (shouldLeaveGroup) {
-            assertEquals(1, rebalanceListener.lostCount);
-            assertEquals(singleton(t1p), rebalanceListener.lost);
+            assertEquals(1, rebalanceListener.revokedCount);
+            assertEquals(singleton(t1p), rebalanceListener.revoked);
         }
     }
 
diff --git a/core/src/main/scala/kafka/tools/MirrorMaker.scala b/core/src/main/scala/kafka/tools/MirrorMaker.scala
index b6cd60b201776..8a603c36e1b52 100755
--- a/core/src/main/scala/kafka/tools/MirrorMaker.scala
+++ b/core/src/main/scala/kafka/tools/MirrorMaker.scala
@@ -354,6 +354,8 @@ object MirrorMaker extends Logging with KafkaMetricsGroup {
                                           customRebalanceListener: Option[ConsumerRebalanceListener])
     extends ConsumerRebalanceListener {
 
+    override def onPartitionsLost(partitions: util.Collection[TopicPartition]) {}
+
     override def onPartitionsRevoked(partitions: util.Collection[TopicPartition]) {
       producer.flush()
       commitOffsets(consumerWrapper)
diff --git a/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala b/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala
index b616744628620..45743b2c97713 100644
--- a/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/AbstractConsumerTest.scala
@@ -61,6 +61,8 @@ abstract class AbstractConsumerTest extends BaseRequestTest {
   this.consumerConfig.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
   this.consumerConfig.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false")
   this.consumerConfig.setProperty(ConsumerConfig.METADATA_MAX_AGE_CONFIG, "100")
+  this.consumerConfig.setProperty(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, "6000")
+
 
   override protected def brokerPropertyOverrides(properties: Properties): Unit = {
     properties.setProperty(KafkaConfig.ControlledShutdownEnableProp, "false") // speed up shutdown
@@ -349,7 +351,7 @@ abstract class AbstractConsumerTest extends BaseRequestTest {
     if (partitionsToAssign.isEmpty) {
       consumer.subscribe(topicsToSubscribe.asJava, rebalanceListener)
     } else {
-      consumer.assign(partitionAssignment.asJava)
+      consumer.assign(partitionsToAssign.asJava)
     }
 
     def consumerAssignment(): Set[TopicPartition] = {
diff --git a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
index 4ad59d5bacddd..e9229f073e8e6 100644
--- a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
@@ -1640,6 +1640,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
 
     // stop polling and close one of the consumers, should trigger partition re-assignment among alive consumers
     timeoutPoller.shutdown()
+    consumerPollers -= timeoutPoller
     if (closeConsumer)
       timeoutConsumer.close()
 

From 6082f2415ca3f794d741c6bd006a9d1737bc47f8 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Tue, 30 Jul 2019 09:02:24 -0700
Subject: [PATCH 46/59] fix unit tests

---
 .../org/apache/kafka/clients/consumer/KafkaConsumer.java    | 2 +-
 .../clients/consumer/internals/ConsumerCoordinator.java     | 6 ++++--
 .../clients/consumer/internals/ConsumerCoordinatorTest.java | 1 +
 .../scala/integration/kafka/api/ConsumerBounceTest.scala    | 4 +++-
 .../scala/integration/kafka/api/PlaintextConsumerTest.scala | 2 +-
 5 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index 675e13672b89f..64e5d6385c50f 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -2170,8 +2170,8 @@ public void close(Duration timeout) {
         acquire();
         try {
             if (!closed) {
-                closed = true;
                 close(timeout.toMillis(), false);
+                closed = true;
             }
         } finally {
             release();
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index e0244858a1d06..288d8f6d22364 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -682,7 +682,6 @@ public void resetGeneration(String causeMessage, boolean dueToError) {
             // if reset due to error then we have to give up all owned partitions as if they've lost;
             // otherwise we can give them up as revoked
             Set droppedPartitions = new HashSet<>(subscriptions.assignedPartitions());
-            subscriptions.assignFromSubscribed(Collections.emptySet());
 
             Exception e;
 
@@ -692,6 +691,8 @@ public void resetGeneration(String causeMessage, boolean dueToError) {
                 e = maybeInvokePartitionsRevoked(droppedPartitions);
             }
 
+            subscriptions.assignFromSubscribed(Collections.emptySet());
+
             if (e != null) {
                 throw new KafkaException("User rebalance callback throws an error", e);
             }
@@ -718,11 +719,12 @@ public boolean rejoinNeededOrPending() {
 
             if (!noLongerExistingPartitions.isEmpty()) {
                 ownedPartitions.removeAll(noLongerExistingPartitions);
-                subscriptions.assignFromSubscribed(ownedPartitions);
 
                 Exception e = maybeInvokePartitionsLost("topic metadata has changed and therefore some topics may not exist any more",
                     noLongerExistingPartitions);
 
+                subscriptions.assignFromSubscribed(ownedPartitions);
+
                 if (e != null) {
                     throw new KafkaException("User rebalance callback throws an error", e);
                 }
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index f18eb76fb02c4..159d715ce9f17 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -52,6 +52,7 @@
 import org.apache.kafka.common.metrics.Metrics;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.record.RecordBatch;
+import org.apache.kafka.common.requests.AbstractRequest;
 import org.apache.kafka.common.requests.FindCoordinatorResponse;
 import org.apache.kafka.common.requests.HeartbeatResponse;
 import org.apache.kafka.common.requests.JoinGroupRequest;
diff --git a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
index 1dbc20a6bc3b8..634bcda16a7bb 100644
--- a/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
+++ b/core/src/test/scala/integration/kafka/api/ConsumerBounceTest.scala
@@ -246,7 +246,9 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
     killBroker(findCoordinator(manualGroup))
 
     val future1 = submitCloseAndValidate(consumer1, Long.MaxValue, None, gracefulCloseTimeMs)
+
     val future2 = submitCloseAndValidate(consumer2, Long.MaxValue, None, gracefulCloseTimeMs)
+
     future1.get
     future2.get
 
@@ -286,7 +288,7 @@ class ConsumerBounceTest extends AbstractConsumerTest with Logging {
 
     servers.foreach(server => killBroker(server.config.brokerId))
     val closeTimeout = 2000
-    val future1 = submitCloseAndValidate(consumer1, closeTimeout, Some(closeTimeout), Some(closeTimeout))
+    val future1 = submitCloseAndValidate(consumer1, closeTimeout, None, Some(closeTimeout))
     val future2 = submitCloseAndValidate(consumer2, Long.MaxValue, Some(requestTimeout), Some(requestTimeout))
     future1.get
     future2.get
diff --git a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
index e9229f073e8e6..9ee4055570a53 100644
--- a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
@@ -207,7 +207,7 @@ class PlaintextConsumerTest extends BaseConsumerTest {
     val listener = new TestConsumerReassignmentListener {
       override def onPartitionsLost(partitions: util.Collection[TopicPartition]): Unit = {}
       override def onPartitionsRevoked(partitions: util.Collection[TopicPartition]): Unit = {
-        if (!partitions.isEmpty) {
+        if (!partitions.isEmpty && partitions.contains(tp)) {
           // on the second rebalance (after we have joined the group initially), sleep longer
           // than session timeout and then try a commit. We should still be in the group,
           // so the commit should succeed

From 90e6050481bbf535d3f266848cf3b2e2210dd9f2 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Tue, 30 Jul 2019 09:12:10 -0700
Subject: [PATCH 47/59] minor fixes

---
 .../scala/integration/kafka/api/PlaintextConsumerTest.scala     | 2 ++
 .../apache/kafka/streams/processor/internals/StreamThread.java  | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
index 9ee4055570a53..30b1d13aad81e 100644
--- a/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
+++ b/core/src/test/scala/integration/kafka/api/PlaintextConsumerTest.scala
@@ -185,12 +185,14 @@ class PlaintextConsumerTest extends BaseConsumerTest {
     // rebalance to get the initial assignment
     awaitRebalance(consumer, listener)
     assertEquals(1, listener.callsToAssigned)
+    assertEquals(0, listener.callsToRevoked)
 
     Thread.sleep(3500)
 
     // we should fall out of the group and need to rebalance
     awaitRebalance(consumer, listener)
     assertEquals(2, listener.callsToAssigned)
+    assertEquals(1, listener.callsToRevoked)
   }
 
   @Test
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index 02df3781708ed..1633b30223b62 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -305,7 +305,7 @@ public void onPartitionsAssigned(final Collection assignment) {
         }
 
         @Override
-        public void  onPartitionsRevoked(final Collection assignment) {
+        public void onPartitionsRevoked(final Collection assignment) {
             log.debug("at state {}: partitions {} revoked at the beginning of consumer rebalance.\n" +
                     "\tcurrent assigned active tasks: {}\n" +
                     "\tcurrent assigned standby tasks: {}\n",

From 700be688d0f0504c73565979ccc51b2d596f0fad Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Wed, 31 Jul 2019 12:39:39 -0700
Subject: [PATCH 48/59] add unit test

---
 .../consumer/ConsumerPartitionAssignor.java   | 26 ++++----
 .../internals/AbstractCoordinator.java        |  9 ++-
 .../internals/ConsumerCoordinator.java        | 15 +++--
 .../clients/consumer/KafkaConsumerTest.java   | 64 ++++++++++++++++++-
 .../internals/ConsumerCoordinatorTest.java    | 10 +--
 5 files changed, 96 insertions(+), 28 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
index be3b336d2af68..82c3ebadbe273 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
@@ -179,19 +179,21 @@ public Map groupAssignment() {
     }
 
     /**
-     * {@link RebalanceProtocol#EAGER} rebalance protocol will let a consumer to always revoke all its owned partitions
-     * before participating the rebalance event, and therefore allows a complete reshuffling of the assignment.
+     * The rebalance protocol defines partition assignment and revocation semantics. The purpose is to establish a
+     * consistent set of rules that all consumers in a group follow in order to transfer ownership of a partition.
+     * {@link ConsumerPartitionAssignor} implementors can claim supporting one or more rebalance protocols via the
+     * {@link ConsumerPartitionAssignor#supportedProtocols()}, and it is their responsible to respect the rules
+     * of those protocols in their {@link ConsumerPartitionAssignor#assign(Cluster, GroupSubscription)} implementations.
+     * Failures to follow the rules of the supported protocols would lead to runtime error or undefined behavior.
      *
-     * {@link RebalanceProtocol#COOPERATIVE} rebalance protocol will let a consumer to retain its currently assigned
-     * partitions before participating the rebalance event, as a result the assignor would not reassign its owned
-     * partitions immediately but may only indicate it to revoke first before the next rebalance event, and therefore is
-     * suitable for cooperative adjustments of assignments.
-     * {@link ConsumerPartitionAssignor} implementors who claim to support {@link RebalanceProtocol#COOPERATIVE} protocol
-     * from {@link ConsumerPartitionAssignor#supportedProtocols()} needs to make sure its
-     * {@link ConsumerPartitionAssignor#assign(Cluster, GroupSubscription)} logic respects the specified
-     * {@link Subscription#ownedPartitions()} list and not reassign them to other consumers; instead it can decide
-     * whether to remove them from the new assignment of the current owner consumer so that they can be revoked first
-     * before reassigned to other consumers in future rebalances.
+     * The {@link RebalanceProtocol#EAGER} rebalance protocol requires a consumer to always revoke all its owned
+     * partitions before participating in a rebalance event. It therefore allows a complete reshuffling of the assignment.
+     *
+     * {@link RebalanceProtocol#COOPERATIVE} rebalance protocol allows a consumer to retain its currently owned
+     * partitions before participating in a rebalance event. The assignor should not reassign any owned partitions
+     * immediately, but instead may indicate consumers the need for partition revocation so that the revoked
+     * partitions can be reassigned to other consumers in the next rebalance event. This is designed for sticky assignment
+     * logic which attempts to minimize partition reassignment with cooperative adjustments.
      */
     enum RebalanceProtocol {
         EAGER((byte) 0), COOPERATIVE((byte) 1);
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 63e0e351d4198..1c6495418a734 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -378,8 +378,8 @@ boolean joinGroupIfNeeded(final Timer timer) {
             // refresh which changes the matched subscription set) can occur while another rebalance is
             // still in progress.
             if (needsJoinPrepare) {
-                onJoinPrepare(generation.generationId, generation.memberId);
                 needsJoinPrepare = false;
+                onJoinPrepare(generation.generationId, generation.memberId);
             }
 
             final RequestFuture future = initiateJoinGroup();
@@ -861,6 +861,11 @@ protected void close(Timer timer) {
      * @throws KafkaException if the callback throws exception
      */
     public synchronized RequestFuture maybeLeaveGroup(String leaveReason) {
+        // we need to reset generation first in order to trigger
+        // the rebalance callback if necessary, before sending the leave group
+        // which may trigger the rebalance
+        resetGeneration("consumer proactively leaving group", false);
+
         RequestFuture future = null;
         // Starting from 2.3, only dynamic members will send LeaveGroupRequest to the broker,
         // consumer with valid group.instance.id is viewed as static member that never sends LeaveGroup,
@@ -881,8 +886,6 @@ public synchronized RequestFuture maybeLeaveGroup(String leaveReason) {
             client.pollNoWakeup();
         }
 
-        resetGeneration("consumer proactively leaving group", false);
-
         return future;
     }
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 288d8f6d22364..468b675ef35c7 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -18,6 +18,7 @@
 
 import org.apache.kafka.clients.GroupRebalanceConfig;
 import org.apache.kafka.clients.consumer.CommitFailedException;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor;
 import org.apache.kafka.clients.consumer.ConsumerGroupMetadata;
 import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription;
@@ -434,8 +435,8 @@ public boolean poll(Timer timer) {
 
         if (subscriptions.partitionsAutoAssigned()) {
             if (protocol == null) {
-                throw new IllegalStateException("User configured ConsumerConfig#PARTITION_ASSIGNMENT_STRATEGY_CONFIG to empty " +
-                    "while trying to subscribe for group protocol to auto assign partitions");
+                throw new IllegalStateException("User configured " + ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG +
+                    " to empty while trying to subscribe for group protocol to auto assign partitions");
             }
             // Always update the heartbeat last poll time so that the heartbeat thread does not leave the
             // group proactively due to application inactivity even if (say) the coordinator cannot be found.
@@ -634,15 +635,15 @@ protected void onJoinPrepare(int generation, String memberId) {
 
         // execute the user's callback before rebalance
         final Set revokedPartitions;
-        final AtomicReference firstException = new AtomicReference<>(null);
 
         // note we should only change the assignment AFTER the callback is triggered
         // so that users can still access the pre-assigned partitions to commit offsets etc.
+        Exception exception = null;
         switch (protocol) {
             case EAGER:
                 // revoke all partitions
                 revokedPartitions = new HashSet<>(subscriptions.assignedPartitions());
-                firstException.compareAndSet(null, maybeInvokePartitionsRevoked(revokedPartitions));
+                exception = maybeInvokePartitionsRevoked(revokedPartitions);
 
                 subscriptions.assignFromSubscribed(Collections.emptySet());
 
@@ -656,7 +657,7 @@ protected void onJoinPrepare(int generation, String memberId) {
                     .collect(Collectors.toSet());
 
                 if (!revokedPartitions.isEmpty()) {
-                    firstException.compareAndSet(null, maybeInvokePartitionsRevoked(revokedPartitions));
+                    exception = maybeInvokePartitionsRevoked(revokedPartitions);
 
                     ownedPartitions.removeAll(revokedPartitions);
                     subscriptions.assignFromSubscribed(ownedPartitions);
@@ -668,8 +669,8 @@ protected void onJoinPrepare(int generation, String memberId) {
         isLeader = false;
         subscriptions.resetGroupSubscription();
 
-        if (firstException.get() != null) {
-            throw new KafkaException("User rebalance callback throws an error", firstException.get());
+        if (exception != null) {
+            throw new KafkaException("User rebalance callback throws an error", exception);
         }
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
index 1227c27ad6035..7892d6d1c92db 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java
@@ -1332,7 +1332,7 @@ public void testCloseInterrupt() throws Exception {
     }
 
     @Test
-    public void closeShouldBeIdempotent() {
+    public void testCloseShouldBeIdempotent() {
         KafkaConsumer consumer = newConsumer((String) null);
         consumer.close();
         consumer.close();
@@ -1419,7 +1419,7 @@ public void testMetricConfigRecordingLevel() {
     }
 
     @Test
-    public void shouldAttemptToRejoinGroupAfterSyncGroupFailed() throws Exception {
+    public void testShouldAttemptToRejoinGroupAfterSyncGroupFailed() throws Exception {
         Time time = new MockTime();
         SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST);
         ConsumerMetadata metadata = createMetadata(subscription);
@@ -1639,6 +1639,52 @@ public void testCommittedAuthenticationFaiure() {
         consumer.committed(tp0);
     }
 
+    @Test
+    public void testRebalanceException() {
+        Time time = new MockTime();
+        SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST);
+        ConsumerMetadata metadata = createMetadata(subscription);
+        MockClient client = new MockClient(time, metadata);
+
+        initMetadata(client, Collections.singletonMap(topic, 1));
+        Node node = metadata.fetch().nodes().get(0);
+
+        ConsumerPartitionAssignor assignor = new RoundRobinAssignor();
+
+        KafkaConsumer consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
+
+        consumer.subscribe(singleton(topic), getExceptionConsumerRebalanceListener());
+        Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port());
+
+        client.prepareResponseFrom(FindCoordinatorResponse.prepareResponse(Errors.NONE, node), node);
+        client.prepareResponseFrom(joinGroupFollowerResponse(assignor, 1, "memberId", "leaderId", Errors.NONE), coordinator);
+        client.prepareResponseFrom(syncGroupResponse(singletonList(tp0), Errors.NONE), coordinator);
+
+        // assign throws
+        try {
+            consumer.updateAssignmentMetadataIfNeeded(time.timer(Long.MAX_VALUE));
+            fail("Should throw exception");
+        } catch (Throwable e) {
+            assertEquals("boom!", e.getCause().getMessage());
+        }
+
+        // the assignment is still updated regardless of the exception
+        assertEquals(singleton(tp0), subscription.assignedPartitions());
+
+        // close's revoke throws
+        try {
+            consumer.close(Duration.ofMillis(0));
+            fail("Should throw exception");
+        } catch (Throwable e) {
+            assertEquals("boom!", e.getCause().getCause().getMessage());
+        }
+
+        consumer.close(Duration.ofMillis(0));
+
+        // the assignment is still updated regardless of the exception
+        assertTrue(subscription.assignedPartitions().isEmpty());
+    }
+
     private KafkaConsumer consumerWithPendingAuthenticationError() {
         Time time = new MockTime();
         SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST);
@@ -1669,6 +1715,20 @@ public void onPartitionsAssigned(Collection partitions) {
         };
     }
 
+    private ConsumerRebalanceListener getExceptionConsumerRebalanceListener() {
+        return new ConsumerRebalanceListener() {
+            @Override
+            public void onPartitionsRevoked(Collection partitions) {
+                throw new RuntimeException("boom!");
+            }
+
+            @Override
+            public void onPartitionsAssigned(Collection partitions) {
+                throw new RuntimeException("boom!");
+            }
+        };
+    }
+
     private ConsumerMetadata createMetadata(SubscriptionState subscription) {
         return new ConsumerMetadata(0, Long.MAX_VALUE, false, false,
                                     subscription, new LogContext(), new ClusterResourceListeners());
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index 159d715ce9f17..ddc775a89d22a 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -101,6 +101,7 @@
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
+import static org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.RebalanceProtocol.COOPERATIVE;
 import static org.apache.kafka.test.TestUtils.toSet;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -202,18 +203,18 @@ public void teardown() {
     public void testSelectRebalanceProtcol() {
         List assignors = new ArrayList<>();
         assignors.add(new MockPartitionAssignor(Collections.singletonList(ConsumerPartitionAssignor.RebalanceProtocol.EAGER)));
-        assignors.add(new MockPartitionAssignor(Collections.singletonList(ConsumerPartitionAssignor.RebalanceProtocol.COOPERATIVE)));
+        assignors.add(new MockPartitionAssignor(Collections.singletonList(COOPERATIVE)));
 
         // no commonly supported protocols
         assertThrows(IllegalArgumentException.class, () -> buildCoordinator(rebalanceConfig, new Metrics(), assignors, false));
 
         assignors.clear();
-        assignors.add(new MockPartitionAssignor(Arrays.asList(ConsumerPartitionAssignor.RebalanceProtocol.EAGER, ConsumerPartitionAssignor.RebalanceProtocol.COOPERATIVE)));
-        assignors.add(new MockPartitionAssignor(Arrays.asList(ConsumerPartitionAssignor.RebalanceProtocol.EAGER, ConsumerPartitionAssignor.RebalanceProtocol.COOPERATIVE)));
+        assignors.add(new MockPartitionAssignor(Arrays.asList(ConsumerPartitionAssignor.RebalanceProtocol.EAGER, COOPERATIVE)));
+        assignors.add(new MockPartitionAssignor(Arrays.asList(ConsumerPartitionAssignor.RebalanceProtocol.EAGER, COOPERATIVE)));
 
         // select higher indexed (more advanced) protocols
         try (ConsumerCoordinator coordinator = buildCoordinator(rebalanceConfig, new Metrics(), assignors, false)) {
-            assertEquals(ConsumerPartitionAssignor.RebalanceProtocol.COOPERATIVE, coordinator.getProtocol());
+            assertEquals(COOPERATIVE, coordinator.getProtocol());
         }
     }
 
@@ -701,6 +702,7 @@ public boolean matches(AbstractRequest body) {
         Collection lost = Collections.singleton(t2p);
         revoked = getRevoked(newAssigned, oldAssigned);
         revoked.removeAll(lost);
+        assertEquals(protocol == COOPERATIVE, revoked.isEmpty());
         revokedCount += revoked.isEmpty() ? 0 : 1;
         Collection added = getAdded(newAssigned, oldAssigned);
 

From 8edb9e1843b7e0c2f21c63b65a5d001fde79cf14 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Wed, 31 Jul 2019 12:43:28 -0700
Subject: [PATCH 49/59] add more comments

---
 .../java/org/apache/kafka/clients/consumer/KafkaConsumer.java   | 2 ++
 .../kafka/clients/consumer/internals/AbstractCoordinator.java   | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index 64e5d6385c50f..61f87861cbe9c 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -2170,6 +2170,8 @@ public void close(Duration timeout) {
         acquire();
         try {
             if (!closed) {
+                // need to close before setting the flag since the close function
+                // itself may trigger rebalance callback that needs the consumer to be open still
                 close(timeout.toMillis(), false);
                 closed = true;
             }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 1c6495418a734..47505e6c039fa 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -378,6 +378,8 @@ boolean joinGroupIfNeeded(final Timer timer) {
             // refresh which changes the matched subscription set) can occur while another rebalance is
             // still in progress.
             if (needsJoinPrepare) {
+                // need to set the flag before calling onJoinPrepare since the user callback may throw
+                // exception, in which case upon retry we should not retry onJoinPrepare either.
                 needsJoinPrepare = false;
                 onJoinPrepare(generation.generationId, generation.memberId);
             }

From 4b0e170913dd41e61d9304921206d3c9d572fa72 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Wed, 31 Jul 2019 19:04:14 -0700
Subject: [PATCH 50/59] refactoring leave-group ordering

---
 .../internals/AbstractCoordinator.java        | 28 +++++++++++--------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 47505e6c039fa..c177904f8f69a 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -863,12 +863,7 @@ protected void close(Timer timer) {
      * @throws KafkaException if the callback throws exception
      */
     public synchronized RequestFuture maybeLeaveGroup(String leaveReason) {
-        // we need to reset generation first in order to trigger
-        // the rebalance callback if necessary, before sending the leave group
-        // which may trigger the rebalance
-        resetGeneration("consumer proactively leaving group", false);
-
-        RequestFuture future = null;
+        LeaveGroupRequest.Builder request = null;
         // Starting from 2.3, only dynamic members will send LeaveGroupRequest to the broker,
         // consumer with valid group.instance.id is viewed as static member that never sends LeaveGroup,
         // and the membership expiration is only controlled by session timeout.
@@ -878,17 +873,26 @@ public synchronized RequestFuture maybeLeaveGroup(String leaveReason) {
             // attempt any resending if the request fails or times out.
             log.info("Member {} sending LeaveGroup request to coordinator {} due to {}",
                      generation.memberId, coordinator, leaveReason);
-            LeaveGroupRequest.Builder request = new LeaveGroupRequest.Builder(
+            request = new LeaveGroupRequest.Builder(
                 rebalanceConfig.groupId,
-                Collections.singletonList(new MemberIdentity()
-                                              .setMemberId(generation.memberId))
+                Collections.singletonList(new MemberIdentity().setMemberId(generation.memberId))
             );
-            future = client.send(coordinator, request)
-                    .compose(new LeaveGroupResponseHandler());
+        }
+
+        // we need to reset generation first in order to trigger
+        // the rebalance callback if necessary, before sending
+        // the leave group which may trigger the rebalance
+        resetGeneration("consumer proactively leaving group", false);
+
+        if (request != null) {
+            RequestFuture future = client.send(coordinator, request)
+                .compose(new LeaveGroupResponseHandler());
             client.pollNoWakeup();
+
+            return future;
         }
 
-        return future;
+        return null;
     }
 
     protected boolean isDynamicMember() {

From f1baeb47717caf2b28b97a826a7214ac78fb1d23 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Fri, 2 Aug 2019 09:13:20 -0700
Subject: [PATCH 51/59] fix on streams

---
 .../consumer/ConsumerPartitionAssignor.java   |  2 +-
 .../kafka/clients/consumer/KafkaConsumer.java |  2 +-
 .../internals/AbstractCoordinator.java        | 52 +++++-----
 .../internals/ConsumerCoordinator.java        | 96 ++++++++++++-------
 .../internals/ConsumerCoordinatorTest.java    |  6 ++
 .../apache/kafka/streams/KafkaStreams.java    |  5 +-
 .../processor/internals/StreamThread.java     |  5 +-
 .../StreamTableJoinIntegrationTest.java       |  5 +-
 .../utils/IntegrationTestUtils.java           | 17 +---
 9 files changed, 115 insertions(+), 75 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
index 60629d4f1412e..f9a42171a0447 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerPartitionAssignor.java
@@ -184,7 +184,7 @@ public Map groupAssignment() {
      * The rebalance protocol defines partition assignment and revocation semantics. The purpose is to establish a
      * consistent set of rules that all consumers in a group follow in order to transfer ownership of a partition.
      * {@link ConsumerPartitionAssignor} implementors can claim supporting one or more rebalance protocols via the
-     * {@link ConsumerPartitionAssignor#supportedProtocols()}, and it is their responsible to respect the rules
+     * {@link ConsumerPartitionAssignor#supportedProtocols()}, and it is their responsibility to respect the rules
      * of those protocols in their {@link ConsumerPartitionAssignor#assign(Cluster, GroupSubscription)} implementations.
      * Failures to follow the rules of the supported protocols would lead to runtime error or undefined behavior.
      *
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index b649ec48ed354..543f1e11555a2 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -2174,9 +2174,9 @@ public void close(Duration timeout) {
                 // need to close before setting the flag since the close function
                 // itself may trigger rebalance callback that needs the consumer to be open still
                 close(timeout.toMillis(), false);
-                closed = true;
             }
         } finally {
+            closed = true;
             release();
         }
     }
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index c177904f8f69a..148f33f089016 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -46,6 +46,7 @@
 import org.apache.kafka.common.metrics.stats.Max;
 import org.apache.kafka.common.metrics.stats.Meter;
 import org.apache.kafka.common.metrics.stats.WindowedCount;
+import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.requests.FindCoordinatorRequest;
 import org.apache.kafka.common.requests.FindCoordinatorRequest.CoordinatorType;
@@ -526,7 +527,7 @@ public void handle(JoinGroupResponse joinResponse, RequestFuture fut
                 future.raise(error);
             } else if (error == Errors.UNKNOWN_MEMBER_ID) {
                 // reset the member id and retry immediately
-                resetGeneration("encountering " + error + " on join response");
+                resetGenerationOnResponseError(ApiKeys.JOIN_GROUP, error);
                 log.debug("Attempt to join group failed due to unknown member id.");
                 future.raise(error);
             } else if (error == Errors.COORDINATOR_NOT_AVAILABLE
@@ -649,7 +650,7 @@ public void handle(SyncGroupResponse syncResponse,
                 } else if (error == Errors.UNKNOWN_MEMBER_ID
                         || error == Errors.ILLEGAL_GENERATION) {
                     log.debug("SyncGroup failed: {}", error.message());
-                    resetGeneration("encountering " + error + " on sync group response");
+                    resetGenerationOnResponseError(ApiKeys.SYNC_GROUP, error);
                     future.raise(error);
                 } else if (error == Errors.COORDINATOR_NOT_AVAILABLE
                         || error == Errors.NOT_COORDINATOR) {
@@ -799,24 +800,20 @@ final synchronized boolean hasValidMemberId() {
         return generation != null && generation.hasMemberId();
     }
 
-
-    public synchronized void resetGeneration(String causeMessage, boolean dueToError) {
-        if (dueToError) {
-            log.debug("Resetting generation and requesting re-join group due to {}", causeMessage);
-        } else {
-            log.debug("Resetting generation pro-actively due to {}", causeMessage);
-        }
-
+    private synchronized void resetGeneration() {
         this.generation = Generation.NO_GENERATION;
         this.state = MemberState.UNJOINED;
         this.rejoinNeeded = true;
     }
 
-    /**
-     * Reset the generation and memberId because we have fallen out of the group.
-     */
-    protected synchronized void resetGeneration(String errorMessage) {
-        resetGeneration(errorMessage, true);
+    synchronized void resetGenerationOnResponseError(ApiKeys api, Errors error) {
+        log.debug("Resetting generation after encountering " + error + " from " + api + " response");
+        resetGeneration();
+    }
+
+    synchronized void resetGenerationOnLeaveGroup() {
+        log.debug("Resetting generation due to consumer pro-actively leaving the group");
+        resetGeneration();
     }
 
     protected synchronized void requestRejoin() {
@@ -857,12 +854,16 @@ protected void close(Timer timer) {
         }
     }
 
+    public synchronized RequestFuture maybeLeaveGroup(String leaveReason) {
+        return maybeLeaveGroup(leaveReason, false);
+    }
+
     /**
      * Leave the current group and reset local generation/memberId.
      * @param leaveReason reason to attempt leaving the group
      * @throws KafkaException if the callback throws exception
      */
-    public synchronized RequestFuture maybeLeaveGroup(String leaveReason) {
+    public synchronized RequestFuture maybeLeaveGroup(String leaveReason, boolean dueToHeartbeatExpiration) {
         LeaveGroupRequest.Builder request = null;
         // Starting from 2.3, only dynamic members will send LeaveGroupRequest to the broker,
         // consumer with valid group.instance.id is viewed as static member that never sends LeaveGroup,
@@ -879,10 +880,15 @@ public synchronized RequestFuture maybeLeaveGroup(String leaveReason) {
             );
         }
 
-        // we need to reset generation first in order to trigger
-        // the rebalance callback if necessary, before sending
-        // the leave group which may trigger the rebalance
-        resetGeneration("consumer proactively leaving group", false);
+        // we need to reset generation first in order to trigger the rebalance callback if necessary, before sending
+        // the leave group which may trigger the rebalance; if leave-group is caused by heartbeat poll expiration
+        // at the heartbeat thread, we should not trigger the rebalance callback and will wait until the caller thread
+        // calling consumer.poll to trigger it.
+        if (dueToHeartbeatExpiration) {
+            resetGeneration();
+        } else {
+            resetGenerationOnLeaveGroup();
+        }
 
         if (request != null) {
             RequestFuture future = client.send(coordinator, request)
@@ -952,14 +958,14 @@ public void handle(HeartbeatResponse heartbeatResponse, RequestFuture futu
                 future.raise(error);
             } else if (error == Errors.ILLEGAL_GENERATION) {
                 log.info("Attempt to heartbeat failed since generation {} is not current", generation.generationId);
-                resetGeneration("encountering " + error + " on heartbeat response");
+                resetGenerationOnResponseError(ApiKeys.HEARTBEAT, error);
                 future.raise(error);
             } else if (error == Errors.FENCED_INSTANCE_ID) {
                 log.error("Received fatal exception: group.instance.id gets fenced");
                 future.raise(error);
             } else if (error == Errors.UNKNOWN_MEMBER_ID) {
                 log.info("Attempt to heartbeat failed for since member id {} is not valid.", generation.memberId);
-                resetGeneration("encountering " + error + " on heartbeat response");
+                resetGenerationOnResponseError(ApiKeys.HEARTBEAT, error);
                 future.raise(error);
             } else if (error == Errors.GROUP_AUTHORIZATION_FAILED) {
                 future.raise(GroupAuthorizationException.forGroupId(rebalanceConfig.groupId));
@@ -1135,7 +1141,7 @@ public void run() {
                                                     "the poll loop is spending too much time processing messages. " +
                                                     "You can address this either by increasing max.poll.interval.ms or by reducing " +
                                                     "the maximum size of batches returned in poll() with max.poll.records.";
-                            maybeLeaveGroup(leaveReason);
+                            maybeLeaveGroup(leaveReason, true);
                         } else if (!heartbeat.shouldHeartbeat(now)) {
                             // poll again after waiting for the retry backoff in case the heartbeat failed or the
                             // coordinator disconnected
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 5a5de3534a808..f08fbb33570dd 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -49,6 +49,7 @@
 import org.apache.kafka.common.metrics.Sensor;
 import org.apache.kafka.common.metrics.stats.Avg;
 import org.apache.kafka.common.metrics.stats.Max;
+import org.apache.kafka.common.protocol.ApiKeys;
 import org.apache.kafka.common.protocol.Errors;
 import org.apache.kafka.common.record.RecordBatch;
 import org.apache.kafka.common.requests.OffsetCommitRequest;
@@ -384,7 +385,7 @@ protected void onJoinComplete(int generation,
                     Utils.join(addedPartitions, ", "),
                     Utils.join(revokedPartitions, ", "));
 
-                // revoked partitions that was previously owned but no longer assigned
+                // revoke partitions that was previously owned but no longer assigned
                 firstException.compareAndSet(null, maybeInvokePartitionsRevoked(revokedPartitions));
 
                 subscriptions.assignFromSubscribed(assignedPartitions);
@@ -640,33 +641,46 @@ protected void onJoinPrepare(int generation, String memberId) {
         // note we should only change the assignment AFTER the callback is triggered
         // so that users can still access the pre-assigned partitions to commit offsets etc.
         Exception exception = null;
-        switch (protocol) {
-            case EAGER:
-                // revoke all partitions
-                revokedPartitions = new HashSet<>(subscriptions.assignedPartitions());
-                exception = maybeInvokePartitionsRevoked(revokedPartitions);
 
-                subscriptions.assignFromSubscribed(Collections.emptySet());
+        if (generation == Generation.NO_GENERATION.generationId &&
+            memberId.equals(Generation.NO_GENERATION.memberId)) {
 
-                break;
-
-            case COOPERATIVE:
-                // only revoke those partitions that are not in the subscription any more.
-                Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions());
-                revokedPartitions = ownedPartitions.stream()
-                    .filter(tp -> !subscriptions.subscription().contains(tp.topic()))
-                    .collect(Collectors.toSet());
+            revokedPartitions = new HashSet<>(subscriptions.assignedPartitions());
+            exception = maybeInvokePartitionsLost(
+                "generation has been reset because consumer has been kicked out of the group",
+                revokedPartitions);
 
-                if (!revokedPartitions.isEmpty()) {
+            subscriptions.assignFromSubscribed(Collections.emptySet());
+        } else {
+            switch (protocol) {
+                case EAGER:
+                    // revoke all partitions
+                    revokedPartitions = new HashSet<>(subscriptions.assignedPartitions());
                     exception = maybeInvokePartitionsRevoked(revokedPartitions);
 
-                    ownedPartitions.removeAll(revokedPartitions);
-                    subscriptions.assignFromSubscribed(ownedPartitions);
-                }
+                    subscriptions.assignFromSubscribed(Collections.emptySet());
 
-                break;
+                    break;
+
+                case COOPERATIVE:
+                    // only revoke those partitions that are not in the subscription any more.
+                    Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions());
+                    revokedPartitions = ownedPartitions.stream()
+                        .filter(tp -> !subscriptions.subscription().contains(tp.topic()))
+                        .collect(Collectors.toSet());
+
+                    if (!revokedPartitions.isEmpty()) {
+                        exception = maybeInvokePartitionsRevoked(revokedPartitions);
+
+                        ownedPartitions.removeAll(revokedPartitions);
+                        subscriptions.assignFromSubscribed(ownedPartitions);
+                    }
+
+                    break;
+            }
         }
 
+
         isLeader = false;
         subscriptions.resetGroupSubscription();
 
@@ -675,20 +689,15 @@ protected void onJoinPrepare(int generation, String memberId) {
         }
     }
 
-    /**
-     * @throws KafkaException if the rebalance callback throws exception
-     */
-    @Override
-    public void resetGeneration(String causeMessage, boolean dueToError) {
-        if (subscriptions.partitionsAutoAssigned()) {
-            // if reset due to error then we have to give up all owned partitions as if they've lost;
-            // otherwise we can give them up as revoked
-            Set droppedPartitions = new HashSet<>(subscriptions.assignedPartitions());
+    private void resetAssignment(boolean lostPartitions) {
+        Set droppedPartitions = new HashSet<>(subscriptions.assignedPartitions());
+
+        if (subscriptions.partitionsAutoAssigned() && !droppedPartitions.isEmpty()) {
 
             Exception e;
 
-            if (dueToError) {
-                e = maybeInvokePartitionsLost(causeMessage, droppedPartitions);
+            if (lostPartitions) {
+                e = maybeInvokePartitionsLost("reset generation", droppedPartitions);
             } else {
                 e = maybeInvokePartitionsRevoked(droppedPartitions);
             }
@@ -699,8 +708,29 @@ public void resetGeneration(String causeMessage, boolean dueToError) {
                 throw new KafkaException("User rebalance callback throws an error", e);
             }
         }
+    }
 
-        super.resetGeneration(causeMessage, dueToError);
+    /**
+     * @throws KafkaException if the rebalance callback throws exception
+     */
+    @Override
+    synchronized void resetGenerationOnResponseError(ApiKeys api, Errors error) {
+        // heartbeat error handling is executed by heartbeat thread, in which case
+        // we would not drop partitions and trigger rebalance callback as it should
+        // only be triggered by the caller thread
+        if (api != ApiKeys.HEARTBEAT) {
+            resetAssignment(true);
+        }
+        super.resetGenerationOnResponseError(api, error);
+    }
+
+    /**
+     * @throws KafkaException if the rebalance callback throws exception
+     */
+    @Override
+    synchronized void resetGenerationOnLeaveGroup() {
+        resetAssignment(false);
+        super.resetGenerationOnLeaveGroup();
     }
 
     /**
@@ -1149,7 +1179,7 @@ public void handle(OffsetCommitResponse commitResponse, RequestFuture futu
                         } else if (error == Errors.UNKNOWN_MEMBER_ID
                                 || error == Errors.ILLEGAL_GENERATION) {
                             // need to reset generation and re-join group
-                            resetGeneration("encountering " + error + " on offset commit response");
+                            resetGenerationOnResponseError(ApiKeys.OFFSET_COMMIT, error);
                             future.raise(new CommitFailedException());
                             return;
                         } else {
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index ddc775a89d22a..706463ff6dda4 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -391,6 +391,9 @@ public void testIllegalGeneration() {
         assertTrue(future.failed());
         assertEquals(Errors.ILLEGAL_GENERATION.exception(), future.exception());
         assertTrue(coordinator.rejoinNeededOrPending());
+
+        coordinator.poll(time.timer(0));
+
         assertEquals(1, rebalanceListener.lostCount);
         assertEquals(Collections.singleton(t1p), rebalanceListener.lost);
     }
@@ -417,6 +420,9 @@ public void testUnknownMemberId() {
         assertTrue(future.failed());
         assertEquals(Errors.UNKNOWN_MEMBER_ID.exception(), future.exception());
         assertTrue(coordinator.rejoinNeededOrPending());
+
+        coordinator.poll(time.timer(0));
+
         assertEquals(1, rebalanceListener.lostCount);
         assertEquals(Collections.singleton(t1p), rebalanceListener.lost);
     }
diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
index bbf9f9c509745..412d5e4166947 100644
--- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
+++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java
@@ -190,8 +190,11 @@ public class KafkaStreams implements AutoCloseable {
      * - Of special importance: If the global stream thread dies, or all stream threads die (or both) then
      *   the instance will be in the ERROR state. The user will need to close it.
      */
+    // TODO: the current transitions from other states directly to RUNNING is due to
+    //       the fact that onPartitionsRevoked may not be triggered. we need to refactor the
+    //       state diagram more thoroughly after we refactor StreamsPartitionAssignor to support COOPERATIVE
     public enum State {
-        CREATED(1, 3), REBALANCING(2, 3, 5), RUNNING(1, 3, 5), PENDING_SHUTDOWN(4), NOT_RUNNING, ERROR(3);
+        CREATED(1, 2, 3), REBALANCING(2, 3, 5), RUNNING(1, 2, 3, 5), PENDING_SHUTDOWN(4), NOT_RUNNING, ERROR(3);
 
         private final Set validTransitions = new HashSet<>();
 
diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
index 1633b30223b62..294e4a9c2cf07 100644
--- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
+++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamThread.java
@@ -127,7 +127,10 @@ public class StreamThread extends Thread {
      * 
      */
     public enum State implements ThreadStateTransitionValidator {
-        CREATED(1, 5), STARTING(2, 5), PARTITIONS_REVOKED(3, 5), PARTITIONS_ASSIGNED(2, 4, 5), RUNNING(2, 5), PENDING_SHUTDOWN(6), DEAD;
+        // TODO: the current transitions from other states directly to PARTITIONS_REVOKED is due to
+        //       the fact that onPartitionsRevoked may not be triggered. we need to refactor the
+        //       state diagram more thoroughly after we refactor StreamsPartitionAssignor to support COOPERATIVE
+        CREATED(1, 5), STARTING(2, 3, 5), PARTITIONS_REVOKED(3, 5), PARTITIONS_ASSIGNED(2, 3, 4, 5), RUNNING(2, 3, 5), PENDING_SHUTDOWN(6), DEAD;
 
         private final Set validTransitions = new HashSet<>();
 
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/StreamTableJoinIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/StreamTableJoinIntegrationTest.java
index 772c91d7b36af..4ddab57099d33 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/StreamTableJoinIntegrationTest.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/StreamTableJoinIntegrationTest.java
@@ -79,11 +79,10 @@ public void testShouldAutoShutdownOnIncompleteMetadata() throws InterruptedExcep
         streams.setStreamThreadStateListener(listener);
         streams.start();
 
-        TestUtils.waitForCondition(listener::revokedToPendingShutdownSeen, "Did not seen thread state transited to PENDING_SHUTDOWN");
+        TestUtils.waitForCondition(listener::transitToPendingShutdownSeen, "Did not seen thread state transited to PENDING_SHUTDOWN");
 
         streams.close();
-        assertTrue(listener.createdToRevokedSeen());
-        assertTrue(listener.revokedToPendingShutdownSeen());
+        assertTrue(listener.transitToPendingShutdownSeen());
     }
 
     @Test
diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java b/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
index 46515aa84690c..a0a61b978e749 100644
--- a/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
+++ b/streams/src/test/java/org/apache/kafka/streams/integration/utils/IntegrationTestUtils.java
@@ -79,25 +79,18 @@ public class IntegrationTestUtils {
      * Records state transition for StreamThread
      */
     public static class StateListenerStub implements StreamThread.StateListener {
-        boolean startingToRevokedSeen = false;
-        boolean revokedToPendingShutdownSeen = false;
+        boolean toPendingShutdownSeen = false;
         @Override
         public void onChange(final Thread thread,
                              final ThreadStateTransitionValidator newState,
                              final ThreadStateTransitionValidator oldState) {
-            if (oldState == StreamThread.State.STARTING && newState == StreamThread.State.PARTITIONS_REVOKED) {
-                startingToRevokedSeen = true;
-            } else if (oldState == StreamThread.State.PARTITIONS_REVOKED && newState == StreamThread.State.PENDING_SHUTDOWN) {
-                revokedToPendingShutdownSeen = true;
+            if (newState == StreamThread.State.PENDING_SHUTDOWN) {
+                toPendingShutdownSeen = true;
             }
         }
 
-        public boolean revokedToPendingShutdownSeen() {
-            return revokedToPendingShutdownSeen;
-        }
-
-        public boolean createdToRevokedSeen() {
-            return startingToRevokedSeen;
+        public boolean transitToPendingShutdownSeen() {
+            return toPendingShutdownSeen;
         }
     }
 

From 0e5121747abccaa9248e4a942929797d495c1b4b Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Fri, 2 Aug 2019 17:01:10 -0700
Subject: [PATCH 52/59] fixed another minor bug

---
 .../clients/consumer/internals/SubscriptionState.java    | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
index 3a33406ee57ce..7c965ae58a55d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/SubscriptionState.java
@@ -652,8 +652,13 @@ public synchronized void resume(TopicPartition tp) {
     }
 
     synchronized void requestFailed(Set partitions, long nextRetryTimeMs) {
-        for (TopicPartition partition : partitions)
-            assignedState(partition).requestFailed(nextRetryTimeMs);
+        for (TopicPartition partition : partitions) {
+            // by the time the request failed, the assignment may no longer
+            // contain this partition any more, in which case we would just ignore.
+            final TopicPartitionState state = assignedStateOrNull(partition);
+            if (state != null)
+                state.requestFailed(nextRetryTimeMs);
+        }
     }
 
     synchronized void movePartitionToEnd(TopicPartition tp) {

From 56b54a2fa0e5ade20acef057446d8f2adb1fe9f8 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Mon, 5 Aug 2019 14:53:34 -0700
Subject: [PATCH 53/59] let revoke still be called with EAGER

---
 .../internals/ConsumerCoordinator.java        | 79 ++++++++++---------
 .../internals/ConsumerCoordinatorTest.java    |  4 +-
 2 files changed, 42 insertions(+), 41 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index f08fbb33570dd..5e97817194244 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -263,7 +263,7 @@ private void maybeUpdateJoinedSubscription(Set assignedPartition
         }
     }
 
-    private Exception maybeInvokePartitionsAssigned(final Set assignedPartitions) {
+    private Exception invokePartitionsAssigned(final Set assignedPartitions) {
         log.info("Adding newly assigned partitions: {}", Utils.join(assignedPartitions, ", "));
 
         ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
@@ -280,39 +280,35 @@ private Exception maybeInvokePartitionsAssigned(final Set assign
         return null;
     }
 
-    private Exception maybeInvokePartitionsRevoked(final Set revokedPartitions) {
-        if (!revokedPartitions.isEmpty()) {
-            log.info("Revoke previously assigned partitions {}", Utils.join(revokedPartitions, ", "));
+    private Exception invokePartitionsRevoked(final Set revokedPartitions) {
+        log.info("Revoke previously assigned partitions {}", Utils.join(revokedPartitions, ", "));
 
-            ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
-            try {
-                listener.onPartitionsRevoked(revokedPartitions);
-            } catch (WakeupException | InterruptException e) {
-                throw e;
-            } catch (Exception e) {
-                log.error("User provided listener {} failed on invocation of onPartitionsRevoked for partitions {}",
-                    listener.getClass().getName(), revokedPartitions, e);
-                return e;
-            }
+        ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
+        try {
+            listener.onPartitionsRevoked(revokedPartitions);
+        } catch (WakeupException | InterruptException e) {
+            throw e;
+        } catch (Exception e) {
+            log.error("User provided listener {} failed on invocation of onPartitionsRevoked for partitions {}",
+                listener.getClass().getName(), revokedPartitions, e);
+            return e;
         }
 
         return null;
     }
 
-    private Exception maybeInvokePartitionsLost(final String rootCause, final Set lostPartitions) {
-        if (!lostPartitions.isEmpty()) {
-            log.info("Lost previously assigned partitions {} due to {}", Utils.join(lostPartitions, ", "), rootCause);
+    private Exception invokePartitionsLost(final String rootCause, final Set lostPartitions) {
+        log.info("Lost previously assigned partitions {} due to {}", Utils.join(lostPartitions, ", "), rootCause);
 
-            ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
-            try {
-                listener.onPartitionsLost(lostPartitions);
-            } catch (WakeupException | InterruptException e) {
-                throw e;
-            } catch (Exception e) {
-                log.error("User provided listener {} failed on invocation of onPartitionsLost for partitions {}",
-                    listener.getClass().getName(), lostPartitions, e);
-                return e;
-            }
+        ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
+        try {
+            listener.onPartitionsLost(lostPartitions);
+        } catch (WakeupException | InterruptException e) {
+            throw e;
+        } catch (Exception e) {
+            log.error("User provided listener {} failed on invocation of onPartitionsLost for partitions {}",
+                listener.getClass().getName(), lostPartitions, e);
+            return e;
         }
 
         return null;
@@ -370,7 +366,7 @@ protected void onJoinComplete(int generation,
                 // assign partitions that are not yet owned
                 subscriptions.assignFromSubscribed(assignedPartitions);
 
-                firstException.compareAndSet(null, maybeInvokePartitionsAssigned(addedPartitions));
+                firstException.compareAndSet(null, invokePartitionsAssigned(addedPartitions));
 
                 break;
 
@@ -386,12 +382,14 @@ protected void onJoinComplete(int generation,
                     Utils.join(revokedPartitions, ", "));
 
                 // revoke partitions that was previously owned but no longer assigned
-                firstException.compareAndSet(null, maybeInvokePartitionsRevoked(revokedPartitions));
+                if (!revokedPartitions.isEmpty()) {
+                    firstException.compareAndSet(null, invokePartitionsRevoked(revokedPartitions));
+                }
 
                 subscriptions.assignFromSubscribed(assignedPartitions);
 
                 // add partitions that were not previously owned but are now assigned
-                firstException.compareAndSet(null, maybeInvokePartitionsAssigned(addedPartitions));
+                firstException.compareAndSet(null, invokePartitionsAssigned(addedPartitions));
 
                 // if revoked any partitions, need to re-join the group afterwards
                 if (!revokedPartitions.isEmpty()) {
@@ -646,17 +644,20 @@ protected void onJoinPrepare(int generation, String memberId) {
             memberId.equals(Generation.NO_GENERATION.memberId)) {
 
             revokedPartitions = new HashSet<>(subscriptions.assignedPartitions());
-            exception = maybeInvokePartitionsLost(
-                "generation has been reset because consumer has been kicked out of the group",
-                revokedPartitions);
 
-            subscriptions.assignFromSubscribed(Collections.emptySet());
+            if (!revokedPartitions.isEmpty()) {
+                exception = invokePartitionsLost(
+                    "generation has been reset because consumer has been kicked out of the group",
+                    revokedPartitions);
+
+                subscriptions.assignFromSubscribed(Collections.emptySet());
+            }
         } else {
             switch (protocol) {
                 case EAGER:
                     // revoke all partitions
                     revokedPartitions = new HashSet<>(subscriptions.assignedPartitions());
-                    exception = maybeInvokePartitionsRevoked(revokedPartitions);
+                    exception = invokePartitionsRevoked(revokedPartitions);
 
                     subscriptions.assignFromSubscribed(Collections.emptySet());
 
@@ -670,7 +671,7 @@ protected void onJoinPrepare(int generation, String memberId) {
                         .collect(Collectors.toSet());
 
                     if (!revokedPartitions.isEmpty()) {
-                        exception = maybeInvokePartitionsRevoked(revokedPartitions);
+                        exception = invokePartitionsRevoked(revokedPartitions);
 
                         ownedPartitions.removeAll(revokedPartitions);
                         subscriptions.assignFromSubscribed(ownedPartitions);
@@ -697,9 +698,9 @@ private void resetAssignment(boolean lostPartitions) {
             Exception e;
 
             if (lostPartitions) {
-                e = maybeInvokePartitionsLost("reset generation", droppedPartitions);
+                e = invokePartitionsLost("reset generation", droppedPartitions);
             } else {
-                e = maybeInvokePartitionsRevoked(droppedPartitions);
+                e = invokePartitionsRevoked(droppedPartitions);
             }
 
             subscriptions.assignFromSubscribed(Collections.emptySet());
@@ -752,7 +753,7 @@ public boolean rejoinNeededOrPending() {
             if (!noLongerExistingPartitions.isEmpty()) {
                 ownedPartitions.removeAll(noLongerExistingPartitions);
 
-                Exception e = maybeInvokePartitionsLost("topic metadata has changed and therefore some topics may not exist any more",
+                Exception e = invokePartitionsLost("topic metadata has changed and therefore some topics may not exist any more",
                     noLongerExistingPartitions);
 
                 subscriptions.assignFromSubscribed(ownedPartitions);
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index 706463ff6dda4..23fdc1a89cb1c 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -102,6 +102,7 @@
 import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
 import static org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.RebalanceProtocol.COOPERATIVE;
+import static org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.RebalanceProtocol.EAGER;
 import static org.apache.kafka.test.TestUtils.toSet;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -555,8 +556,7 @@ public void testOutdatedCoordinatorAssignment() {
         assertFalse(coordinator.rejoinNeededOrPending());
         assertEquals(toSet(newAssignment), subscriptions.assignedPartitions());
         assertEquals(toSet(newSubscription), subscriptions.groupSubscription());
-        assertEquals(0, rebalanceListener.revokedCount);
-        assertNull(rebalanceListener.revoked);
+        assertEquals(protocol == EAGER ? 1 : 0, rebalanceListener.revokedCount);
         assertEquals(1, rebalanceListener.assignedCount);
         assertEquals(assigned, rebalanceListener.assigned);
     }

From 8220d8edec4d37b6800a0f057bfe2c83a1a599ff Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Tue, 6 Aug 2019 15:57:14 -0700
Subject: [PATCH 54/59] github comments

---
 .../consumer/internals/ConsumerCoordinator.java    | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 5e97817194244..de0e0126df93c 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -633,16 +633,16 @@ protected void onJoinPrepare(int generation, String memberId) {
         // commit offsets prior to rebalance if auto-commit enabled
         maybeAutoCommitOffsetsSync(time.timer(rebalanceConfig.rebalanceTimeoutMs));
 
-        // execute the user's callback before rebalance
-        final Set revokedPartitions;
-
-        // note we should only change the assignment AFTER the callback is triggered
-        // so that users can still access the pre-assigned partitions to commit offsets etc.
+        // the generation / member-id can possibly be reset by the heartbeat thread
+        // upon getting errors or heartbeat timeouts; in this case whatever is previously
+        // owned partitions would be lost, we should trigger the callback and cleanup the assignment;
+        // otherwise we can proceed normally and revoke the partitions depending on the protocol,
+        // and in that case we should only change the assignment AFTER the revoke callback is triggered
+        // so that users can still access the previously owned partitions to commit offsets etc.
         Exception exception = null;
-
+        final Set revokedPartitions;
         if (generation == Generation.NO_GENERATION.generationId &&
             memberId.equals(Generation.NO_GENERATION.memberId)) {
-
             revokedPartitions = new HashSet<>(subscriptions.assignedPartitions());
 
             if (!revokedPartitions.isEmpty()) {

From 5ed5a0a58140bbbacd848c1b976e0c75c114b430 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Tue, 6 Aug 2019 16:05:14 -0700
Subject: [PATCH 55/59] github comments

---
 .../clients/consumer/internals/ConsumerCoordinator.java      | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index de0e0126df93c..32e25bc0045e0 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -360,7 +360,6 @@ protected void onJoinComplete(int generation,
         Set addedPartitions = new HashSet<>(assignedPartitions);
         addedPartitions.removeAll(ownedPartitions);
 
-        // note that we should only change the assignment AFTER we've triggered the rebalance callback
         switch (protocol) {
             case EAGER:
                 // assign partitions that are not yet owned
@@ -381,7 +380,9 @@ protected void onJoinComplete(int generation,
                     Utils.join(addedPartitions, ", "),
                     Utils.join(revokedPartitions, ", "));
 
-                // revoke partitions that was previously owned but no longer assigned
+                // revoke partitions that was previously owned but no longer assigned;
+                // note that we should only change the assignment AFTER we've triggered
+                // the revoke callback
                 if (!revokedPartitions.isEmpty()) {
                     firstException.compareAndSet(null, invokePartitionsRevoked(revokedPartitions));
                 }

From 58913e46fd2fd3e3860ecb68d7212dd4f0c0d203 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Tue, 6 Aug 2019 22:12:46 -0700
Subject: [PATCH 56/59] github comments

---
 .../internals/ConsumerCoordinator.java        | 38 ++++---------------
 1 file changed, 8 insertions(+), 30 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 32e25bc0045e0..3a6c19624de37 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -648,7 +648,7 @@ protected void onJoinPrepare(int generation, String memberId) {
 
             if (!revokedPartitions.isEmpty()) {
                 exception = invokePartitionsLost(
-                    "generation has been reset because consumer has been kicked out of the group",
+                    "generation has been reset since consumer is no longer part of the group",
                     revokedPartitions);
 
                 subscriptions.assignFromSubscribed(Collections.emptySet());
@@ -691,18 +691,17 @@ protected void onJoinPrepare(int generation, String memberId) {
         }
     }
 
-    private void resetAssignment(boolean lostPartitions) {
+    /**
+     * @throws KafkaException if the rebalance callback throws exception
+     */
+    @Override
+    synchronized void resetGenerationOnLeaveGroup() {
+        // we can reset assignment upon leaving group and trigger the callback as well
         Set droppedPartitions = new HashSet<>(subscriptions.assignedPartitions());
 
         if (subscriptions.partitionsAutoAssigned() && !droppedPartitions.isEmpty()) {
 
-            Exception e;
-
-            if (lostPartitions) {
-                e = invokePartitionsLost("reset generation", droppedPartitions);
-            } else {
-                e = invokePartitionsRevoked(droppedPartitions);
-            }
+            final Exception e = invokePartitionsRevoked(droppedPartitions);
 
             subscriptions.assignFromSubscribed(Collections.emptySet());
 
@@ -710,28 +709,7 @@ private void resetAssignment(boolean lostPartitions) {
                 throw new KafkaException("User rebalance callback throws an error", e);
             }
         }
-    }
 
-    /**
-     * @throws KafkaException if the rebalance callback throws exception
-     */
-    @Override
-    synchronized void resetGenerationOnResponseError(ApiKeys api, Errors error) {
-        // heartbeat error handling is executed by heartbeat thread, in which case
-        // we would not drop partitions and trigger rebalance callback as it should
-        // only be triggered by the caller thread
-        if (api != ApiKeys.HEARTBEAT) {
-            resetAssignment(true);
-        }
-        super.resetGenerationOnResponseError(api, error);
-    }
-
-    /**
-     * @throws KafkaException if the rebalance callback throws exception
-     */
-    @Override
-    synchronized void resetGenerationOnLeaveGroup() {
-        resetAssignment(false);
         super.resetGenerationOnLeaveGroup();
     }
 

From 6d07d42b9b6f3a60478f43c4465052a7ce6a8111 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Wed, 7 Aug 2019 17:05:11 -0700
Subject: [PATCH 57/59] adding more debugging

---
 .../clients/consumer/internals/ConsumerCoordinator.java    | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 3a6c19624de37..1e5c4e7cea138 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -319,6 +319,8 @@ protected void onJoinComplete(int generation,
                                   String memberId,
                                   String assignmentStrategy,
                                   ByteBuffer assignmentBuffer) {
+        log.debug("Executing onJoinComplete with generation {} and memberId {}", generation, memberId);
+
         // only the leader is responsible for monitoring for metadata changes (i.e. partition changes)
         if (!isLeader)
             assignmentSnapshot = null;
@@ -631,6 +633,7 @@ private void validateCooperativeAssignment(final Map
Date: Wed, 7 Aug 2019 18:50:58 -0700
Subject: [PATCH 58/59] do not lost partitions upon metadata change

---
 .../consumer/ConsumerRebalanceListener.java   |  8 +++-
 .../kafka/clients/consumer/KafkaConsumer.java |  4 +-
 .../internals/AbstractCoordinator.java        | 48 +++++++------------
 .../internals/ConsumerCoordinator.java        | 43 ++++-------------
 .../internals/ConsumerCoordinatorTest.java    |  9 ++--
 5 files changed, 38 insertions(+), 74 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
index 38e5d31f6019a..6046ef954682d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/ConsumerRebalanceListener.java
@@ -16,6 +16,7 @@
  */
 package org.apache.kafka.clients.consumer;
 
+import java.time.Duration;
 import java.util.Collection;
 
 import org.apache.kafka.common.TopicPartition;
@@ -118,7 +119,9 @@ public interface ConsumerRebalanceListener {
     /**
      * A callback method the user can implement to provide handling of offset commits to a customized store on the start
      * of a rebalance operation. This method will be called before a rebalance operation starts and after the consumer
-     * stops fetching data. It is recommended that offsets should be committed in this callback to either Kafka or a
+     * stops fetching data. It can also be called when consumer is being closed ({@link KafkaConsumer#close(Duration)})
+     * or is unsubscribing ({@link KafkaConsumer#unsubscribe()}).
+     * It is recommended that offsets should be committed in this callback to either Kafka or a
      * custom offset store to prevent duplicate data.
      * 

      * For examples on usage of this API, see Usage Examples section of {@link KafkaConsumer KafkaConsumer}
@@ -170,7 +173,8 @@ public interface ConsumerRebalanceListener {
      * to other consumers during a rebalance event. However, during exceptional scenarios when the consumer realized that it
      * does not own this partition any longer, i.e. not revoked via a normal rebalance event, then this method would be invoked.
      * 

-     * For example, this function is called if a consumer's session timeout has expired.
+     * For example, this function is called if a consumer's session timeout has expired, or if a fatal error has been
+     * received indicating the consumer is no longer part of the group.
      * 

      * By default it will just trigger {@link ConsumerRebalanceListener#onPartitionsRevoked}; for users who want to distinguish
      * the handling logic of revoked partitions v.s. lost partitions, they can override the default implementation.
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index a3cbec975ede9..f4af61860bb56 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -1054,8 +1054,10 @@ public void unsubscribe() {
         try {
             fetcher.clearBufferedDataForUnassignedPartitions(Collections.emptySet());
             this.subscriptions.unsubscribe();
-            if (this.coordinator != null)
+            if (this.coordinator != null) {
+                this.coordinator.onLeaveGroup();
                 this.coordinator.maybeLeaveGroup("the consumer unsubscribed from all topics");
+            }
             log.info("Unsubscribed all topics or patterns and assigned partitions");
         } finally {
             release();
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 148f33f089016..52214a187eca2 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -202,6 +202,11 @@ protected abstract void onJoinComplete(int generation,
                                            String protocol,
                                            ByteBuffer memberAssignment);
 
+    /**
+     * Invoked prior to each leave group event. This is typically used to cleanup assigned partitions
+     */
+    protected void onLeaveGroup() {}
+
     /**
      * Visible for testing.
      *
@@ -839,6 +844,7 @@ protected void close(Timer timer) {
             // needs this lock to complete and terminate after close flag is set.
             synchronized (this) {
                 if (rebalanceConfig.leaveGroupOnClose) {
+                    onLeaveGroup();
                     maybeLeaveGroup("the consumer is being closed");
                 }
 
@@ -854,51 +860,33 @@ protected void close(Timer timer) {
         }
     }
 
-    public synchronized RequestFuture maybeLeaveGroup(String leaveReason) {
-        return maybeLeaveGroup(leaveReason, false);
-    }
-
     /**
-     * Leave the current group and reset local generation/memberId.
-     * @param leaveReason reason to attempt leaving the group
-     * @throws KafkaException if the callback throws exception
+     * @throws KafkaException if the rebalance callback throws exception
      */
-    public synchronized RequestFuture maybeLeaveGroup(String leaveReason, boolean dueToHeartbeatExpiration) {
-        LeaveGroupRequest.Builder request = null;
+    public synchronized RequestFuture maybeLeaveGroup(String leaveReason) {
+        RequestFuture future = null;
+
         // Starting from 2.3, only dynamic members will send LeaveGroupRequest to the broker,
         // consumer with valid group.instance.id is viewed as static member that never sends LeaveGroup,
         // and the membership expiration is only controlled by session timeout.
         if (isDynamicMember() && !coordinatorUnknown() &&
-                state != MemberState.UNJOINED && generation.hasMemberId()) {
+            state != MemberState.UNJOINED && generation.hasMemberId()) {
             // this is a minimal effort attempt to leave the group. we do not
             // attempt any resending if the request fails or times out.
             log.info("Member {} sending LeaveGroup request to coordinator {} due to {}",
-                     generation.memberId, coordinator, leaveReason);
-            request = new LeaveGroupRequest.Builder(
+                generation.memberId, coordinator, leaveReason);
+            LeaveGroupRequest.Builder request = new LeaveGroupRequest.Builder(
                 rebalanceConfig.groupId,
                 Collections.singletonList(new MemberIdentity().setMemberId(generation.memberId))
             );
-        }
 
-        // we need to reset generation first in order to trigger the rebalance callback if necessary, before sending
-        // the leave group which may trigger the rebalance; if leave-group is caused by heartbeat poll expiration
-        // at the heartbeat thread, we should not trigger the rebalance callback and will wait until the caller thread
-        // calling consumer.poll to trigger it.
-        if (dueToHeartbeatExpiration) {
-            resetGeneration();
-        } else {
-            resetGenerationOnLeaveGroup();
-        }
-
-        if (request != null) {
-            RequestFuture future = client.send(coordinator, request)
-                .compose(new LeaveGroupResponseHandler());
+            future = client.send(coordinator, request).compose(new LeaveGroupResponseHandler());
             client.pollNoWakeup();
-
-            return future;
         }
 
-        return null;
+        resetGenerationOnLeaveGroup();
+
+        return future;
     }
 
     protected boolean isDynamicMember() {
@@ -1141,7 +1129,7 @@ public void run() {
                                                     "the poll loop is spending too much time processing messages. " +
                                                     "You can address this either by increasing max.poll.interval.ms or by reducing " +
                                                     "the maximum size of batches returned in poll() with max.poll.records.";
-                            maybeLeaveGroup(leaveReason, true);
+                            maybeLeaveGroup(leaveReason);
                         } else if (!heartbeat.shouldHeartbeat(now)) {
                             // poll again after waiting for the retry backoff in case the heartbeat failed or the
                             // coordinator disconnected
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 1e5c4e7cea138..40b3c4ac5051d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -297,8 +297,8 @@ private Exception invokePartitionsRevoked(final Set revokedParti
         return null;
     }
 
-    private Exception invokePartitionsLost(final String rootCause, final Set lostPartitions) {
-        log.info("Lost previously assigned partitions {} due to {}", Utils.join(lostPartitions, ", "), rootCause);
+    private Exception invokePartitionsLost(final Set lostPartitions) {
+        log.info("Lost previously assigned partitions {}", Utils.join(lostPartitions, ", "));
 
         ConsumerRebalanceListener listener = subscriptions.rebalanceListener();
         try {
@@ -650,9 +650,9 @@ protected void onJoinPrepare(int generation, String memberId) {
             revokedPartitions = new HashSet<>(subscriptions.assignedPartitions());
 
             if (!revokedPartitions.isEmpty()) {
-                exception = invokePartitionsLost(
-                    "generation has been reset since consumer is no longer part of the group",
-                    revokedPartitions);
+                log.info("Giving away all assigned partitions as lost since generation has been reset," +
+                    "indicating that consumer is no longer part of the group");
+                exception = invokePartitionsLost(revokedPartitions);
 
                 subscriptions.assignFromSubscribed(Collections.emptySet());
             }
@@ -694,16 +694,12 @@ protected void onJoinPrepare(int generation, String memberId) {
         }
     }
 
-    /**
-     * @throws KafkaException if the rebalance callback throws exception
-     */
     @Override
-    synchronized void resetGenerationOnLeaveGroup() {
-        // we can reset assignment upon leaving group and trigger the callback as well
+    public void onLeaveGroup() {
+        // we should reset assignment and trigger the callback before leaving group
         Set droppedPartitions = new HashSet<>(subscriptions.assignedPartitions());
 
         if (subscriptions.partitionsAutoAssigned() && !droppedPartitions.isEmpty()) {
-
             final Exception e = invokePartitionsRevoked(droppedPartitions);
 
             subscriptions.assignFromSubscribed(Collections.emptySet());
@@ -712,8 +708,6 @@ synchronized void resetGenerationOnLeaveGroup() {
                 throw new KafkaException("User rebalance callback throws an error", e);
             }
         }
-
-        super.resetGenerationOnLeaveGroup();
     }
 
     /**
@@ -726,29 +720,8 @@ public boolean rejoinNeededOrPending() {
 
         // we need to rejoin if we performed the assignment and metadata has changed;
         // also for those owned-but-no-longer-existed partitions we should drop them as lost
-        if (assignmentSnapshot != null && !assignmentSnapshot.matches(metadataSnapshot)) {
-            Set ownedPartitions = new HashSet<>(subscriptions.assignedPartitions());
-            Set noLongerExistingPartitions = ownedPartitions.stream()
-                .filter(tp -> !metadataSnapshot.partitionsPerTopic().containsKey(tp.topic()))
-                .collect(Collectors.toSet());
-
-            if (!noLongerExistingPartitions.isEmpty()) {
-                ownedPartitions.removeAll(noLongerExistingPartitions);
-
-                Exception e = invokePartitionsLost("topic metadata has changed and partitions " +
-                        noLongerExistingPartitions + " do not exist any more, owned partitions left are " +
-                        ownedPartitions,
-                    noLongerExistingPartitions);
-
-                subscriptions.assignFromSubscribed(ownedPartitions);
-
-                if (e != null) {
-                    throw new KafkaException("User rebalance callback throws an error", e);
-                }
-            }
-
+        if (assignmentSnapshot != null && !assignmentSnapshot.matches(metadataSnapshot))
             return true;
-        }
 
         // we need to join if our subscription has changed since the last join
         if (joinedSubscription != null && !joinedSubscription.equals(subscriptions.subscription())) {
diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
index 23fdc1a89cb1c..91d3529ea10e4 100644
--- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
+++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java
@@ -705,11 +705,9 @@ public boolean matches(AbstractRequest body) {
 
         coordinator.poll(time.timer(Long.MAX_VALUE));
 
-        Collection lost = Collections.singleton(t2p);
         revoked = getRevoked(newAssigned, oldAssigned);
-        revoked.removeAll(lost);
-        assertEquals(protocol == COOPERATIVE, revoked.isEmpty());
-        revokedCount += revoked.isEmpty() ? 0 : 1;
+        assertFalse(revoked.isEmpty());
+        revokedCount += 1;
         Collection added = getAdded(newAssigned, oldAssigned);
 
         assertFalse(coordinator.rejoinNeededOrPending());
@@ -719,8 +717,7 @@ public boolean matches(AbstractRequest body) {
         assertEquals(revoked.isEmpty() ? null : revoked, rebalanceListener.revoked);
         assertEquals(3, rebalanceListener.assignedCount);
         assertEquals(added, rebalanceListener.assigned);
-        assertEquals(1, rebalanceListener.lostCount);
-        assertEquals(lost, rebalanceListener.lost);
+        assertEquals(0, rebalanceListener.lostCount);
     }
 
     @Test

From 6041a792f58b0b9a38983a60e052e9018319a6e6 Mon Sep 17 00:00:00 2001
From: Guozhang Wang 
Date: Thu, 8 Aug 2019 14:28:00 -0700
Subject: [PATCH 59/59] github comments

---
 .../org/apache/kafka/clients/consumer/KafkaConsumer.java  | 2 +-
 .../clients/consumer/internals/AbstractCoordinator.java   | 8 +++++---
 .../clients/consumer/internals/ConsumerCoordinator.java   | 2 +-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
index f4af61860bb56..be3e176c2eeb9 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java
@@ -1055,7 +1055,7 @@ public void unsubscribe() {
             fetcher.clearBufferedDataForUnassignedPartitions(Collections.emptySet());
             this.subscriptions.unsubscribe();
             if (this.coordinator != null) {
-                this.coordinator.onLeaveGroup();
+                this.coordinator.onLeavePrepare();
                 this.coordinator.maybeLeaveGroup("the consumer unsubscribed from all topics");
             }
             log.info("Unsubscribed all topics or patterns and assigned partitions");
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
index 52214a187eca2..4c71a89f3bdcd 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java
@@ -203,9 +203,11 @@ protected abstract void onJoinComplete(int generation,
                                            ByteBuffer memberAssignment);
 
     /**
-     * Invoked prior to each leave group event. This is typically used to cleanup assigned partitions
+     * Invoked prior to each leave group event. This is typically used to cleanup assigned partitions;
+     * note it is triggered by the consumer's API caller thread (i.e. background heartbeat thread would
+     * not trigger it even if it tries to force leaving group upon heartbeat session expiration)
      */
-    protected void onLeaveGroup() {}
+    protected void onLeavePrepare() {}
 
     /**
      * Visible for testing.
@@ -844,7 +846,7 @@ protected void close(Timer timer) {
             // needs this lock to complete and terminate after close flag is set.
             synchronized (this) {
                 if (rebalanceConfig.leaveGroupOnClose) {
-                    onLeaveGroup();
+                    onLeavePrepare();
                     maybeLeaveGroup("the consumer is being closed");
                 }
 
diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
index 40b3c4ac5051d..a3aaface7822d 100644
--- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
+++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java
@@ -695,7 +695,7 @@ protected void onJoinPrepare(int generation, String memberId) {
     }
 
     @Override
-    public void onLeaveGroup() {
+    public void onLeavePrepare() {
         // we should reset assignment and trigger the callback before leaving group
         Set droppedPartitions = new HashSet<>(subscriptions.assignedPartitions());