diff --git a/clients/src/main/java/org/apache/kafka/clients/Metadata.java b/clients/src/main/java/org/apache/kafka/clients/Metadata.java index 6c663cfac93b8..c8fa66b7e3b62 100644 --- a/clients/src/main/java/org/apache/kafka/clients/Metadata.java +++ b/clients/src/main/java/org/apache/kafka/clients/Metadata.java @@ -124,15 +124,27 @@ public synchronized void add(String topic) { } } + /** + * Return the next time when the current cluster info can be updated (i.e., backoff time has elapsed). + * + * @param nowMs current time in ms + * @return remaining time in ms till the cluster info can be updated again + */ + public synchronized long timeToAllowUpdate(long nowMs) { + return Math.max(this.lastRefreshMs + this.refreshBackoffMs - nowMs, 0); + } + /** * The next time to update the cluster info is the maximum of the time the current info will expire and the time the * current info can be updated (i.e. backoff time has elapsed); If an update has been request then the expiry time * is now + * + * @param nowMs current time in ms + * @return remaining time in ms till updating the cluster info */ - public synchronized long timeToNextUpdate(long nowMs) { + public synchronized long timeToNextUpdate(long nowMs) { long timeToExpire = needUpdate ? 0 : Math.max(this.lastSuccessfulRefreshMs + this.metadataExpireMs - nowMs, 0); - long timeToAllowUpdate = this.lastRefreshMs + this.refreshBackoffMs - nowMs; - return Math.max(timeToExpire, timeToAllowUpdate); + return Math.max(timeToExpire, timeToAllowUpdate(nowMs)); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index f9b77e921cd30..8f25d6e8eee5b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -328,6 +328,16 @@ public boolean poll(final long timeoutMs) { // we need to ensure that the metadata is fresh before joining initially. This ensures // that we have matched the pattern against the cluster's topics at least once before joining. if (subscriptions.hasPatternSubscription()) { + // For consumer group that uses pattern-based subscription, after a topic is created, + // any consumer that discovers the topic after metadata refresh can trigger rebalance + // across the entire consumer group. Multiple rebalances can be triggered after one topic + // creation if consumers refresh metadata at vastly different times. We can significantly + // reduce the number of rebalances caused by single topic creation by asking consumer to + // refresh metadata before re-joining the group as long as the refresh backoff time has + // passed. + if (this.metadata.timeToAllowUpdate(currentTime) == 0) { + this.metadata.requestUpdate(); + } if (!client.ensureFreshMetadata(remainingTimeAtLeastZero(timeoutMs, elapsed))) { return false; } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index ba392c6f4cbf7..cec56b07e0002 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -513,6 +513,50 @@ public boolean matches(AbstractRequest body) { assertEquals(newAssignmentSet, rebalanceListener.assigned); } + @Test + public void testForceMetadataRefreshForPatternSubscriptionDuringRebalance() { + // Set up a non-leader consumer with pattern subscription and a cluster containing one topic matching the + // pattern. + final String consumerId = "consumer"; + + subscriptions.subscribe(Pattern.compile(".*"), rebalanceListener); + metadata.update(TestUtils.singletonCluster(topic1, 1), Collections.emptySet(), + time.milliseconds()); + assertEquals(singleton(topic1), subscriptions.subscription()); + + client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); + coordinator.ensureCoordinatorReady(Long.MAX_VALUE); + + // Instrument the test so that metadata will contain two topics after next refresh. + client.prepareMetadataUpdate(cluster, Collections.emptySet()); + + client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE)); + client.prepareResponse(new MockClient.RequestMatcher() { + @Override + public boolean matches(AbstractRequest body) { + SyncGroupRequest sync = (SyncGroupRequest) body; + return sync.memberId().equals(consumerId) && + sync.generationId() == 1 && + sync.groupAssignment().isEmpty(); + } + }, syncGroupResponse(singletonList(t1p), Errors.NONE)); + + partitionAssignor.prepare(singletonMap(consumerId, singletonList(t1p))); + + // This will trigger rebalance. + coordinator.poll(Long.MAX_VALUE); + + // Make sure that the metadata was refreshed during the rebalance and thus subscriptions now contain two topics. + final Set updatedSubscriptionSet = new HashSet<>(Arrays.asList(topic1, topic2)); + assertEquals(updatedSubscriptionSet, subscriptions.subscription()); + + // Refresh the metadata again. Since there have been no changes since the last refresh, it won't trigger + // rebalance again. + metadata.requestUpdate(); + client.poll(Long.MAX_VALUE, time.milliseconds()); + assertFalse(coordinator.rejoinNeededOrPending()); + } + @Test public void testWakeupDuringJoin() { final String consumerId = "leader";